c4a59d7bf2
Some checks failed
Auto-tag / Create date-tag (push) Waiting to run
CI / Backend (.NET 8) (push) Waiting to run
CI / Web (React + Vite) (push) Waiting to run
CI / POS (WPF, Windows) (push) Waiting to run
Docker API / Build + push API (push) Waiting to run
Docker API / Deploy API on stage (push) Blocked by required conditions
Docker Public / Build + push Public (test + latest) (push) Has been cancelled
Docker Web / Build + push Web (push) Has been cancelled
Docker Public / Deploy :test → test.food-market.kz (push) Has been cancelled
Docker Web / Deploy Web on stage (push) Has been cancelled
05.10.2026 общий postgres-инстанс с чужими боевыми базами (sklad, purchase, calcman, calcman_staging, makesales + мусорная старая food_market) вынесен из compose прода food-market в отдельный compose-проект shared-postgres (контейнер shared-postgres, порт 5434, том food-market-postgres-data external). Прод food-market поднимается в собственный контейнер food-market-prod-postgres (порт 5437, том food-market-prod-postgres-data). Простой при разделении — 12с, соседи живы. См. docs/shared-postgres-followup-progress.md. Этот коммит синхронизирует репозиторий, чтобы следующий деплой из main не воскресил старую раскладку: - deploy/docker-compose.yml: container_name/port/volume приведены к прод-файлу (food-market-prod-postgres, 127.0.0.1:5437, food-market-prod-postgres-data). - deploy/shared-postgres/ (новое): документационная копия соседнего compose'а, с явной пометкой «food-market им не управляет». - deploy/food-market-backup.sh / anonymize-prod.sh / db-schema-diff.sh: defaults FM_PG_CONTAINER / FM_PROD_CONT = food-market-prod-postgres. - docs/RUNBOOK.md: новый раздел «Postgres на prod-vm» с таблицей трёх инстансов и явным запретом трогать shared-postgres из наших скриптов. - docs/backup-restore.md: контейнер в таблице и примерах — новый. Guards (не в репе — живут в ~/deploy-stage.sh и ~/quality-watchdog.sh) расширены симметрично: стенду запрещены shared-postgres, food-market-prod- postgres[-data], 5434/5437; проду — food-market-postgres-data (чужие данные). Проверка: docker compose config/ps на prod-vm shared-postgres не упоминают, grep по репе — ни один скрипт не делает docker exec в shared-postgres.
190 lines
8.5 KiB
Bash
Executable file
190 lines
8.5 KiB
Bash
Executable file
#!/usr/bin/env bash
|
||
#
|
||
# Sprint 22: создание anonymized stage-dump'a из прод-БД.
|
||
#
|
||
# Алгоритм:
|
||
# 1. pg_dump прода в кастомном формате (-Fc)
|
||
# 2. pg_restore во временную staging-БД (`food_market_anon`)
|
||
# 3. UPDATE PII-полей в staging:
|
||
# - email → user{N}@example.kz
|
||
# - phone → +7700111{N:04}
|
||
# - passwordHash → один общий тестовый hash для пароля "Test12345!"
|
||
# - IIN / БИН → синтетические но валидные
|
||
# - имена/адреса → "Test Tester {N}" / "Тестовый адрес {N}"
|
||
# 4. pg_dump anonymized → .sql.gz файл
|
||
# 5. Удалить staging-БД
|
||
#
|
||
# Результат используется на dev-vm для воспроизведения багов на реальном
|
||
# объёме данных без утечки persistent PII.
|
||
#
|
||
# Usage:
|
||
# deploy/anonymize-prod.sh [--source <conn-uri>] [--target <conn-uri>]
|
||
# [--out <file>] [--dry-run]
|
||
#
|
||
# Default source: ssh prod docker exec food-market-prod-postgres pg_dump
|
||
# Default target: local postgres@14 with TEMP DB food_market_anon
|
||
# Default out: /home/nns/food-market-anon-YYYYMMDD.sql.gz
|
||
#
|
||
# ВАЖНО: контейнер food-market-prod-postgres (с 2026-10-05). НЕ shared-postgres —
|
||
# тот принадлежит соседнему compose-проекту с чужими боевыми базами (sklad,
|
||
# purchase, calcman, calcman_staging, makesales). См. docs/shared-postgres-followup-progress.md.
|
||
|
||
set -uo pipefail
|
||
|
||
SOURCE_HOST="${FM_PROD_HOST:-nns@admin.food-market.kz}"
|
||
SOURCE_CONTAINER="${FM_PROD_CONT:-food-market-prod-postgres}"
|
||
SOURCE_DB="${FM_PG_DB:-food_market}"
|
||
SOURCE_USER="${FM_PG_USER:-food_market}"
|
||
LOCAL_USER="${PGUSER:-nns}"
|
||
LOCAL_HOST="${PGHOST:-localhost}"
|
||
LOCAL_PORT="${PGPORT:-5432}"
|
||
TARGET_DB="food_market_anon_$$"
|
||
OUT="${FM_ANON_OUT:-$HOME/food-market-anon-$(date +%Y%m%d-%H%M%S).sql.gz}"
|
||
DRY_RUN=0
|
||
|
||
while [[ $# -gt 0 ]]; do
|
||
case "$1" in
|
||
--source-host) SOURCE_HOST="$2"; shift 2 ;;
|
||
--source-container) SOURCE_CONTAINER="$2"; shift 2 ;;
|
||
--out) OUT="$2"; shift 2 ;;
|
||
--dry-run) DRY_RUN=1; shift ;;
|
||
--help|-h) grep -E '^#( |$)' "$0" | sed 's/^# \?//'; exit 0 ;;
|
||
*) echo "Unknown: $1" >&2; exit 2 ;;
|
||
esac
|
||
done
|
||
|
||
log() { echo "[$(date -Iseconds)] $*"; }
|
||
|
||
run() {
|
||
if [[ $DRY_RUN -eq 1 ]]; then echo "[dry-run] $*";
|
||
else echo "[exec] $*"; "$@"; fi
|
||
}
|
||
|
||
cleanup() {
|
||
log "cleanup: drop $TARGET_DB"
|
||
if [[ $DRY_RUN -eq 0 ]]; then
|
||
psql -h "$LOCAL_HOST" -p "$LOCAL_PORT" -U "$LOCAL_USER" -d postgres \
|
||
-c "DROP DATABASE IF EXISTS $TARGET_DB;" 2>/dev/null || true
|
||
fi
|
||
}
|
||
trap cleanup EXIT
|
||
|
||
# ── 1. pg_dump из прода ──────────────────────────────────────────────
|
||
DUMP="/tmp/food-market-prod-$$.dump"
|
||
log "Step 1/5: pg_dump from $SOURCE_HOST → $DUMP (Fc format)"
|
||
if [[ $DRY_RUN -eq 1 ]]; then
|
||
log "[dry-run] ssh $SOURCE_HOST docker exec $SOURCE_CONTAINER pg_dump -Fc -U $SOURCE_USER -d $SOURCE_DB > $DUMP"
|
||
else
|
||
ssh -o ConnectTimeout=10 "$SOURCE_HOST" \
|
||
"docker exec $SOURCE_CONTAINER pg_dump -Fc --no-owner --no-privileges -U $SOURCE_USER -d $SOURCE_DB" \
|
||
> "$DUMP" || { log "FAIL pg_dump"; exit 1; }
|
||
log "dump size: $(du -h "$DUMP" | cut -f1)"
|
||
fi
|
||
|
||
# ── 2. Создать temp-БД и restore ─────────────────────────────────────
|
||
log "Step 2/5: create $TARGET_DB + pg_restore"
|
||
if [[ $DRY_RUN -eq 0 ]]; then
|
||
psql -h "$LOCAL_HOST" -p "$LOCAL_PORT" -U "$LOCAL_USER" -d postgres \
|
||
-c "CREATE DATABASE $TARGET_DB;" || { log "FAIL create db"; exit 1; }
|
||
pg_restore -h "$LOCAL_HOST" -p "$LOCAL_PORT" -U "$LOCAL_USER" -d "$TARGET_DB" \
|
||
--no-owner --no-privileges "$DUMP" 2>/dev/null || \
|
||
log "(некритично) pg_restore warnings — продолжаем"
|
||
fi
|
||
|
||
# ── 3. Anonymize PII ─────────────────────────────────────────────────
|
||
# Hash для пароля "Test12345!" — генерируется через идентичный
|
||
# алгоритм ASP.NET Identity (PBKDF2 SHA256, 10000 iter). Чтобы не
|
||
# вводить криптографию в скрипт — берём заранее известный hash.
|
||
# Сгенерировать новый можно через `dotnet run --project dev-tools/hash-pass.cs`.
|
||
TEST_PASS_HASH='AQAAAAIAAYagAAAAEHJsxbHF3MoBGSe+1bktB4O9aERPI4j5Jt6w0iN4dCqU/5jL+i5xT8E+UEqcVf0Vqg=='
|
||
|
||
log "Step 3/5: anonymize PII in $TARGET_DB"
|
||
if [[ $DRY_RUN -eq 0 ]]; then
|
||
psql -h "$LOCAL_HOST" -p "$LOCAL_PORT" -U "$LOCAL_USER" -d "$TARGET_DB" -v ON_ERROR_STOP=1 <<SQL
|
||
-- 1. AspNetUsers (Identity): email + phone + password hash + security stamp
|
||
WITH numbered AS (
|
||
SELECT "Id", row_number() OVER (ORDER BY "Id") AS rn FROM "AspNetUsers"
|
||
)
|
||
UPDATE "AspNetUsers" u
|
||
SET
|
||
"Email" = 'user' || n.rn || '@example.kz',
|
||
"NormalizedEmail" = upper('user' || n.rn || '@example.kz'),
|
||
"UserName" = 'user' || n.rn || '@example.kz',
|
||
"NormalizedUserName" = upper('user' || n.rn || '@example.kz'),
|
||
"PhoneNumber" = '+7700111' || lpad(n.rn::text, 4, '0'),
|
||
"PasswordHash" = '$TEST_PASS_HASH',
|
||
"SecurityStamp" = encode(gen_random_bytes(16), 'hex'),
|
||
"ConcurrencyStamp" = gen_random_uuid()::text
|
||
FROM numbered n
|
||
WHERE u."Id" = n."Id";
|
||
|
||
-- 2. employees — рабочий email/телефон + полное имя
|
||
WITH numbered AS (
|
||
SELECT "Id", row_number() OVER (ORDER BY "Id") AS rn FROM employees
|
||
)
|
||
UPDATE employees e
|
||
SET
|
||
"Email" = CASE WHEN e."Email" IS NOT NULL THEN 'emp' || n.rn || '@example.kz' END,
|
||
"Phone" = CASE WHEN e."Phone" IS NOT NULL THEN '+7700222' || lpad(n.rn::text, 4, '0') END,
|
||
"FirstName" = 'Тест',
|
||
"LastName" = 'Тестов' || n.rn,
|
||
"MiddleName" = NULL,
|
||
"TaxNumber" = NULL
|
||
FROM numbered n
|
||
WHERE e."Id" = n."Id";
|
||
|
||
-- 3. counterparties — БИН/ИИН/имена/контакты
|
||
WITH numbered AS (
|
||
SELECT "Id", row_number() OVER (ORDER BY "Id") AS rn FROM counterparties
|
||
)
|
||
UPDATE counterparties c
|
||
SET
|
||
"Name" = 'Контрагент-' || n.rn,
|
||
"LegalName" = CASE WHEN c."LegalName" IS NOT NULL THEN 'ТОО Контрагент-' || n.rn END,
|
||
-- Синтетические BIN/IIN (12 цифр, не валидируем checksum здесь).
|
||
"Bin" = CASE WHEN c."Bin" IS NOT NULL THEN lpad(n.rn::text, 12, '9') END,
|
||
"Iin" = CASE WHEN c."Iin" IS NOT NULL THEN lpad(n.rn::text, 12, '8') END,
|
||
"Phone" = CASE WHEN c."Phone" IS NOT NULL THEN '+7700333' || lpad(n.rn::text, 4, '0') END,
|
||
"Email" = CASE WHEN c."Email" IS NOT NULL THEN 'cp' || n.rn || '@example.kz' END,
|
||
"Address" = CASE WHEN c."Address" IS NOT NULL THEN 'г. Тестовый, ул. Тестовая ' || n.rn END,
|
||
"BankAccount" = CASE WHEN c."BankAccount" IS NOT NULL THEN 'KZ000000' || lpad(n.rn::text, 14, '0') END,
|
||
"ContactPerson" = CASE WHEN c."ContactPerson" IS NOT NULL THEN 'Контакт ' || n.rn END,
|
||
"Notes" = NULL
|
||
FROM numbered n
|
||
WHERE c."Id" = n."Id";
|
||
|
||
-- 4. organizations: имя/БИН/телефон владельца/MoySkladToken
|
||
UPDATE organizations
|
||
SET
|
||
"MoySkladToken" = NULL,
|
||
"OwnerTelegramChatId" = NULL,
|
||
"Bin" = CASE WHEN "Bin" IS NOT NULL THEN '700700700700' END,
|
||
"Name" = 'TestOrg-' || substr("Id"::text, 1, 8);
|
||
|
||
-- 5. refresh tokens revoke all (чтобы старые stage-токены не работали)
|
||
UPDATE "OpenIddictTokens" SET "Status" = 'revoked' WHERE "Status" = 'valid';
|
||
|
||
-- 6. чистим аудит-логи и feedback (могут содержать персональные тексты)
|
||
TRUNCATE TABLE org_audit_log;
|
||
TRUNCATE TABLE super_admin_audit_log;
|
||
SQL
|
||
log "anonymize done"
|
||
fi
|
||
|
||
# ── 4. pg_dump anonymized → out ──────────────────────────────────────
|
||
log "Step 4/5: pg_dump $TARGET_DB → $OUT"
|
||
if [[ $DRY_RUN -eq 0 ]]; then
|
||
pg_dump -h "$LOCAL_HOST" -p "$LOCAL_PORT" -U "$LOCAL_USER" -d "$TARGET_DB" \
|
||
--no-owner --no-privileges --clean --if-exists \
|
||
| gzip > "$OUT"
|
||
log "anonymized dump: $(du -h "$OUT" | cut -f1) → $OUT"
|
||
fi
|
||
|
||
# ── 5. Cleanup (через trap) ──────────────────────────────────────────
|
||
log "Step 5/5: cleanup"
|
||
rm -f "$DUMP" 2>/dev/null || true
|
||
|
||
log "✓ Готово: $OUT"
|
||
log "Восстановить можно: gunzip -c $OUT | psql -d food_market_dev"
|
||
exit 0
|