После 24 спринтов regress-suite разросся; нестабильность блокирует доверие. Этот спринт: ловит flaky тесты, добавляет observability (Grafana + Prometheus alerts + RUNBOOK), сертифицирует 10× cert-прогон. 1. tests/regression/find-flaky.sh — 10× прогон + JSON-агрегатор → docs/flaky-tests.md (per-test pass/fail sequence + reproduce). 2. OrgFactory.signupWithRetry теперь honors Retry-After header (api-client.ts:ApiError.retryAfterSec). Stage rate-limit поднят: RATE_SIGNUP_HOUR=5000, RATE_PER_IP_MIN=5000 (~/food-market-stage/deploy/.env). 3. fullyParallel=true + workers=4 = тесты идут в недетерминированном порядке; isolation работает (OrgFactory per-test). 4. workers=4 даёт **2.4× ускорение** (66.6s → 27.7s). Worker-scoped фикстура lib/worker-org.ts добавлена как opt-in. 5. deploy/grafana/dashboards/quality-watchdog.json (10 панелей: smoke success ratio 7d, incidents, multi-tenant violations, current emoji, p95 by endpoint, step failures, RPS, DB p95, docs posted, disk free) + dashboards/README.md. quality-watchdog.sh пишет Prometheus textfile экспорт в ~/.fm-watchdog/textfile/quality_watchdog.prom для node_exporter. 6. deploy/prometheus/alerts.yml — 10 правил, 4 группы (uptime, errors, database, quality-watchdog). MultiTenantViolation = P0. deploy/prometheus/prometheus.yml — reference config. 7. docs/RUNBOOK.md +178 строк: action per alert (api-down, rps-drop, http-errors-spike/growing, doc-posting-errors, db-p95-high, disk-free-low, watchdog-red, multi-tenant-violation, watchdog-incident). Junior-friendly с конкретными командами. **Cert-прогон (10× workers=4):** 420/420 passed, 0 flaky, avg 30.1s/run, total 300.6s (< 5min budget). Изменения вне репо: - ~/food-market-stage/deploy/.env — RATE_* limits bumped. - ~/quality-watchdog.sh — добавлен .prom textfile экспорт. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
287 lines
11 KiB
TypeScript
287 lines
11 KiB
TypeScript
/**
|
||
* Sprint 16: OrgFactory — главная фабрика тестовых данных для regression.
|
||
*
|
||
* Создаёт через API за O(N) HTTP вызовов:
|
||
* - Organization + Admin user (signup),
|
||
* - access_token,
|
||
* - N товаров,
|
||
* - M контрагентов (опционально),
|
||
* - K документов (опционально).
|
||
*
|
||
* Возвращает один объект OrgSession + опционально кеш ссылок (refIds,
|
||
* products, counterparties, supplies). Используется в каждом
|
||
* regression-flow вместо повторного signup + form-fill.
|
||
*
|
||
* Дизайн: builder-pattern с цепочкой `.withProducts(N).withSupplies(M).build()`,
|
||
* чтобы тест запросил ровно тот фикстур-набор который нужен. Без
|
||
* "монстро-фабрики" которая всегда делает всё.
|
||
*/
|
||
import { ApiError, request, sleep } from './api-client.js'
|
||
import {
|
||
Endpoints,
|
||
type CounterpartyRef,
|
||
type DocumentRef,
|
||
type OrgSession,
|
||
type ProductRef,
|
||
type RefIds,
|
||
} from './types.js'
|
||
|
||
interface SignupResult {
|
||
organizationId: string
|
||
email: string
|
||
}
|
||
|
||
interface TokenResult {
|
||
access_token: string
|
||
refresh_token: string
|
||
}
|
||
|
||
export interface BuiltOrg {
|
||
session: OrgSession
|
||
refs: RefIds
|
||
products: ProductRef[]
|
||
counterparties: CounterpartyRef[]
|
||
supplies: DocumentRef[]
|
||
/** Headers с Bearer-токеном для запросов из теста. */
|
||
authHeaders: { Authorization: string; 'Content-Type': string }
|
||
}
|
||
|
||
interface FactoryOptions {
|
||
productsCount: number
|
||
counterpartiesCount: number
|
||
/** Создать приёмки (Supply.Post) — N штук. У каждой одна линия с productCount/N
|
||
* товарами по 100шт. Без них товары будут без остатка. */
|
||
suppliesCount: number
|
||
/** Префикс для slug'а (email/orgName) — удобно для дебага. */
|
||
slug: string
|
||
}
|
||
|
||
const DEFAULT_OPTS: FactoryOptions = {
|
||
productsCount: 0,
|
||
counterpartiesCount: 0,
|
||
suppliesCount: 0,
|
||
slug: 'reg',
|
||
}
|
||
|
||
export class OrgFactory {
|
||
private opts: FactoryOptions = { ...DEFAULT_OPTS }
|
||
|
||
static for(slug: string): OrgFactory {
|
||
const f = new OrgFactory()
|
||
f.opts.slug = slug
|
||
return f
|
||
}
|
||
|
||
withProducts(n: number): this { this.opts.productsCount = n; return this }
|
||
withCounterparties(n: number): this { this.opts.counterpartiesCount = n; return this }
|
||
withSupplies(n: number): this { this.opts.suppliesCount = n; return this }
|
||
|
||
async build(): Promise<BuiltOrg> {
|
||
const { slug } = this.opts
|
||
const ts = Date.now() + Math.floor(Math.random() * 9999)
|
||
const email = `${slug}-${ts}@food-market.local`
|
||
const password = 'RegTest12345!'
|
||
const orgName = `Reg-${slug}-${ts}`
|
||
|
||
// 1) Signup. Сервер rate-limit'ит signup (Sprint 13: 3/час/IP на prod,
|
||
// 30/час/IP на stage env). Под нагрузкой ждём + повторяем.
|
||
const signup = await this.signupWithRetry(email, password, orgName)
|
||
|
||
// 2) Token.
|
||
const token = await this.getToken(email, password)
|
||
|
||
// 3) Refs (units / groups / stores / retail-points / currencies / price-types).
|
||
const refs = await this.loadRefs(token.access_token)
|
||
|
||
const built: BuiltOrg = {
|
||
session: {
|
||
email, password, orgName,
|
||
orgId: signup.organizationId,
|
||
accessToken: token.access_token,
|
||
refreshToken: token.refresh_token,
|
||
},
|
||
refs,
|
||
products: [],
|
||
counterparties: [],
|
||
supplies: [],
|
||
authHeaders: {
|
||
Authorization: `Bearer ${token.access_token}`,
|
||
'Content-Type': 'application/json',
|
||
},
|
||
}
|
||
|
||
// 4) Products (опц.).
|
||
if (this.opts.productsCount > 0) {
|
||
built.products = await this.createProducts(token.access_token, refs, this.opts.productsCount)
|
||
}
|
||
|
||
// 5) Counterparties (опц.).
|
||
if (this.opts.counterpartiesCount > 0) {
|
||
built.counterparties = await this.createCounterparties(token.access_token, this.opts.counterpartiesCount)
|
||
}
|
||
|
||
// 6) Supplies (опц.) — нужны для тестов на остаток/продажу.
|
||
if (this.opts.suppliesCount > 0) {
|
||
if (built.products.length === 0)
|
||
throw new Error('Supplies require at least 1 product — call .withProducts(N) first')
|
||
if (built.counterparties.length === 0) {
|
||
// авто-создаём 1 поставщика чтобы supplyлогика не падала
|
||
built.counterparties = await this.createCounterparties(token.access_token, 1)
|
||
}
|
||
built.supplies = await this.createPostedSupplies(token.access_token, refs, built, this.opts.suppliesCount)
|
||
}
|
||
|
||
return built
|
||
}
|
||
|
||
// ─ private helpers ────────────────────────────────────────────────────
|
||
|
||
private async signupWithRetry(email: string, password: string, orgName: string): Promise<SignupResult> {
|
||
return await this.retryOn429(4, () => request<SignupResult>(Endpoints.signup, {
|
||
body: { email, password, organizationName: orgName, phone: '+77001234567' },
|
||
}))
|
||
}
|
||
|
||
/**
|
||
* Sprint 26: общий retry-helper для 429-ответов. Honors Retry-After header
|
||
* если сервер прислал; иначе короткий backoff с jitter (1-3s × attempt).
|
||
* Stage rate-limit поднят до 5000/min, поэтому реально 429 ловить почти
|
||
* не должны — этот retry на крайний случай.
|
||
*/
|
||
private async retryOn429<T>(maxAttempts: number, fn: () => Promise<T>): Promise<T> {
|
||
let lastErr: unknown
|
||
for (let i = 0; i < maxAttempts; i++) {
|
||
try {
|
||
return await fn()
|
||
} catch (e) {
|
||
if (e instanceof ApiError && e.status === 429) {
|
||
const headerDelay = (e.retryAfterSec ?? 0) * 1000
|
||
const backoff = 1_000 + i * 1_500 + Math.floor(Math.random() * 1_000) // 1-2s, 2.5-3.5s, 4-5s, 5.5-6.5s
|
||
await sleep(Math.min(Math.max(headerDelay, backoff), 8_000))
|
||
lastErr = e
|
||
continue
|
||
}
|
||
throw e
|
||
}
|
||
}
|
||
throw lastErr ?? new Error('429 retry exhausted')
|
||
}
|
||
|
||
private async getToken(email: string, password: string): Promise<TokenResult> {
|
||
const body = new URLSearchParams({
|
||
grant_type: 'password',
|
||
username: email,
|
||
password,
|
||
client_id: 'food-market-web',
|
||
scope: 'openid profile email roles api offline_access',
|
||
}).toString()
|
||
return await this.retryOn429(8, () => request<TokenResult>(Endpoints.token, {
|
||
body,
|
||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||
}))
|
||
}
|
||
|
||
private async loadRefs(token: string): Promise<RefIds> {
|
||
interface Paged<T> { items: T[] }
|
||
interface ItemBase { id: string }
|
||
interface Unit extends ItemBase { code: string }
|
||
interface PriceType extends ItemBase { isRetail: boolean }
|
||
interface Currency extends ItemBase { code: string }
|
||
interface Store extends ItemBase { isMain: boolean }
|
||
|
||
const [units, groups, stores, rps, curs, pts] = await Promise.all([
|
||
request<Paged<Unit>>(Endpoints.refs.units, { token }),
|
||
request<Paged<ItemBase>>(Endpoints.refs.groups, { token }),
|
||
request<Paged<Store>>(Endpoints.refs.stores, { token }),
|
||
request<Paged<ItemBase>>(Endpoints.refs.retailPoints, { token }),
|
||
request<Paged<Currency>>(Endpoints.refs.currencies, { token }),
|
||
request<Paged<PriceType>>(Endpoints.refs.priceTypes, { token }),
|
||
])
|
||
const unit = units.items.find(u => u.code === '796') ?? units.items[0]
|
||
const group = groups.items[0]
|
||
const store = stores.items.find(s => s.isMain) ?? stores.items[0]
|
||
const rp = rps.items[0]
|
||
const cur = curs.items.find(c => c.code === 'KZT') ?? curs.items[0]
|
||
const pt = pts.items.find(p => p.isRetail) ?? pts.items[0]
|
||
if (!unit || !group || !store || !rp || !cur || !pt) {
|
||
throw new Error(`refs incomplete: unit=${!!unit} group=${!!group} store=${!!store} rp=${!!rp} cur=${!!cur} pt=${!!pt}`)
|
||
}
|
||
return {
|
||
unitId: unit.id, groupId: group.id,
|
||
storeId: store.id, retailPointId: rp.id,
|
||
currencyId: cur.id, priceTypeId: pt.id,
|
||
}
|
||
}
|
||
|
||
private async createProducts(token: string, refs: RefIds, n: number): Promise<ProductRef[]> {
|
||
const out: ProductRef[] = []
|
||
for (let i = 0; i < n; i++) {
|
||
const name = `Товар ${i + 1}`
|
||
const article = `ART-${this.opts.slug}-${Date.now()}-${i}`
|
||
const barcode = this.randomBarcode()
|
||
const product = await request<{ id: string }>(Endpoints.products, {
|
||
token,
|
||
body: {
|
||
name, article, unitOfMeasureId: refs.unitId,
|
||
vat: 12, vatEnabled: true,
|
||
productGroupId: refs.groupId,
|
||
packaging: 1,
|
||
prices: [{ priceTypeId: refs.priceTypeId, amount: 100 + i * 10, currencyId: refs.currencyId }],
|
||
barcodes: [{ code: barcode, type: 1, isPrimary: true }],
|
||
},
|
||
})
|
||
out.push({ id: product.id, name, article })
|
||
}
|
||
return out
|
||
}
|
||
|
||
private async createCounterparties(token: string, n: number): Promise<CounterpartyRef[]> {
|
||
const out: CounterpartyRef[] = []
|
||
for (let i = 0; i < n; i++) {
|
||
const name = `Контрагент ${i + 1}`
|
||
// Чередуем типы: чётные — юрлица, нечётные — физлица.
|
||
const type = i % 2 === 0 ? 1 : 2
|
||
const cp = await request<{ id: string }>(Endpoints.counterparties, {
|
||
token,
|
||
body: { name, type, bin: type === 1 ? '123456789012' : null },
|
||
})
|
||
out.push({ id: cp.id, name, type })
|
||
}
|
||
return out
|
||
}
|
||
|
||
private async createPostedSupplies(
|
||
token: string, refs: RefIds, built: BuiltOrg, n: number,
|
||
): Promise<DocumentRef[]> {
|
||
const supplier = built.counterparties[0]!
|
||
const out: DocumentRef[] = []
|
||
// На каждую приёмку — все products с qty 100 / unitPrice 50.
|
||
// Это даёт быстрый stock=100×N для дальнейших sales-flow.
|
||
const lines = built.products.map(p => ({
|
||
productId: p.id, quantity: 100, unitPrice: 50,
|
||
}))
|
||
for (let i = 0; i < n; i++) {
|
||
const draft = await request<{ id: string; number: string }>(Endpoints.supplies, {
|
||
token,
|
||
body: {
|
||
date: new Date().toISOString(),
|
||
supplierId: supplier.id,
|
||
storeId: refs.storeId,
|
||
currencyId: refs.currencyId,
|
||
lines,
|
||
},
|
||
})
|
||
// Post — отдельный endpoint, NoContent на успехе.
|
||
await request(`${Endpoints.supplies}/${draft.id}/post`, { token, method: 'POST' })
|
||
out.push({ id: draft.id, number: draft.number })
|
||
}
|
||
return out
|
||
}
|
||
|
||
private randomBarcode(): string {
|
||
let s = ''
|
||
for (let i = 0; i < 13; i++) s += Math.floor(Math.random() * 10)
|
||
return s
|
||
}
|
||
}
|