nns/food-market
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

perf(migration): резюм документов по offset (не перечитывать уже перенесённое)

Browse source 
startOffset у всех документных импортов + эндпоинт ?offset=N: при продолжении
пропускаем уже перенесённые через offset API (а не перечитывая сотни страниц).
Идемпотентный skip по msId держит границу.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
nns 2026-10-09 15:52:28 +05:00
parent 417030827f
commit 9506f9edc8
2 changed files with 14 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
src/food-market.api/Controllers/Admin/MoySkladImportController.cs
View file

@ -151,6 +151,7 @@ public async Task<ActionResult<object>> ImportSupplies([FromBody] ImportRequest
return BadRequest(new { error = "Токен не настроен." }); return BadRequest(new { error = "Токен не настроен." });
var orgId = _tenant.OrganizationId ?? throw new InvalidOperationException("No tenant."); var orgId = _tenant.OrganizationId ?? throw new InvalidOperationException("No tenant.");
var type = HttpContext.Request.Query["type"].ToString(); // захватываем ДО Task.Run (HttpContext потом недоступен) var type = HttpContext.Request.Query["type"].ToString(); // захватываем ДО Task.Run (HttpContext потом недоступен)
int startOffset = int.TryParse(HttpContext.Request.Query["offset"].ToString(), out var off) ? off : 0;
var job = _jobs.Create(string.IsNullOrEmpty(type) ? "supplies" : type); var job = _jobs.Create(string.IsNullOrEmpty(type) ? "supplies" : type);
job.Stage = $"Перенос документов ({(string.IsNullOrEmpty(type) ? "приёмки" : type)}) с историей…"; job.Stage = $"Перенос документов ({(string.IsNullOrEmpty(type) ? "приёмки" : type)}) с историей…";
_ = Task.Run(async () => _ = Task.Run(async () =>
@ -165,11 +166,11 @@ public async Task<ActionResult<object>> ImportSupplies([FromBody] ImportRequest
var pr = new foodmarket.Infrastructure.Integrations.MoySklad.MoySkladDocumentImportService.DocImportProgress(); var pr = new foodmarket.Infrastructure.Integrations.MoySklad.MoySkladDocumentImportService.DocImportProgress();
switch (type) switch (type)
{ {
case "enter": await docs.ImportEntersAsync(token, pr, CancellationToken.None); break; case "enter": await docs.ImportEntersAsync(token, startOffset, pr, CancellationToken.None); break;
case "loss": await docs.ImportLossesAsync(token, pr, CancellationToken.None); break; case "loss": await docs.ImportLossesAsync(token, startOffset, pr, CancellationToken.None); break;
case "purchasereturn": await docs.ImportSupplierReturnsAsync(token, pr, CancellationToken.None); break; case "purchasereturn": await docs.ImportSupplierReturnsAsync(token, startOffset, pr, CancellationToken.None); break;
case "retaildemand": await docs.ImportRetailSalesAsync(token, pr, CancellationToken.None); break; case "retaildemand": await docs.ImportRetailSalesAsync(token, startOffset, pr, CancellationToken.None); break;
default: await docs.ImportSuppliesAsync(token, 0, pr, CancellationToken.None); break; default: await docs.ImportSuppliesAsync(token, startOffset, pr, CancellationToken.None); break;
} }
job.Total = pr.Seen; job.Created = pr.Created; job.Skipped = pr.Skipped; job.Total = pr.Seen; job.Created = pr.Created; job.Skipped = pr.Skipped;
job.Message = $"Приёмки: перенесено {pr.Created}, пропущено {pr.Skipped}, ошибок {pr.Errors}. " + job.Message = $"Приёмки: перенесено {pr.Created}, пропущено {pr.Skipped}, ошибок {pr.Errors}. " +

16
src/food-market.infrastructure/Integrations/MoySklad/MoySkladDocumentImportService.cs
View file

@ -195,14 +195,14 @@ private async Task<DocMaps> LoadMapsAsync(Guid orgId, CancellationToken ct)
} }
// ── Оприходования (enter) ──────────────────────────────────────────────── // ── Оприходования (enter) ────────────────────────────────────────────────
public async Task<DocImportProgress> ImportEntersAsync(string token, DocImportProgress progress, CancellationToken ct) public async Task<DocImportProgress> ImportEntersAsync(string token, int startOffset, DocImportProgress progress, CancellationToken ct)
{ {
var orgId = _tenant.OrganizationId!.Value; progress.Entity = "enter"; var orgId = _tenant.OrganizationId!.Value; progress.Entity = "enter";
var m = await LoadMapsAsync(orgId, ct); var m = await LoadMapsAsync(orgId, ct);
var done = (await _db.Enters.Where(x => x.MoySkladExternalId != null).Select(x => x.MoySkladExternalId!).ToListAsync(ct)).ToHashSet(); var done = (await _db.Enters.Where(x => x.MoySkladExternalId != null).Select(x => x.MoySkladExternalId!).ToListAsync(ct)).ToHashSet();
var used = (await _db.Enters.Select(x => x.Number).ToListAsync(ct)).ToHashSet(StringComparer.OrdinalIgnoreCase); var used = (await _db.Enters.Select(x => x.Number).ToListAsync(ct)).ToHashSet(StringComparer.OrdinalIgnoreCase);
string Uniq(string b, int y) { if (used.Add(b)) return b; var w = $"{b}-{y}"; if (used.Add(w)) return w; for (var n = 2; ; n++) { var c = $"{b}-{y}-{n}"; if (used.Add(c)) return c; } } string Uniq(string b, int y) { if (used.Add(b)) return b; var w = $"{b}-{y}"; if (used.Add(w)) return w; for (var n = 2; ; n++) { var c = $"{b}-{y}-{n}"; if (used.Add(c)) return c; } }
await foreach (var d in _client.StreamDocumentsAsync<MsDocument>(token, "entity/enter", "positions,store", 0, null, ct)) await foreach (var d in _client.StreamDocumentsAsync<MsDocument>(token, "entity/enter", "positions,store", startOffset, null, ct))
{ {
progress.Seen++; if (d.Id is null || done.Contains(d.Id)) { progress.Skipped++; continue; } progress.Seen++; if (d.Id is null || done.Contains(d.Id)) { progress.Skipped++; continue; }
try try
@ -224,14 +224,14 @@ await foreach (var d in _client.StreamDocumentsAsync<MsDocument>(token, "entity/
} }
// ── Списания (loss) ────────────────────────────────────────────────────── // ── Списания (loss) ──────────────────────────────────────────────────────
public async Task<DocImportProgress> ImportLossesAsync(string token, DocImportProgress progress, CancellationToken ct) public async Task<DocImportProgress> ImportLossesAsync(string token, int startOffset, DocImportProgress progress, CancellationToken ct)
{ {
var orgId = _tenant.OrganizationId!.Value; progress.Entity = "loss"; var orgId = _tenant.OrganizationId!.Value; progress.Entity = "loss";
var m = await LoadMapsAsync(orgId, ct); var m = await LoadMapsAsync(orgId, ct);
var done = (await _db.Losses.Where(x => x.MoySkladExternalId != null).Select(x => x.MoySkladExternalId!).ToListAsync(ct)).ToHashSet(); var done = (await _db.Losses.Where(x => x.MoySkladExternalId != null).Select(x => x.MoySkladExternalId!).ToListAsync(ct)).ToHashSet();
var used = (await _db.Losses.Select(x => x.Number).ToListAsync(ct)).ToHashSet(StringComparer.OrdinalIgnoreCase); var used = (await _db.Losses.Select(x => x.Number).ToListAsync(ct)).ToHashSet(StringComparer.OrdinalIgnoreCase);
string Uniq(string b, int y) { if (used.Add(b)) return b; var w = $"{b}-{y}"; if (used.Add(w)) return w; for (var n = 2; ; n++) { var c = $"{b}-{y}-{n}"; if (used.Add(c)) return c; } } string Uniq(string b, int y) { if (used.Add(b)) return b; var w = $"{b}-{y}"; if (used.Add(w)) return w; for (var n = 2; ; n++) { var c = $"{b}-{y}-{n}"; if (used.Add(c)) return c; } }
await foreach (var d in _client.StreamDocumentsAsync<MsDocument>(token, "entity/loss", "positions,store", 0, null, ct)) await foreach (var d in _client.StreamDocumentsAsync<MsDocument>(token, "entity/loss", "positions,store", startOffset, null, ct))
{ {
progress.Seen++; if (d.Id is null || done.Contains(d.Id)) { progress.Skipped++; continue; } progress.Seen++; if (d.Id is null || done.Contains(d.Id)) { progress.Skipped++; continue; }
try try
@ -253,14 +253,14 @@ await foreach (var d in _client.StreamDocumentsAsync<MsDocument>(token, "entity/
} }
// ── Возвраты поставщику (purchasereturn) ───────────────────────────────── // ── Возвраты поставщику (purchasereturn) ─────────────────────────────────
public async Task<DocImportProgress> ImportSupplierReturnsAsync(string token, DocImportProgress progress, CancellationToken ct) public async Task<DocImportProgress> ImportSupplierReturnsAsync(string token, int startOffset, DocImportProgress progress, CancellationToken ct)
{ {
var orgId = _tenant.OrganizationId!.Value; progress.Entity = "purchasereturn"; var orgId = _tenant.OrganizationId!.Value; progress.Entity = "purchasereturn";
var m = await LoadMapsAsync(orgId, ct); var m = await LoadMapsAsync(orgId, ct);
var done = (await _db.SupplierReturns.Where(x => x.MoySkladExternalId != null).Select(x => x.MoySkladExternalId!).ToListAsync(ct)).ToHashSet(); var done = (await _db.SupplierReturns.Where(x => x.MoySkladExternalId != null).Select(x => x.MoySkladExternalId!).ToListAsync(ct)).ToHashSet();
var used = (await _db.SupplierReturns.Select(x => x.Number).ToListAsync(ct)).ToHashSet(StringComparer.OrdinalIgnoreCase); var used = (await _db.SupplierReturns.Select(x => x.Number).ToListAsync(ct)).ToHashSet(StringComparer.OrdinalIgnoreCase);
string Uniq(string b, int y) { if (used.Add(b)) return b; var w = $"{b}-{y}"; if (used.Add(w)) return w; for (var n = 2; ; n++) { var c = $"{b}-{y}-{n}"; if (used.Add(c)) return c; } } string Uniq(string b, int y) { if (used.Add(b)) return b; var w = $"{b}-{y}"; if (used.Add(w)) return w; for (var n = 2; ; n++) { var c = $"{b}-{y}-{n}"; if (used.Add(c)) return c; } }
await foreach (var d in _client.StreamDocumentsAsync<MsDocument>(token, "entity/purchasereturn", "positions,agent,store", 0, null, ct)) await foreach (var d in _client.StreamDocumentsAsync<MsDocument>(token, "entity/purchasereturn", "positions,agent,store", startOffset, null, ct))
{ {
progress.Seen++; if (d.Id is null || done.Contains(d.Id)) { progress.Skipped++; continue; } progress.Seen++; if (d.Id is null || done.Contains(d.Id)) { progress.Skipped++; continue; }
try try
@ -284,14 +284,14 @@ await foreach (var d in _client.StreamDocumentsAsync<MsDocument>(token, "entity/
} }
// ── Розничные продажи (retaildemand) ───────────────────────────────────── // ── Розничные продажи (retaildemand) ─────────────────────────────────────
public async Task<DocImportProgress> ImportRetailSalesAsync(string token, DocImportProgress progress, CancellationToken ct) public async Task<DocImportProgress> ImportRetailSalesAsync(string token, int startOffset, DocImportProgress progress, CancellationToken ct)
{ {
var orgId = _tenant.OrganizationId!.Value; progress.Entity = "retaildemand"; var orgId = _tenant.OrganizationId!.Value; progress.Entity = "retaildemand";
var m = await LoadMapsAsync(orgId, ct); var m = await LoadMapsAsync(orgId, ct);
var done = (await _db.RetailSales.Where(x => x.MoySkladExternalId != null).Select(x => x.MoySkladExternalId!).ToListAsync(ct)).ToHashSet(); var done = (await _db.RetailSales.Where(x => x.MoySkladExternalId != null).Select(x => x.MoySkladExternalId!).ToListAsync(ct)).ToHashSet();
var used = (await _db.RetailSales.Select(x => x.Number).ToListAsync(ct)).ToHashSet(StringComparer.OrdinalIgnoreCase); var used = (await _db.RetailSales.Select(x => x.Number).ToListAsync(ct)).ToHashSet(StringComparer.OrdinalIgnoreCase);
string Uniq(string b, int y) { if (used.Add(b)) return b; var w = $"{b}-{y}"; if (used.Add(w)) return w; for (var n = 2; ; n++) { var c = $"{b}-{y}-{n}"; if (used.Add(c)) return c; } } string Uniq(string b, int y) { if (used.Add(b)) return b; var w = $"{b}-{y}"; if (used.Add(w)) return w; for (var n = 2; ; n++) { var c = $"{b}-{y}-{n}"; if (used.Add(c)) return c; } }
await foreach (var d in _client.StreamDocumentsAsync<MsDocument>(token, "entity/retaildemand", "positions,store", 0, null, ct)) await foreach (var d in _client.StreamDocumentsAsync<MsDocument>(token, "entity/retaildemand", "positions,store", startOffset, null, ct))
{ {
progress.Seen++; if (d.Id is null || done.Contains(d.Id)) { progress.Skipped++; continue; } progress.Seen++; if (d.Id is null || done.Contains(d.Id)) { progress.Skipped++; continue; }
try try