Почему импорт раньше обрывался на ~9500/29500 товаров: - StreamPagedAsync бросал исключение при любом сетевом глюке или таймауте HttpClient (90s) на одной из страниц и весь цикл сыпался. - Флаш делался раз в 500 товаров, так что при обрыве на 9500-м можно было потерять последние 499. Фиксы: - Per-page retry до 5 раз с exp-backoff (2,4,8,16с) — обрабатываем только сетевые ошибки (HttpRequestException / TaskCanceledException / IOException). API-ошибки типа 4xx проходят наверх как есть. - SaveChangesAsync теперь каждые 100 товаров вместо 500 — меньше вероятность потерять при внезапном обрыве на границе. - При исчерпании retries — бросаем осмысленное исключение с offset'ом. Пользователь сейчас имеет 9500 из 29509 товаров (группа "Алкоголь" — 20 из 518). Нужно перезапустить импорт в UI с overwriteExisting=true — существующие товары обновит, недостающие подтянет.
141 lines
6.7 KiB
C#
141 lines
6.7 KiB
C#
using System.Net.Http.Headers;
|
|
using System.Net.Http.Json;
|
|
using System.Text.Json;
|
|
|
|
namespace foodmarket.Infrastructure.Integrations.MoySklad;
|
|
|
|
public record MoySkladApiResult<T>(bool Success, T? Value, int? StatusCode, string? Error)
|
|
{
|
|
public static MoySkladApiResult<T> Ok(T value) => new(true, value, 200, null);
|
|
public static MoySkladApiResult<T> Fail(int status, string? error) => new(false, default, status, error);
|
|
}
|
|
|
|
// Thin HTTP wrapper over MoySklad JSON-API 1.2. Caller supplies the personal token per request
|
|
// — we never persist it.
|
|
public class MoySkladClient
|
|
{
|
|
// Trailing slash is critical: otherwise HttpClient drops the last path segment
|
|
// when resolving relative URIs (RFC 3986 §5.3), so "entity/product" would hit
|
|
// "/api/remap/entity/product" instead of "/api/remap/1.2/entity/product".
|
|
private const string BaseUrl = "https://api.moysklad.ru/api/remap/1.2/";
|
|
private static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web);
|
|
|
|
private readonly HttpClient _http;
|
|
|
|
public MoySkladClient(HttpClient http)
|
|
{
|
|
_http = http;
|
|
_http.BaseAddress ??= new Uri(BaseUrl);
|
|
_http.Timeout = TimeSpan.FromSeconds(90);
|
|
}
|
|
|
|
private HttpRequestMessage Build(HttpMethod method, string pathAndQuery, string token)
|
|
{
|
|
var req = new HttpRequestMessage(method, pathAndQuery);
|
|
req.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token);
|
|
// MoySklad requires the exact literal "application/json;charset=utf-8" (no space
|
|
// after ';'). The typed MediaTypeWithQualityHeaderValue API normalizes to
|
|
// "application/json; charset=utf-8" which MoySklad rejects with code 1062.
|
|
req.Headers.TryAddWithoutValidation("Accept", "application/json;charset=utf-8");
|
|
// MoySklad's nginx edge returns 415 for requests without a User-Agent, and we want
|
|
// auto-decompression (Accept-Encoding is added automatically by HttpClient when
|
|
// AutomaticDecompression is set on the primary handler — see Program.cs).
|
|
if (!req.Headers.UserAgent.Any())
|
|
{
|
|
req.Headers.TryAddWithoutValidation("User-Agent", "food-market/0.1 (+https://github.com/nurdotnet/food-market)");
|
|
}
|
|
return req;
|
|
}
|
|
|
|
public async Task<MoySkladApiResult<MsOrganization>> WhoAmIAsync(string token, CancellationToken ct)
|
|
{
|
|
using var req = Build(HttpMethod.Get, "entity/organization?limit=1", token);
|
|
using var res = await _http.SendAsync(req, ct);
|
|
if (!res.IsSuccessStatusCode)
|
|
{
|
|
var body = await res.Content.ReadAsStringAsync(ct);
|
|
return MoySkladApiResult<MsOrganization>.Fail((int)res.StatusCode, body);
|
|
}
|
|
var list = await res.Content.ReadFromJsonAsync<MsListResponse<MsOrganization>>(Json, ct);
|
|
var org = list?.Rows.FirstOrDefault();
|
|
return org is null
|
|
? MoySkladApiResult<MsOrganization>.Fail(200, "Empty organization list returned by MoySklad.")
|
|
: MoySkladApiResult<MsOrganization>.Ok(org);
|
|
}
|
|
|
|
// MoySklad list endpoints по умолчанию возвращают только активных (archived=false).
|
|
// Чтобы получить архивных, нужен явный filter=archived=true. Делаем два прохода:
|
|
// сначала активные (default), затем архивные — и отдаём всё одним потоком.
|
|
|
|
public async IAsyncEnumerable<MsProduct> StreamProductsAsync(
|
|
string token,
|
|
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct)
|
|
{
|
|
await foreach (var p in StreamPagedAsync<MsProduct>(token, "entity/product", archivedOnly: false, ct)) yield return p;
|
|
await foreach (var p in StreamPagedAsync<MsProduct>(token, "entity/product", archivedOnly: true, ct)) yield return p;
|
|
}
|
|
|
|
public async IAsyncEnumerable<MsCounterparty> StreamCounterpartiesAsync(
|
|
string token,
|
|
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct)
|
|
{
|
|
await foreach (var c in StreamPagedAsync<MsCounterparty>(token, "entity/counterparty", archivedOnly: false, ct)) yield return c;
|
|
await foreach (var c in StreamPagedAsync<MsCounterparty>(token, "entity/counterparty", archivedOnly: true, ct)) yield return c;
|
|
}
|
|
|
|
public async Task<List<MsProductFolder>> GetAllFoldersAsync(string token, CancellationToken ct)
|
|
{
|
|
var all = new List<MsProductFolder>();
|
|
await foreach (var f in StreamPagedAsync<MsProductFolder>(token, "entity/productfolder", archivedOnly: false, ct)) all.Add(f);
|
|
await foreach (var f in StreamPagedAsync<MsProductFolder>(token, "entity/productfolder", archivedOnly: true, ct)) all.Add(f);
|
|
return all;
|
|
}
|
|
|
|
private async IAsyncEnumerable<T> StreamPagedAsync<T>(
|
|
string token,
|
|
string path,
|
|
bool archivedOnly,
|
|
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct)
|
|
{
|
|
const int pageSize = 1000;
|
|
const int maxAttempts = 5;
|
|
var offset = 0;
|
|
var filterSuffix = archivedOnly ? "&filter=archived=true" : "";
|
|
while (true)
|
|
{
|
|
MsListResponse<T>? page = null;
|
|
Exception? lastErr = null;
|
|
for (var attempt = 1; attempt <= maxAttempts; attempt++)
|
|
{
|
|
try
|
|
{
|
|
using var req = Build(HttpMethod.Get, $"{path}?limit={pageSize}&offset={offset}{filterSuffix}", token);
|
|
using var res = await _http.SendAsync(req, ct);
|
|
res.EnsureSuccessStatusCode();
|
|
page = await res.Content.ReadFromJsonAsync<MsListResponse<T>>(Json, ct);
|
|
lastErr = null;
|
|
break;
|
|
}
|
|
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or IOException)
|
|
{
|
|
lastErr = ex;
|
|
if (attempt == maxAttempts) break;
|
|
// Exponential-ish backoff: 2s, 4s, 8s, 16s.
|
|
await Task.Delay(TimeSpan.FromSeconds(Math.Pow(2, attempt)), ct);
|
|
}
|
|
}
|
|
if (lastErr is not null)
|
|
{
|
|
// Re-throw after retries so the caller sees a real failure instead of silent halt.
|
|
throw new InvalidOperationException(
|
|
$"MoySklad paging failed at {path} offset={offset} after {maxAttempts} attempts: {lastErr.Message}",
|
|
lastErr);
|
|
}
|
|
if (page is null || page.Rows.Count == 0) yield break;
|
|
foreach (var row in page.Rows) yield return row;
|
|
if (page.Rows.Count < pageSize) yield break;
|
|
offset += pageSize;
|
|
}
|
|
}
|
|
}
|