quentin_duwa/dinggadingga-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(sync): harden the Hiboutik sync, share rate limits across instances, update dependencies (#4)

Browse Source
This commit is contained in:
quentin_duwa
2026-11-08 13:45:08 +00:00
parent 04960379e8
commit d07c4df52d
62 changed files with 1546 additions and 639 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/services/hiboutik/brand.hiboutik.service.ts
View File

@@ -26,7 +26,7 @@ export class BrandHiboutikService {
);
// Hiboutik is trusted: a successful (2xx) answer, even an empty list, is the truth. An HTTP error throws before this point.
const nbDeleted = await this.brandDatabaseRepository.deleteMissing(hiboutikBrands.map((dto) => Number(dto.brand_id)));
const nbDeleted = await this.brandDatabaseRepository.deleteMissing(hiboutikBrands.map((dto) => dto.brand_id));
this.logger.log(
`Brands sync ended: ${hiboutikBrands.length - nbFailed}/${hiboutikBrands.length} synced, ${nbFailed} failed, ${nbDeleted} deleted.`,
);

19
src/services/hiboutik/category.hiboutik.service.spec.ts
View File

@@ -3,16 +3,15 @@ import { CategoryHiboutikService } from './category.hiboutik.service';
import { CategoryConverter } from '../../models/converter/category.converter';
import { CategoryHiboutikDto } from '../../models/hiboutikDTO/category.hiboutik.dto';
const hiboutikCategory = (id: number, parentId: number, overrides: Partial<CategoryHiboutikDto> = {}): CategoryHiboutikDto =>
({
category_id: id,
category_name: `Category ${id}`,
category_id_parent: parentId,
category_enabled: 1,
category_enabled_www: 0,
category_position: 3,
...overrides,
}) as CategoryHiboutikDto;
const hiboutikCategory = (id: number, parentId: number, overrides: Partial<CategoryHiboutikDto> = {}): CategoryHiboutikDto => ({
category_id: id,
category_name: `Category ${id}`,
category_id_parent: parentId,
category_enabled: 1,
category_enabled_www: 0,
category_position: 3,
...overrides,
});
describe('CategoryHiboutikService', () => {
const categoryHiboutikRepository = { getCategories: jest.fn() };

4
src/services/hiboutik/category.hiboutik.service.ts
View File

@@ -32,13 +32,13 @@ export class CategoryHiboutikService {
// Pass 2: parent links, once every category exists (a child can come before its parent in Hiboutik's list)
const nbParentsFailed = await forEachSafely(
hiboutikCategories,
(dto) => this.categoryDatabaseRepository.setParent(Number(dto.category_id), Number(dto.category_id_parent) || null),
(dto) => this.categoryDatabaseRepository.setParent(dto.category_id, dto.category_id_parent || null),
this.logger,
(dto) => `parent of Hiboutik category ${dto.category_id}`,
);
// Hiboutik is trusted: a successful (2xx) answer, even an empty list, is the truth. An HTTP error throws before this point.
const nbDeleted = await this.categoryDatabaseRepository.deleteMissing(hiboutikCategories.map((dto) => Number(dto.category_id)));
const nbDeleted = await this.categoryDatabaseRepository.deleteMissing(hiboutikCategories.map((dto) => dto.category_id));
this.logger.log(
`Categories sync ended: ${hiboutikCategories.length - nbFailed}/${hiboutikCategories.length} synced, ${nbFailed} failed, ` +
`${nbParentsFailed} parent links failed, ${nbDeleted} deleted.`,

90
src/services/hiboutik/customer.hiboutik.service.spec.ts
View File

@@ -2,8 +2,13 @@ import { Logger } from '@nestjs/common';
import { CustomerHiboutikService } from './customer.hiboutik.service';
describe('CustomerHiboutikService', () => {
const customerHiboutikRepository = { getCustomers: jest.fn(), getCustomerById: jest.fn(), getAddressById: jest.fn() };
const customerDatabaseRepository = { upsert: jest.fn(), deleteMissing: jest.fn() };
const customerHiboutikRepository = {
getCustomers: jest.fn(),
getCustomerById: jest.fn(),
findCustomerById: jest.fn(),
getAddressById: jest.fn(),
};
const customerDatabaseRepository = { upsert: jest.fn(), getAllHiboutikIds: jest.fn(), deleteByHiboutikIds: jest.fn() };
const customerConverter = { fromHiboutik: jest.fn() };
const customerAddressDatabaseRepository = { upsert: jest.fn(), deleteMissing: jest.fn() };
const customerAddressConverter = { fromHiboutik: jest.fn((dto: unknown) => dto) };
@@ -23,7 +28,9 @@ describe('CustomerHiboutikService', () => {
customerHiboutikRepository.getCustomers.mockResolvedValue([]);
customerHiboutikRepository.getCustomerById.mockResolvedValue({ customers_id: 1, addresses: [] });
customerDatabaseRepository.upsert.mockResolvedValue({ id: 'customer-uuid' });
customerDatabaseRepository.deleteMissing.mockResolvedValue(0);
customerDatabaseRepository.getAllHiboutikIds.mockResolvedValue([]);
customerDatabaseRepository.deleteByHiboutikIds.mockImplementation((ids: number[]) => Promise.resolve(ids.length));
customerHiboutikRepository.findCustomerById.mockResolvedValue(null);
customerAddressDatabaseRepository.deleteMissing.mockResolvedValue(0);
orderDatabaseRepository.getCompletedSaleIds.mockResolvedValue(new Set());
saleHiboutikRepository.getSaleById.mockImplementation((id: number) => Promise.resolve({ sale_id: id, quotation: 0 }));
@@ -39,27 +46,61 @@ describe('CustomerHiboutikService', () => {
);
});
it('deletes customers missing from Hiboutik', async () => {
customerHiboutikRepository.getCustomers.mockResolvedValue([{ customers_id: 1 }, { customers_id: '2' }]);
describe('deletions', () => {
it('deletes the stored customers missing from the list once Hiboutik confirms they no longer exist', async () => {
customerHiboutikRepository.getCustomers.mockResolvedValue([{ customers_id: 1 }, { customers_id: 2 }]);
customerDatabaseRepository.getAllHiboutikIds.mockResolvedValue([1, 2, 3, 4]);
await service.syncCustomers();
await service.syncCustomers();
expect(customerDatabaseRepository.deleteMissing).toHaveBeenCalledWith([1, 2]);
expect(customerDatabaseRepository.upsert).toHaveBeenCalledTimes(2);
});
expect(customerHiboutikRepository.findCustomerById.mock.calls).toEqual([[3], [4]]);
expect(customerDatabaseRepository.deleteByHiboutikIds).toHaveBeenCalledWith([3, 4]);
expect(customerDatabaseRepository.upsert).toHaveBeenCalledTimes(2);
});
it('deletes every customer when Hiboutik answers with an empty list', async () => {
await service.syncCustomers();
it('keeps and syncs a customer missing from the list (page shift) that still exists in Hiboutik', async () => {
customerHiboutikRepository.getCustomers.mockResolvedValue([{ customers_id: 1 }]);
customerDatabaseRepository.getAllHiboutikIds.mockResolvedValue([1, 2]);
customerHiboutikRepository.findCustomerById.mockResolvedValue({ customers_id: 2 });
expect(customerDatabaseRepository.deleteMissing).toHaveBeenCalledWith([]);
});
await service.syncCustomers();
it('deletes nothing when the Hiboutik call fails', async () => {
customerHiboutikRepository.getCustomers.mockRejectedValue(new Error('Request failed with status code 500'));
expect(customerDatabaseRepository.deleteByHiboutikIds).toHaveBeenCalledWith([]);
expect(customerHiboutikRepository.getCustomerById.mock.calls).toEqual([[1], [2]]);
});
await expect(service.syncCustomers()).rejects.toThrow('500');
it('keeps a customer whose check fails', async () => {
customerDatabaseRepository.getAllHiboutikIds.mockResolvedValue([3, 4]);
customerHiboutikRepository.findCustomerById.mockRejectedValueOnce(new Error('Request failed with status code 500'));
expect(customerDatabaseRepository.deleteMissing).not.toHaveBeenCalled();
await service.syncCustomers();
expect(customerDatabaseRepository.deleteByHiboutikIds).toHaveBeenCalledWith([4]);
});
it('deletes every confirmed customer when Hiboutik answers with an empty list', async () => {
customerDatabaseRepository.getAllHiboutikIds.mockResolvedValue([1, 2]);
await service.syncCustomers();
expect(customerDatabaseRepository.deleteByHiboutikIds).toHaveBeenCalledWith([1, 2]);
});
it('deletes nothing when the Hiboutik list call fails', async () => {
customerHiboutikRepository.getCustomers.mockRejectedValue(new Error('Request failed with status code 500'));
await expect(service.syncCustomers()).rejects.toThrow('500');
expect(customerDatabaseRepository.deleteByHiboutikIds).not.toHaveBeenCalled();
});
it('syncs once a customer listed twice', async () => {
customerHiboutikRepository.getCustomers.mockResolvedValue([{ customers_id: 1 }, { customers_id: 1 }]);
await service.syncCustomers();
expect(customerHiboutikRepository.getCustomerById).toHaveBeenCalledTimes(1);
});
});
it('keeps syncing the other customers when one fails', async () => {
@@ -80,8 +121,8 @@ describe('CustomerHiboutikService', () => {
customerHiboutikRepository.getCustomerById.mockResolvedValue({
customers_id: 1,
addresses: [
{ address_id: 10, customers_id: 1 },
{ address_id: '11', customers_id: 999 },
{ address_id: 10, customers_id: 1, company: '' },
{ address_id: 11, customers_id: 999, company: '' },
],
});
@@ -89,7 +130,7 @@ describe('CustomerHiboutikService', () => {
expect(customerAddressDatabaseRepository.deleteMissing).toHaveBeenCalledWith('customer-uuid', [10, 11]);
expect(customerAddressDatabaseRepository.upsert).toHaveBeenCalledTimes(2);
expect(customerAddressConverter.fromHiboutik).toHaveBeenLastCalledWith({ address_id: '11', customers_id: 1 });
expect(customerAddressConverter.fromHiboutik).toHaveBeenLastCalledWith({ address_id: 11, customers_id: 1, company: '' });
expect(customerAddressDatabaseRepository.deleteMissing.mock.invocationCallOrder[0]).toBeLessThan(
customerAddressDatabaseRepository.upsert.mock.invocationCallOrder[0],
);
@@ -155,7 +196,10 @@ describe('CustomerHiboutikService', () => {
it('keeps syncing the other addresses when one fails', async () => {
customerHiboutikRepository.getCustomerById.mockResolvedValue({
customers_id: 1,
addresses: [{ address_id: 10 }, { address_id: 11 }],
addresses: [
{ address_id: 10, company: '' },
{ address_id: 11, company: '' },
],
});
customerAddressDatabaseRepository.upsert.mockRejectedValueOnce(new Error('boom'));
@@ -171,7 +215,7 @@ describe('CustomerHiboutikService', () => {
customerHiboutikRepository.getCustomerById.mockResolvedValue({
customers_id: 1,
addresses: [],
sales: [{ sale_id: 5 }, { sale_id: '6' }, { sale_id: 7 }],
sales: [{ sale_id: 5 }, { sale_id: 6 }, { sale_id: 7 }],
});
});
@@ -180,7 +224,7 @@ describe('CustomerHiboutikService', () => {
await service.syncCustomers();
expect(orderDatabaseRepository.getCompletedSaleIds).toHaveBeenCalledWith([5, 6, 7]);
expect(orderDatabaseRepository.getCompletedSaleIds).toHaveBeenCalledWith(1, [5, 6, 7]);
expect(saleHiboutikRepository.getSaleById.mock.calls.map((call: unknown[]) => call[0])).toEqual([5, 7]);
expect(orderDatabaseRepository.upsert).toHaveBeenCalledTimes(2);
expect(orderConverter.fromHiboutik).toHaveBeenCalledWith({ sale_id: 5, quotation: 0, customer_id: 1 });

62
src/services/hiboutik/customer.hiboutik.service.ts
View File

@@ -13,6 +13,7 @@ import { forEachSafely } from '../../utils/sync';
import { SaleHiboutikRepository } from '../../hiboutik/sale.hiboutik.repository';
import { OrderDatabaseRepository } from '../../database/order.database.repository';
import { OrderConverter } from '../../models/converter/order.converter';
import { rethrowIfRateLimited } from '../../hiboutik/hiboutik.client';
@Injectable()
export class CustomerHiboutikService {
@@ -30,35 +31,66 @@ export class CustomerHiboutikService {
) {}
/**
* Mirrors Hiboutik customers: deletes the customers missing from Hiboutik, then fetches and upserts each of them
* with its addresses and orders (one API call per customer, as the list endpoint lacks addresses and sales, plus one
* per sale not yet stored as completed).
* Mirrors Hiboutik customers: deletes the customers Hiboutik confirms as deleted, then fetches and upserts each of
* them with its addresses and orders (one API call per customer, as the list endpoint lacks addresses and sales, plus
* one per sale not yet stored as completed).
*/
async syncCustomers() {
const hiboutikCustomers = await this.customerHiboutikRepository.getCustomers();
// A page shift can list a customer twice: sync it once
const listedIds = [...new Set(hiboutikCustomers.map((dto) => dto.customers_id))];
// Hiboutik is trusted: a successful (2xx) answer, even an empty list, is the truth. An HTTP error throws before this point.
const nbDeleted = await this.customerDatabaseRepository.deleteMissing(hiboutikCustomers.map((dto) => Number(dto.customers_id)));
const { nbDeleted, unlistedIds } = await this.deleteMissingCustomers(listedIds);
const customerIds = [...listedIds, ...unlistedIds];
let nbFailed = 0;
let nbFailedAddresses = 0;
let nbFailedOrders = 0;
for (const hiboutikCustomer of hiboutikCustomers) {
for (const customerId of customerIds) {
try {
const hiboutikFullCustomer = await this.customerHiboutikRepository.getCustomerById(hiboutikCustomer.customers_id);
const hiboutikFullCustomer = await this.customerHiboutikRepository.getCustomerById(customerId);
const customer = await this.customerDatabaseRepository.upsert(this.customerConverter.fromHiboutik(hiboutikFullCustomer));
nbFailedAddresses += await this.syncAddresses(customer.id, hiboutikFullCustomer);
nbFailedOrders += await this.syncOrders(hiboutikFullCustomer);
} catch (error) {
rethrowIfRateLimited(error);
nbFailed++;
this.logger.error(`Failed to sync Hiboutik customer ${hiboutikCustomer.customers_id}`, (error as Error)?.stack);
this.logger.error(`Failed to sync Hiboutik customer ${customerId}`, (error as Error)?.stack);
}
}
this.logger.log(
`Customers sync ended: ${hiboutikCustomers.length - nbFailed}/${hiboutikCustomers.length} synced, ${nbFailed} failed, ${nbDeleted} deleted, ${nbFailedAddresses} addresses failed, ${nbFailedOrders} orders failed.`,
`Customers sync ended: ${customerIds.length - nbFailed}/${customerIds.length} synced, ${nbFailed} failed, ${nbDeleted} deleted, ` +
`${unlistedIds.length} missing from the list but still in Hiboutik, ${nbFailedAddresses} addresses failed, ${nbFailedOrders} orders failed.`,
);
}
/**
* Deletes the stored customers missing from the Hiboutik list, once Hiboutik confirms one by one that they no longer
* exist. The list is paginated: a customer created or deleted in Hiboutik while it is walked shifts the following
* pages, so a customer can be missing from it without being deleted. Hiboutik is trusted: a 404 or an empty 2xx
* answer is a confirmation. A customer whose check fails is kept.
* @returns the number of deleted customers, and the ids of the customers that still exist although not listed.
*/
private async deleteMissingCustomers(listedIds: number[]): Promise<{ nbDeleted: number; unlistedIds: number[] }> {
const listed = new Set(listedIds);
const missingIds = (await this.customerDatabaseRepository.getAllHiboutikIds()).filter((id) => !listed.has(id));
const deletedIds: number[] = [];
const unlistedIds: number[] = [];
for (const id of missingIds) {
try {
const customer = await this.customerHiboutikRepository.findCustomerById(id);
(customer ? unlistedIds : deletedIds).push(id);
} catch (error) {
rethrowIfRateLimited(error);
this.logger.warn(`Failed to check whether Hiboutik customer ${id} still exists, it is kept: ${(error as Error)?.message}`);
}
}
const nbDeleted = await this.customerDatabaseRepository.deleteByHiboutikIds(deletedIds);
return { nbDeleted, unlistedIds };
}
/**
* Mirrors the addresses embedded in a full Hiboutik customer: deletes the ones missing from Hiboutik, then upserts
* each of them (a failing address is logged and skipped).
@@ -73,13 +105,13 @@ export class CustomerHiboutikService {
await this.customerAddressDatabaseRepository.deleteMissing(
customerId,
hiboutikCustomer.addresses.map((dto) => Number(dto.address_id)),
hiboutikCustomer.addresses.map((dto) => dto.address_id),
);
return forEachSafely(
hiboutikCustomer.addresses,
// Linked to the parent customer, whatever the embedded customers_id says
async (dto) => {
const address = dto.company?.trim() ? await this.withCompanyDetails(dto) : dto;
const address = dto.company.trim() ? await this.withCompanyDetails(dto) : dto;
await this.customerAddressDatabaseRepository.upsert(
this.customerAddressConverter.fromHiboutik({ ...address, customers_id: hiboutikCustomer.customers_id }),
);
@@ -106,6 +138,7 @@ export class CustomerHiboutikService {
updated_at: details.updated_at,
};
} catch (error) {
rethrowIfRateLimited(error);
this.logger.warn(`Failed to fetch company details of Hiboutik address ${dto.address_id}: ${(error as Error)?.message}`);
return dto;
}
@@ -124,15 +157,16 @@ export class CustomerHiboutikService {
// A completed sale no longer changes in Hiboutik: no need to fetch it again
const completedSaleIds = await this.orderDatabaseRepository.getCompletedSaleIds(
hiboutikCustomer.sales.map((dto) => Number(dto.sale_id)),
hiboutikCustomer.customers_id,
hiboutikCustomer.sales.map((dto) => dto.sale_id),
);
const salesToSync = hiboutikCustomer.sales.filter((dto) => !completedSaleIds.has(Number(dto.sale_id)));
const salesToSync = hiboutikCustomer.sales.filter((dto) => !completedSaleIds.has(dto.sale_id));
return forEachSafely(
salesToSync,
async (dto) => {
const sale = await this.saleHiboutikRepository.getSaleById(dto.sale_id);
if (Number(sale.quotation) === 1) return;
if (sale.quotation === 1) return;
// Linked to the parent customer, whatever the sale says
await this.orderDatabaseRepository.upsert(
this.orderConverter.fromHiboutik({ ...sale, customer_id: hiboutikCustomer.customers_id }),

75
src/services/hiboutik/hiboutik.service.spec.ts
View File

@@ -1,5 +1,6 @@
import { Logger } from '@nestjs/common';
import { Logger, ServiceUnavailableException } from '@nestjs/common';
import { HiboutikService } from './hiboutik.service';
import { HiboutikRateLimitError } from '../../hiboutik/hiboutik.client';
describe('HiboutikService', () => {
const brandService = { syncBrands: jest.fn() };
@@ -9,12 +10,16 @@ describe('HiboutikService', () => {
const customerService = { syncCustomers: jest.fn() };
const syncLock = { tryAcquire: jest.fn() };
const release = jest.fn();
// SYNC_SHUTDOWN_TIMEOUT of 1 s
const configService = { get: jest.fn().mockReturnValue(1) };
let service: HiboutikService;
const flush = () => new Promise((resolve) => setImmediate(resolve));
let logWarn: jest.SpyInstance;
beforeAll(() => {
jest.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined);
jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
logWarn = jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
jest.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined);
});
@@ -34,6 +39,7 @@ describe('HiboutikService', () => {
productService as never,
supplierService as never,
syncLock as never,
configService as never,
);
});
@@ -45,7 +51,8 @@ describe('HiboutikService', () => {
productService.syncProducts.mockImplementation(() => Promise.resolve(order.push('products')));
customerService.syncCustomers.mockImplementation(() => Promise.resolve(order.push('customers')));
await service.syncAll();
// private method, reached through an index access on purpose
await service['syncAll']();
expect(order).toEqual(['brands', 'categories', 'suppliers', 'products', 'customers']);
});
@@ -53,7 +60,8 @@ describe('HiboutikService', () => {
it('continues with the next steps when one fails', async () => {
brandService.syncBrands.mockRejectedValue(new Error('boom'));
await service.syncAll();
// private method, reached through an index access on purpose
await service['syncAll']();
expect(productService.syncProducts).toHaveBeenCalled();
expect(customerService.syncCustomers).toHaveBeenCalled();
@@ -95,11 +103,68 @@ describe('HiboutikService', () => {
});
it('releases the lock even when the synchronization crashes', async () => {
jest.spyOn(service as never, 'syncAll').mockRejectedValue(new Error('crash'));
jest.spyOn(service as unknown as { syncAll: () => Promise<void> }, 'syncAll').mockRejectedValue(new Error('crash'));
await expect(service.startSync()).resolves.toBe(true);
await flush();
expect(release).toHaveBeenCalledTimes(1);
});
it('stops the synchronization when the Hiboutik rate limit is reached', async () => {
productService.syncProducts.mockRejectedValue(new HiboutikRateLimitError('/products'));
await service['syncAll']();
expect(customerService.syncCustomers).not.toHaveBeenCalled();
});
describe('shutdown', () => {
afterEach(() => {
jest.useRealTimers();
});
it('waits for the running synchronization to end, and refuses new ones meanwhile', async () => {
let finishBrands: () => void = () => undefined;
brandService.syncBrands.mockReturnValue(new Promise<void>((resolve) => (finishBrands = resolve)));
await service.startSync();
let shutDown = false;
const shutdown = service.beforeApplicationShutdown().then(() => (shutDown = true));
await flush();
expect(shutDown).toBe(false);
await expect(service.startSync()).rejects.toThrow(ServiceUnavailableException);
finishBrands();
await shutdown;
expect(release).toHaveBeenCalledTimes(1);
});
it('gives up waiting after the timeout', async () => {
jest.useFakeTimers();
brandService.syncBrands.mockReturnValue(new Promise<void>(() => undefined));
await service.startSync();
const shutdown = service.beforeApplicationShutdown();
await jest.advanceTimersByTimeAsync(1_000);
await expect(shutdown).resolves.toBeUndefined();
expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('did not end in time'));
});
it('returns at once when no synchronization is running', async () => {
await expect(service.beforeApplicationShutdown()).resolves.toBeUndefined();
});
it('releases the lock and refuses when the shutdown began while the lock was requested', async () => {
syncLock.tryAcquire.mockImplementation(async () => {
await service.beforeApplicationShutdown();
return release;
});
await expect(service.startSync()).rejects.toThrow(ServiceUnavailableException);
expect(release).toHaveBeenCalledTimes(1);
expect(brandService.syncBrands).not.toHaveBeenCalled();
});
});
});

63
src/services/hiboutik/hiboutik.service.ts
View File

@@ -1,15 +1,24 @@
import { Injectable, Logger } from '@nestjs/common';
import { BeforeApplicationShutdown, Injectable, Logger, ServiceUnavailableException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { BrandHiboutikService } from './brand.hiboutik.service';
import { CategoryHiboutikService } from './category.hiboutik.service';
import { CustomerHiboutikService } from './customer.hiboutik.service';
import { SupplierHiboutikService } from './supplier.hiboutik.service';
import { ProductHiboutikService } from './product.hiboutik.service';
import { ReleaseLock, SyncLockDatabaseRepository } from '../../database/syncLock.database.repository';
import { HiboutikRateLimitError } from '../../hiboutik/hiboutik.client';
/** Default of `SYNC_SHUTDOWN_TIMEOUT`: below the 30 s grace period that Kubernetes gives before killing a container. */
const DEFAULT_SHUTDOWN_TIMEOUT_SECONDS = 25;
/** Entry point of the Hiboutik → database synchronization: runs each entity sync in dependency order. */
@Injectable()
export class HiboutikService {
export class HiboutikService implements BeforeApplicationShutdown {
private readonly logger = new Logger('HiboutikService');
private readonly shutdownTimeoutMs: number;
/** Synchronization running on this instance, if any. */
private running: Promise<void> | null = null;
private stopping = false;
constructor(
private readonly brandHiboutikService: BrandHiboutikService,
@@ -18,23 +27,60 @@ export class HiboutikService {
private readonly productHiboutikService: ProductHiboutikService,
private readonly supplierHiboutikService: SupplierHiboutikService,
private readonly syncLock: SyncLockDatabaseRepository,
) {}
configService: ConfigService,
) {
this.shutdownTimeoutMs = (configService.get<number>('SYNC_SHUTDOWN_TIMEOUT') ?? DEFAULT_SHUTDOWN_TIMEOUT_SECONDS) * 1000;
}
/**
* Starts a full synchronization in the background, if no other one is running on this instance or any other.
* The Postgres advisory lock is the only guard: each request takes it on its own connection (its own session),
* so it also refuses concurrent calls on this very instance. It is held until the synchronization ends.
* @returns false if a synchronization is already running (nothing is started).
* @throws ServiceUnavailableException if this instance is shutting down (another instance can take it).
* @throws if the lock cannot be requested (e.g. database unreachable).
*/
async startSync(): Promise<boolean> {
this.refuseIfStopping();
const release = await this.syncLock.tryAcquire();
if (!release) return false;
void this.runLocked(release);
// The shutdown may have begun while the lock was requested: it would not wait for this synchronization
if (this.stopping) {
await release();
this.refuseIfStopping();
}
this.running = this.runLocked(release).finally(() => (this.running = null));
return true;
}
/**
* Shutdown hook: refuses new synchronizations and waits for the running one to end, up to `SYNC_SHUTDOWN_TIMEOUT`
* seconds. Runs before Prisma disconnects (`PrismaService.onApplicationShutdown`). Past the timeout, the shutdown goes
* on: the synchronization then fails on its next database call, without corrupting anything (one transaction per
* product, the lock is released by Postgres with the connection).
*/
async beforeApplicationShutdown(): Promise<void> {
this.stopping = true;
const running = this.running;
if (!running) return;
this.logger.log(`Shutdown requested: waiting for the running Hiboutik synchronization (up to ${this.shutdownTimeoutMs / 1000} s)`);
let timer: NodeJS.Timeout | undefined;
const timedOut = new Promise<true>((resolve) => (timer = setTimeout(() => resolve(true), this.shutdownTimeoutMs)));
const result = await Promise.race([running.then(() => false as const), timedOut]);
clearTimeout(timer);
if (result) {
this.logger.warn('The Hiboutik synchronization did not end in time: shutting down anyway, it will be interrupted');
}
}
private refuseIfStopping(): void {
if (this.stopping) throw new ServiceUnavailableException('This instance is shutting down, retry in a moment');
}
/** Runs the synchronization, then always releases the lock. */
private async runLocked(release: ReleaseLock): Promise<void> {
try {
@@ -46,7 +92,10 @@ export class HiboutikService {
}
}
/** Runs every sync step in order. A failing step is logged and does not stop the next ones. */
/**
* Runs every sync step in order. A failing step is logged and does not stop the next ones, except when the Hiboutik
* rate limit is reached: the next steps would only fail too, so the synchronization stops there.
*/
private async syncAll() {
this.logger.log('Starting global Hiboutik synchronization');
// Order matters: products connect to brands, categories and suppliers by hiboutikId.
@@ -64,6 +113,10 @@ export class HiboutikService {
await step();
} catch (error) {
failedSteps.push(name);
if (error instanceof HiboutikRateLimitError) {
this.logger.error(`Hiboutik synchronization stopped during the ${name} step: ${error.message}`);
return;
}
this.logger.error(`Hiboutik ${name} sync failed`, (error as Error)?.stack);
}
}

84
src/services/hiboutik/product.hiboutik.service.spec.ts
View File

@@ -12,22 +12,20 @@ const hiboutikProduct = (id: number, overrides: Partial<ProductHiboutikDto> = {}
product_supplier: 0,
product_price: '10.00',
product_discount_price: '0.00',
product_supply_price: '5.00',
product_category: 0,
product_display_www: 1,
product_arch: 0,
products_desc: '',
updated_at: '2026-06-28 17:02:55',
product_specific_rules: [],
product_size_details: [],
...overrides,
});
const stock = (productId: number, size: number, available: string): StockHiboutikDto => ({
const stock = (productId: number, size: number, available: number): StockHiboutikDto => ({
product_id: productId,
product_size: size,
stock_available: available,
product_barcode: '',
product_supplier_reference: '',
});
describe('ProductHiboutikService', () => {
@@ -37,6 +35,9 @@ describe('ProductHiboutikService', () => {
const stockHiboutikRepository = { getStocks: jest.fn() };
const productDatabaseRepository = { getAll: jest.fn(), upsert: jest.fn(), deleteAll: jest.fn() };
const productVersionDatabaseRepository = { upsert: jest.fn(), deleteMissing: jest.fn() };
// The transaction client handed to the repositories inside $transaction
const tx = { transaction: true };
const prisma = { $transaction: jest.fn() };
beforeAll(() => {
jest.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined);
@@ -47,6 +48,7 @@ describe('ProductHiboutikService', () => {
beforeEach(() => {
jest.clearAllMocks();
calls = [];
prisma.$transaction.mockImplementation((run: (client: unknown) => Promise<unknown>) => run(tx));
stockHiboutikRepository.getStocks.mockResolvedValue([]);
productDatabaseRepository.getAll.mockResolvedValue([]);
productDatabaseRepository.deleteAll.mockImplementation((ids: string[]) => {
@@ -73,6 +75,7 @@ describe('ProductHiboutikService', () => {
productVersionDatabaseRepository as never,
new ProductConverter(),
stockHiboutikRepository as never,
prisma as never,
);
});
@@ -109,16 +112,17 @@ describe('ProductHiboutikService', () => {
it('creates a single version for a product without sizes', async () => {
productHiboutikRepository.getProducts.mockResolvedValue([hiboutikProduct(1)]);
stockHiboutikRepository.getStocks.mockResolvedValue([stock(1, 0, '4'), stock(2, 0, '99')]);
stockHiboutikRepository.getStocks.mockResolvedValue([stock(1, 0, 4), stock(2, 0, 99)]);
await service.syncProducts();
expect(productDatabaseRepository.upsert).toHaveBeenCalledWith(expect.objectContaining({ hiboutikId: 1, globalStock: 4 }));
expect(productVersionDatabaseRepository.deleteMissing).toHaveBeenCalledWith('uuid-1', [0]);
expect(productDatabaseRepository.upsert).toHaveBeenCalledWith(expect.objectContaining({ hiboutikId: 1, globalStock: 4 }), tx);
expect(productVersionDatabaseRepository.deleteMissing).toHaveBeenCalledWith('uuid-1', [0], tx);
expect(productVersionDatabaseRepository.upsert).toHaveBeenCalledTimes(1);
expect(productVersionDatabaseRepository.upsert).toHaveBeenCalledWith(
'uuid-1',
expect.objectContaining({ hiboutikSizeId: 0, barcode: 'barcode-1', stock: 4, versionName: 'Product 1', discountPrice: null }),
tx,
);
});
@@ -129,34 +133,44 @@ describe('ProductHiboutikService', () => {
{ size_id: 10, size_name: 'S', barcode: 'b-s' },
{ size_id: 11, size_name: 'M', barcode: '' },
],
product_specific_rules: [{ size_id: 10, barcode: 'b-s', product_price: '12.00', product_discount_price: '0.00' }],
product_specific_rules: [{ size_id: 10, product_price: '12.00', product_discount_price: '0.00' }],
}),
]);
stockHiboutikRepository.getStocks.mockResolvedValue([stock(1, 10, '3'), stock(1, 11, '2')]);
stockHiboutikRepository.getStocks.mockResolvedValue([stock(1, 10, 3), stock(1, 11, 2)]);
await service.syncProducts();
expect(productDatabaseRepository.upsert).toHaveBeenCalledWith(expect.objectContaining({ globalStock: 5 }));
expect(productVersionDatabaseRepository.deleteMissing).toHaveBeenCalledWith('uuid-1', [10, 11]);
expect(productDatabaseRepository.upsert).toHaveBeenCalledWith(expect.objectContaining({ globalStock: 5 }), tx);
expect(productVersionDatabaseRepository.deleteMissing).toHaveBeenCalledWith('uuid-1', [10, 11], tx);
expect(calls).toEqual(['deleteProducts', 'upsertProduct', 'deleteVersions', 'upsertVersion', 'upsertVersion']);
expect(productVersionDatabaseRepository.upsert).toHaveBeenNthCalledWith(1, 'uuid-1', {
barcode: 'b-s',
discountPrice: null,
hiboutikSizeId: 10,
price: '12.00',
product: { connect: { id: 'uuid-1' } },
stock: 3,
versionName: 'S',
});
expect(productVersionDatabaseRepository.upsert).toHaveBeenNthCalledWith(2, 'uuid-1', {
barcode: null,
discountPrice: null,
hiboutikSizeId: 11,
price: null,
product: { connect: { id: 'uuid-1' } },
stock: 2,
versionName: 'M',
});
expect(productVersionDatabaseRepository.upsert).toHaveBeenNthCalledWith(
1,
'uuid-1',
{
barcode: 'b-s',
discountPrice: null,
hiboutikSizeId: 10,
price: '12.00',
product: { connect: { id: 'uuid-1' } },
stock: 3,
versionName: 'S',
},
tx,
);
expect(productVersionDatabaseRepository.upsert).toHaveBeenNthCalledWith(
2,
'uuid-1',
{
barcode: null,
discountPrice: null,
hiboutikSizeId: 11,
price: null,
product: { connect: { id: 'uuid-1' } },
stock: 2,
versionName: 'M',
},
tx,
);
});
it('keeps syncing the other products when one fails', async () => {
@@ -168,4 +182,16 @@ describe('ProductHiboutikService', () => {
expect(productDatabaseRepository.upsert).toHaveBeenCalledTimes(2);
expect(productVersionDatabaseRepository.upsert).toHaveBeenCalledTimes(1);
});
it('writes a product and its versions in one transaction per product, and counts a failed one', async () => {
productHiboutikRepository.getProducts.mockResolvedValue([hiboutikProduct(1), hiboutikProduct(2)]);
productVersionDatabaseRepository.upsert.mockRejectedValueOnce(new Error('boom'));
await expect(service.syncProducts()).resolves.toBeUndefined();
expect(prisma.$transaction).toHaveBeenCalledTimes(2);
// the failure inside the transaction is what makes Prisma roll the product back
await expect(prisma.$transaction.mock.results[0].value).rejects.toThrow('boom');
await expect(prisma.$transaction.mock.results[1].value).resolves.toBeUndefined();
});
});

59
src/services/hiboutik/product.hiboutik.service.ts
View File

@@ -9,6 +9,8 @@ import { StockHiboutikDto } from '../../models/hiboutikDTO/stock.hiboutik.dto';
import { Product } from '../../../prisma/generated/client';
import { ProductVersionCreateInput } from '../../../prisma/generated/models/ProductVersion';
import { parseOptionalHiboutikPrice } from '../../utils/price';
import { PrismaService } from '../../../prisma/prisma.service';
import { rethrowIfRateLimited } from '../../hiboutik/hiboutik.client';
/** Version id used for products that have no sizes in Hiboutik. */
const NO_SIZE_ID = 0;
@@ -23,6 +25,7 @@ export class ProductHiboutikService {
private readonly productVersionDatabaseRepository: ProductVersionDatabaseRepository,
private readonly productConverter: ProductConverter,
private readonly stockHiboutikRepository: StockHiboutikRepository,
private readonly prisma: PrismaService,
) {}
/** Mirrors Hiboutik products, their versions (sizes) and stocks. Products missing from Hiboutik are deleted. */
@@ -33,16 +36,24 @@ export class ProductHiboutikService {
this.productDatabaseRepository.getAll(),
]);
// Deletion happens before upserts so that a product recreated in Hiboutik (new id, same barcode)
// does not collide with the old one on the unique barcode.
// Deletion happens before the upserts, so that order lines of a deleted product are unlinked before anything else changes
const nbDeleted = await this.deleteMissingProducts(productsHiboutik, existingProducts);
// Grouped once by product: one pass over the stocks instead of one per product
const stocksByProduct = new Map<number, StockHiboutikDto[]>();
for (const stock of hiboutikStocks) {
const productStocks = stocksByProduct.get(stock.product_id);
if (productStocks) productStocks.push(stock);
else stocksByProduct.set(stock.product_id, [stock]);
}
let nbFailed = 0;
for (const productHiboutik of productsHiboutik) {
const hiboutikProductStocks = hiboutikStocks.filter((stock) => stock.product_id === Number(productHiboutik.product_id));
const hiboutikProductStocks = stocksByProduct.get(productHiboutik.product_id) ?? [];
try {
await this.updateProduct(productHiboutik, hiboutikProductStocks);
} catch (error) {
rethrowIfRateLimited(error);
nbFailed++;
this.logger.error(`Failed to sync Hiboutik product ${productHiboutik.product_id}`, (error as Error)?.stack);
}
@@ -56,27 +67,35 @@ export class ProductHiboutikService {
/** @returns the number of deleted products. */
private async deleteMissingProducts(productsHiboutik: ProductHiboutikDto[], existingProducts: Product[]): Promise<number> {
// Hiboutik is trusted: a successful (2xx) answer, even an empty list, is the truth. An HTTP error throws before this point.
const hiboutikIds = new Set(productsHiboutik.map((product) => Number(product.product_id)));
const hiboutikIds = new Set(productsHiboutik.map((product) => product.product_id));
const missingProductIds = existingProducts.filter((product) => !hiboutikIds.has(product.hiboutikId)).map((product) => product.id);
return this.productDatabaseRepository.deleteAll(missingProductIds);
}
/** Upserts a product with its global stock, then replaces its versions. */
/**
* Upserts a product with its global stock, then replaces its versions, in a single transaction: on failure, nothing
* is changed (no global stock out of step with the versions).
*/
private async updateProduct(productHiboutik: ProductHiboutikDto, productStocks: StockHiboutikDto[]): Promise<void> {
const newProduct = this.productConverter.fromHiboutik(productHiboutik);
newProduct.globalStock = productStocks.map((stock) => Number(stock.stock_available)).reduce((sum, stock) => sum + stock, 0);
newProduct.globalStock = productStocks.reduce((sum, stock) => sum + stock.stock_available, 0);
const savedProduct: Product = await this.productDatabaseRepository.upsert(newProduct);
const versions = this.buildVersions(productHiboutik, productStocks, savedProduct);
await this.prisma.$transaction(
async (tx) => {
const savedProduct: Product = await this.productDatabaseRepository.upsert(newProduct, tx);
const versions = this.buildVersions(productHiboutik, productStocks, savedProduct);
// Same reason as for products: delete removed versions first to free their barcodes
await this.productVersionDatabaseRepository.deleteMissing(
savedProduct.id,
versions.map((version) => version.hiboutikSizeId),
await this.productVersionDatabaseRepository.deleteMissing(
savedProduct.id,
versions.map((version) => version.hiboutikSizeId),
tx,
);
for (const version of versions) {
await this.productVersionDatabaseRepository.upsert(savedProduct.id, version, tx);
}
},
{ timeout: 15_000 },
);
for (const version of versions) {
await this.productVersionDatabaseRepository.upsert(savedProduct.id, version);
}
}
/**
@@ -88,8 +107,8 @@ export class ProductHiboutikService {
productStocks: StockHiboutikDto[],
savedProduct: Product,
): ProductVersionCreateInput[] {
const specificRules = productHiboutik.product_specific_rules ?? [];
const sizesDetails = productHiboutik.product_size_details ?? [];
const specificRules = productHiboutik.product_specific_rules;
const sizesDetails = productHiboutik.product_size_details;
if (sizesDetails.length === 0) {
return [
@@ -106,8 +125,8 @@ export class ProductHiboutikService {
}
return sizesDetails.map((sizeDetails) => {
const sizeId = Number(sizeDetails.size_id);
const specificRule = specificRules.find((rule) => Number(rule.size_id) === sizeId);
const sizeId = sizeDetails.size_id;
const specificRule = specificRules.find((rule) => rule.size_id === sizeId);
const stock = productStocks.find((stock) => stock.product_size === sizeId);
// Values are always set explicitly (null / 0 rather than undefined), otherwise Prisma would keep
// the previous value on update when a rule or a stock is removed in Hiboutik.
@@ -117,7 +136,7 @@ export class ProductHiboutikService {
hiboutikSizeId: sizeId,
price: parseOptionalHiboutikPrice(specificRule?.product_price),
product: { connect: { id: savedProduct.id } },
stock: stock ? Number(stock.stock_available) : 0,
stock: stock?.stock_available ?? 0,
versionName: sizeDetails.size_name,
};
});

2
src/services/hiboutik/supplier.hiboutik.service.ts
View File

@@ -26,7 +26,7 @@ export class SupplierHiboutikService {
);
// Hiboutik is trusted: a successful (2xx) answer, even an empty list, is the truth. An HTTP error throws before this point.
const nbDeleted = await this.supplierDatabaseRepository.deleteMissing(hiboutikSuppliers.map((dto) => Number(dto.supplier_id)));
const nbDeleted = await this.supplierDatabaseRepository.deleteMissing(hiboutikSuppliers.map((dto) => dto.supplier_id));
this.logger.log(
`Suppliers sync ended: ${hiboutikSuppliers.length - nbFailed}/${hiboutikSuppliers.length} synced, ${nbFailed} failed, ${nbDeleted} deleted.`,
);