From c0f801684d0036b764da39fe951cb2da66c22481 Mon Sep 17 00:00:00 2001 From: Niv Greenstein <88280771+NivGreenstein@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:15:37 +0300 Subject: [PATCH 1/5] feat: add a shigola mode that purges or seeds each metatile (MAPCO-11992) With app.map.provider set to shigola, retiler keeps shigola's tile cache fresh instead of rendering tiles: each metatile job becomes one request to shigola's Cache Admin API, POST {app.map.url}/cache/{purge|seed}, with the metatile's lng/lat bbox and its zoom as both ends of the zoom range. A seed always sends overwrite. No map is fetched, split or stored, and detiler is not used. The metatile processor is now chosen by the map provider: TileProcessor for wms and arcgis, ShigolaCacheProcessor for shigola. A 4xx that repeating cannot fix (400, 422) ends the job without a retry. As pg-boss can only fail a fetched job into a retry, such a job is cancelled, through a NonRetryableError the queue provider recognises. 401, 403, 404, 429, 5xx, timeouts and connection errors fail the job for pg-boss to retry, since a token or url mistake is fixed by a deploy and would otherwise cancel every metatile meanwhile. Shigola mode has its own request, tile and duration metrics, labelled by operation and outcome, and a shigola.cache span per job. The helm chart takes the admin token from its secret or an existing one, and defaults the tile matrix sets to WorldCRS84Quad, the grid the metatile zoom is in. Needs the shigola config of @map-colonies/schemas' retiler v2 schema. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 34 ++- helm/templates/configmap.yaml | 8 + helm/templates/deployment.yaml | 9 + helm/templates/secret.yaml | 3 + helm/values.yaml | 14 ++ src/app.ts | 9 +- src/common/constants.ts | 2 + src/common/errors.ts | 10 + src/common/metrics.ts | 18 ++ src/common/tracing/job.ts | 7 + src/containerConfig.ts | 15 ++ src/retiler/interfaces.ts | 4 + .../jobQueueProvider/pgBossJobQueue.ts | 11 +- src/retiler/shigola/shigolaCacheProcessor.ts | 193 ++++++++++++++++++ src/retiler/shigola/types.ts | 26 +++ src/retiler/tileProcessor.ts | 4 +- tests/unit/containerConfig.spec.ts | 61 ++++++ .../jobQueueProvider/pgBossJobQueue.spec.ts | 20 ++ .../shigola/shigolaCacheProcessor.spec.ts | 185 +++++++++++++++++ 19 files changed, 622 insertions(+), 11 deletions(-) create mode 100644 src/common/errors.ts create mode 100644 src/retiler/shigola/shigolaCacheProcessor.ts create mode 100644 src/retiler/shigola/types.ts create mode 100644 tests/unit/containerConfig.spec.ts create mode 100644 tests/unit/shigola/shigolaCacheProcessor.spec.ts diff --git a/README.md b/README.md index 2afab8f..e1db0cc 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ flowchart TD `app.jobQueue.noSupervisor`: flag for maintenance and monitoring operations on the job queue. defaults to true (meaning no supervision) -`app.map.provider`: the map provider type, wms or arcgis +`app.map.provider`: the map provider type, wms, arcgis or shigola. see [Shigola mode](#shigola-mode) `app.map.url`: the url of the map provider service to fetch the map from @@ -88,6 +88,38 @@ will result in the key: "prefix/3/10/4.png" `app.tilesStorage.shouldFilterBlankTiles`: determine if tiles or their sub-tiles will not be storaged on configured providers if found as blanked +## Shigola mode + +With `app.map.provider` set to `shigola`, retiler keeps [shigola](https://github.com/MapColonies/shigola)'s tile cache fresh instead of rendering tiles itself. Each metatile job becomes exactly one request to shigola's Cache Admin API, `POST {app.map.url}/cache/{purge|seed}`, with the metatile's lng/lat bbox and its zoom as both the min and max of the zoom range. Shigola purges or renders the covered tiles in its own cache, so no map is fetched, split or stored: `app.map.wms` and `app.tilesStorage` are unused, and so is detiler, even when enabled. + +`app.map.url`: shigola's Cache Admin API, served on its own admin port + +`app.map.shigola.operation`: `purge` (default) or `seed`. a seed always sends `"overwrite": true`, and needs a shigola with `POST /cache/seed` + +`app.map.shigola.targets`: the shigola maps to work on, as `[{"map": "osm", "layers": ["roads"]}]`. `layers` is optional and only valid for purge, since shigola seeds whole maps only + +`app.map.shigola.tileMatrixSets`: optional, work on only these tile matrix sets. omitted, every set each map serves. the zoom sent is the metatile's zoom in WorldCRS84Quad, so `["WorldCRS84Quad"]` (the helm chart's default) keeps each request to exactly the metatile's tiles, while other sets would be refreshed at that zoom number over a different extent + +`app.map.shigola.token`: optional, the Cache Admin API bearer token (shigola's `webserver.admin.token`). omitted or empty, no token is sent. the helm chart takes it either as `env.map.shigola.token`, stored in the chart's secret, or from a key of an existing secret with `env.map.shigola.existingSecret.name` and `.key` + +`app.map.shigola.timeoutMs`: the request timeout in ms, defaults to 70000. keep it above shigola's `request_timeout_ms` (60000 by default) so shigola's own timeout answer arrives first + +Each job ends by the request's outcome: + +| response | job | why | +| --- | --- | --- | +| 2xx | completed | shigola answers 500 when a tile fails, so a 2xx is done. any failed count it still reports goes to the tiles metric | +| 4xx other than 401, 403, 404 and 429 (bad bbox, unknown map, too many tiles) | cancelled, never retried | repeating cannot fix it. pg-boss can only fail a fetched job into a retry, so cancel is how it ends without one: the job ends `cancelled`, with the reason in the logs, metrics and trace only | +| 401, 403, 404, 429, 5xx, timeout or connection error | failed, retried by pg-boss | it may pass later, using the queue's retry and backoff settings. a missing or rotated token, or a wrong `app.map.url` (shigola itself never answers 404), is fixed by a deploy, so it is retried rather than losing every metatile meanwhile | + +Shigola answers 429 once `max_concurrent_requests` (4 by default) are in flight, so keep `app.parallelism` at or below it across retiler replicas. + +Shigola mode has its own metrics and traces: +- `retiler_shigola_requests_count{operation, outcome, code}`: outcome is `completed`, `rejected` (not retried) or `failed` (retried), code the http status or `none` +- `retiler_shigola_tiles_count{operation, result}`: the tiles shigola reported as `done` (purged or seeded), `skipped` or `failed` +- `retiler_shigola_request_duration_seconds{operation, z}` +- a `shigola.cache` span per job, with `shigola.operation`, `shigola.outcome`, `http.response.status_code` and the tile counts + ## Run Locally Clone the project diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index c76844f..1c59f72 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -64,6 +64,14 @@ data: MAP_WMS_VERSION: {{ .map.wms.version }} MAP_WMS_LAYERS: {{ .map.wms.layers | quote }} MAP_WMS_STYLES: {{ .map.wms.styles | quote }} + {{- if eq .map.provider "shigola" }} + MAP_SHIGOLA_OPERATION: {{ .map.shigola.operation }} + MAP_SHIGOLA_TARGETS: {{ .map.shigola.targets | quote }} + MAP_SHIGOLA_TIMEOUT_MS: {{ .map.shigola.timeoutMs | quote }} + {{- if .map.shigola.tileMatrixSets }} + MAP_SHIGOLA_TILE_MATRIX_SETS: {{ .map.shigola.tileMatrixSets | quote }} + {{- end }} + {{- end }} TILES_STORAGE_PROVIDERS: {{ .tilesStorage.providers | quote }} TILES_STORAGE_LAYOUT_FORMAT: {{ .tilesStorage.layout.format }} TILES_STORAGE_LAYOUT_SHOULD_FLIP_Y: {{ .tilesStorage.layout.shouldFlipY | quote }} diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index 5e8f282..4d3d123 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -73,6 +73,15 @@ spec: - name: NODE_EXTRA_CA_CERTS value: {{ printf "[%s/%s]" .Values.caPath .Values.caKey | quote }} {{- end }} + {{- with .Values.env.map }} + {{- if and (eq .provider "shigola") (.shigola.existingSecret | default dict).name }} + - name: MAP_SHIGOLA_TOKEN + valueFrom: + secretKeyRef: + name: {{ .shigola.existingSecret.name | quote }} + key: {{ .shigola.existingSecret.key | quote }} + {{- end }} + {{- end }} {{- if .Values.extraEnvVars }} {{- toYaml .Values.extraEnvVars | nindent 12 }} {{- end }} diff --git a/helm/templates/secret.yaml b/helm/templates/secret.yaml index 0d66f69..994234b 100644 --- a/helm/templates/secret.yaml +++ b/helm/templates/secret.yaml @@ -17,4 +17,7 @@ type: Opaque data: DB_USERNAME: {{ .Values.dbConfig.username | b64enc }} DB_PASSWORD: {{ .Values.dbConfig.password | b64enc }} + {{- if and (eq .Values.env.map.provider "shigola") .Values.env.map.shigola.token (not (.Values.env.map.shigola.existingSecret | default dict).name) }} + MAP_SHIGOLA_TOKEN: {{ .Values.env.map.shigola.token | b64enc }} + {{- end }} {{- end }} diff --git a/helm/values.yaml b/helm/values.yaml index 8222beb..a147835 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -93,6 +93,20 @@ env: version: 1.3.0 layers: MAP_WMS_LAYERS styles: MAP_WMS_STYLES + # used when provider is shigola, url is then shigola's cache admin api + shigola: + operation: purge # purge or seed, a seed always overwrites + targets: '[{"map": "osm"}]' # json array of {map, layers?}, layers for purge only + # json array, empty for every set each map serves. the zoom sent is the metatile's WorldCRS84Quad + # zoom, so other sets would be refreshed at that zoom number over a different extent + tileMatrixSets: '["WorldCRS84Quad"]' + # the admin api bearer token, empty if shigola has none. either a value stored in the chart's secret, + # or a key of an existing secret, which takes precedence + token: '' + existingSecret: + name: '' + key: token + timeoutMs: 70000 # keep above shigola's request_timeout_ms client: timeoutMs: 60000 tilesStorage: diff --git a/src/app.ts b/src/app.ts index 6a8fad2..f7c9bc9 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,16 +1,13 @@ import type { Logger } from '@map-colonies/js-logger'; import type { FactoryFunction } from 'tsyringe'; -import { JOB_QUEUE_PROVIDER, SERVICES, TILES_STORAGE_PROVIDERS } from './common/constants'; +import { JOB_QUEUE_PROVIDER, METATILE_PROCESSOR, SERVICES } from './common/constants'; import { timerify } from './common/util'; -import type { JobQueueProvider, TilesStorageProvider } from './retiler/interfaces'; -import { TileProcessor } from './retiler/tileProcessor'; +import type { JobQueueProvider, MetatileProcessor } from './retiler/interfaces'; import type { TileWithMetadata } from './retiler/types'; import type { ConfigType } from './common/config'; export const consumeAndProcessFactory: FactoryFunction<() => Promise> = (container) => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars -- used for tiles storage providers factory initialization before the tiles processor - const tilesStorageProviders = container.resolve(TILES_STORAGE_PROVIDERS); - const processor = container.resolve(TileProcessor); + const processor = container.resolve(METATILE_PROCESSOR); const queueProv = container.resolve(JOB_QUEUE_PROVIDER); const logger = container.resolve(SERVICES.LOGGER); const config = container.resolve(SERVICES.CONFIG); diff --git a/src/common/constants.ts b/src/common/constants.ts index d4382d1..d4316d7 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -24,6 +24,8 @@ export const METRICS_REGISTRY = Symbol('MetricsRegistry'); export const MAP_URL = Symbol('MapURL'); export const MAP_FORMAT = Symbol('MapFormat'); export const MAP_PROVIDER_CONFIG = Symbol('MapProviderConfig'); +export const SHIGOLA_CONFIG = Symbol('ShigolaConfig'); +export const METATILE_PROCESSOR = Symbol('MetatileProcessor'); export const S3_BUCKET = Symbol('S3Bucket'); export const TILES_STORAGE_LAYOUT = Symbol('TilesStorageLayout'); diff --git a/src/common/errors.ts b/src/common/errors.ts new file mode 100644 index 0000000..ffda64f --- /dev/null +++ b/src/common/errors.ts @@ -0,0 +1,10 @@ +/** + * An error that repeating the job cannot fix, such as a request the server refused as invalid. + * The job queue ends such a job instead of retrying it. + */ +export class NonRetryableError extends Error { + public constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = 'NonRetryableError'; + } +} diff --git a/src/common/metrics.ts b/src/common/metrics.ts index d4cd682..3890de6 100644 --- a/src/common/metrics.ts +++ b/src/common/metrics.ts @@ -34,6 +34,24 @@ export const enum SubTileStatus { OUT_OF_BOUNDS = 'out_of_bounds', } +/** How a Shigola Cache Admin API request ended, and so what becomes of its job. */ +export const enum ShigolaOutcome { + /** 2xx, the job completes */ + COMPLETED = 'completed', + /** a 4xx that repeating cannot fix, the job ends without a retry */ + REJECTED = 'rejected', + /** 401, 403, 404, 429, 5xx, timeout or connection error, the job is retried */ + FAILED = 'failed', +} + +/** The tiles a Shigola cache operation reported. */ +export const enum ShigolaTileResult { + /** purged or seeded */ + DONE = 'done', + SKIPPED = 'skipped', + FAILED = 'failed', +} + export const endMetricTimer = (timer?: ReturnType): void => { if (timer) { timer(); diff --git a/src/common/tracing/job.ts b/src/common/tracing/job.ts index 52daafd..74b1021 100644 --- a/src/common/tracing/job.ts +++ b/src/common/tracing/job.ts @@ -13,6 +13,12 @@ export const jobAttributes = { TILES_BLANK_COUNT: 'tiles.blank_count', TILES_OUT_OF_BOUNDS_COUNT: 'tiles.out_of_bounds_count', MAP_PROVIDER: 'map.provider', + HTTP_STATUS: 'http.response.status_code', + SHIGOLA_OPERATION: 'shigola.operation', + SHIGOLA_OUTCOME: 'shigola.outcome', + SHIGOLA_TILES_DONE: 'shigola.tiles.done', + SHIGOLA_TILES_SKIPPED: 'shigola.tiles.skipped', + SHIGOLA_TILES_FAILED: 'shigola.tiles.failed', } satisfies Record; export const spanName = { @@ -24,4 +30,5 @@ export const spanName = { TILE_DELETE: 'tile.delete', TILE_POSTPROCESS: 'tile.post_process', JOB_PROCESS: 'job.process', + SHIGOLA_CACHE: 'shigola.cache', } satisfies Record; diff --git a/src/containerConfig.ts b/src/containerConfig.ts index e0c4267..1033baf 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -28,6 +28,8 @@ import { METRICS_REGISTRY, ON_SIGNAL, HEALTHCHECK, + METATILE_PROCESSOR, + SHIGOLA_CONFIG, } from './common/constants'; import type { InjectionObject } from './common/dependencyRegistration'; import { registerDependencies } from './common/dependencyRegistration'; @@ -39,6 +41,8 @@ import { ArcgisMapProvider } from './retiler/mapProvider/arcgis/arcgisMapProvide import { SharpMapSplitter } from './retiler/mapSplitterProvider/sharp'; import { consumeAndProcessFactory } from './app'; import { WmsMapProvider } from './retiler/mapProvider/wms/wmsMapProvider'; +import { ShigolaCacheProcessor } from './retiler/shigola/shigolaCacheProcessor'; +import { TileProcessor } from './retiler/tileProcessor'; import { tilesStorageProvidersFactory } from './retiler/tilesStorageProvider/factory'; import type { ConfigType } from './common/config'; import { getConfig } from './common/config'; @@ -206,11 +210,22 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise const config = container.resolve(SERVICES.CONFIG); const mapProviderType = config.get('app.map.provider'); + if (mapProviderType === 'shigola') { + // shigola caches the tiles itself, so no map provider, splitter, tiles storage or detiler is used + if (config.get('detiler.enabled')) { + container.resolve(SERVICES.LOGGER).warn({ msg: 'detiler is enabled but not used with the shigola map provider' }); + } + container.register(SHIGOLA_CONFIG, { useValue: config.get('app.map.shigola') }); + container.register(METATILE_PROCESSOR, { useClass: ShigolaCacheProcessor }, { lifecycle: Lifecycle.Singleton }); + return Promise.resolve(); + } + if (mapProviderType === 'wms') { container.register(MAP_PROVIDER, { useClass: WmsMapProvider }); } else { container.register(MAP_PROVIDER, { useClass: ArcgisMapProvider }); } + container.register(METATILE_PROCESSOR, { useClass: TileProcessor }, { lifecycle: Lifecycle.Singleton }); return Promise.resolve(); }, }, diff --git a/src/retiler/interfaces.ts b/src/retiler/interfaces.ts index 4075445..5649fdc 100644 --- a/src/retiler/interfaces.ts +++ b/src/retiler/interfaces.ts @@ -7,6 +7,10 @@ export interface JobQueueProvider { stopQueue: () => Promise; } +export interface MetatileProcessor { + processTile: (tile: TileWithMetadata) => Promise; +} + export interface MapProvider { getMap: (tile: TileWithMetadata) => Promise; } diff --git a/src/retiler/jobQueueProvider/pgBossJobQueue.ts b/src/retiler/jobQueueProvider/pgBossJobQueue.ts index 5158f84..b1872ae 100644 --- a/src/retiler/jobQueueProvider/pgBossJobQueue.ts +++ b/src/retiler/jobQueueProvider/pgBossJobQueue.ts @@ -8,6 +8,7 @@ import { serializeError } from 'serialize-error'; import { type Tracer, SpanStatusCode } from '@opentelemetry/api'; import { jobAttributes, spanName } from '@src/common/tracing/job'; import { METRICS_REGISTRY, QUEUE_EMPTY_TIMEOUT, QUEUE_NAME, SERVICES } from '../../common/constants'; +import { NonRetryableError } from '../../common/errors'; import { JobQueueProvider } from '../interfaces'; @injectable() @@ -123,8 +124,14 @@ export class PgBossJobQueueProvider implements JobQueueProvider { span.setStatus({ code: SpanStatusCode.OK }); } catch (err) { const error = err as Error; - this.logger.error({ err: error, jobId: job.id, job }); - await this.pgBoss.fail(this.queueName, job.id, serializeError(error)); + if (error instanceof NonRetryableError) { + // pg-boss fail() retries while retries remain, cancel() is its only terminal state without a retry + this.logger.error({ msg: 'job cannot succeed on retry, cancelling it', err: error, jobId: job.id, job }); + await this.pgBoss.cancel(this.queueName, job.id); + } else { + this.logger.error({ err: error, jobId: job.id, job }); + await this.pgBoss.fail(this.queueName, job.id, serializeError(error)); + } span.setStatus({ code: SpanStatusCode.ERROR }); span.recordException(error); } finally { diff --git a/src/retiler/shigola/shigolaCacheProcessor.ts b/src/retiler/shigola/shigolaCacheProcessor.ts new file mode 100644 index 0000000..d334d0f --- /dev/null +++ b/src/retiler/shigola/shigolaCacheProcessor.ts @@ -0,0 +1,193 @@ +import { type AxiosError, type AxiosInstance, type AxiosRequestConfig, type AxiosResponse } from 'axios'; +import { StatusCodes } from 'http-status-codes'; +import { Counter, Histogram, Registry } from 'prom-client'; +import { TILEGRID_WORLD_CRS84, tileToBoundingBox, type Tile } from '@map-colonies/tile-calc'; +import { type Logger } from '@map-colonies/js-logger'; +import { type Span, SpanStatusCode, type Tracer } from '@opentelemetry/api'; +import { inject, injectable } from 'tsyringe'; +import { jobAttributes, spanName } from '@src/common/tracing/job'; +import { MAP_URL, METRICS_BUCKETS, METRICS_REGISTRY, SERVICES, SHIGOLA_CONFIG } from '../../common/constants'; +import { NonRetryableError } from '../../common/errors'; +import { endMetricTimer, ShigolaOutcome, ShigolaTileResult } from '../../common/metrics'; +import { timerify } from '../../common/util'; +import { MetatileProcessor } from '../interfaces'; +import { TileWithMetadata } from '../types'; +import type { ShigolaCacheRequest, ShigolaCacheResponse, ShigolaConfig, ShigolaErrorResponse, ShigolaOperation } from './types'; + +const NO_HTTP_STATUS = 'none'; +const CLIENT_ERROR_MIN: number = StatusCodes.BAD_REQUEST; +const SERVER_ERROR_MIN: number = StatusCodes.INTERNAL_SERVER_ERROR; +// a missing or rotated token, or a wrong admin url (shigola reports a bad request as 400 or 422, never 404), +// is fixed by a deploy rather than by changing the job, so it is retried instead of losing every metatile meanwhile +const RETRYABLE_CLIENT_ERRORS: number[] = [StatusCodes.UNAUTHORIZED, StatusCodes.FORBIDDEN, StatusCodes.NOT_FOUND, StatusCodes.TOO_MANY_REQUESTS]; + +/** + * Keeps Shigola's tile cache fresh in place of the TileProcessor: each metatile becomes one request to + * Shigola's Cache Admin API with the metatile's bbox and zoom, and Shigola purges or renders the tiles + * in its own cache. Nothing is fetched, split or stored by retiler, and detiler is not used. + */ +@injectable() +export class ShigolaCacheProcessor implements MetatileProcessor { + private readonly requestUrl: string; + private readonly requestConfig: AxiosRequestConfig; + private readonly requestsCounter?: Counter<'operation' | 'outcome' | 'code'>; + private readonly tilesCounter?: Counter<'operation' | 'result'>; + private readonly requestDurationHistogram?: Histogram<'operation' | 'z'>; + + public constructor( + @inject(SERVICES.HTTP_CLIENT) private readonly axiosClient: AxiosInstance, + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(SERVICES.TRACER) private readonly tracer: Tracer, + @inject(MAP_URL) mapUrl: string, + @inject(SHIGOLA_CONFIG) private readonly shigolaConfig: ShigolaConfig, + @inject(METRICS_REGISTRY) registry?: Registry, + @inject(METRICS_BUCKETS) metricsBuckets?: number[] + ) { + const { operation, token, timeoutMs, targets, tileMatrixSets } = shigolaConfig; + this.requestUrl = new URL(`cache/${operation}`, mapUrl.endsWith('/') ? mapUrl : `${mapUrl}/`).toString(); + + const hasToken = token !== undefined && token !== ''; + this.requestConfig = { timeout: timeoutMs, ...(hasToken && { headers: { authorization: `Bearer ${token}` } }) }; + + if (registry !== undefined) { + this.requestsCounter = new Counter({ + name: 'retiler_shigola_requests_count', + help: 'The total number of Shigola Cache Admin API requests by operation, outcome and http status', + labelNames: ['operation', 'outcome', 'code'] as const, + registers: [registry], + }); + + this.tilesCounter = new Counter({ + name: 'retiler_shigola_tiles_count', + help: 'The total number of tiles Shigola reported as done (purged or seeded), skipped or failed', + labelNames: ['operation', 'result'] as const, + registers: [registry], + }); + + this.requestDurationHistogram = new Histogram({ + name: 'retiler_shigola_request_duration_seconds', + help: 'Shigola Cache Admin API request duration by operation', + buckets: metricsBuckets, + labelNames: ['operation', 'z'] as const, + registers: [registry], + }); + } + + this.logger.info({ + msg: 'initializing shigola cache processor', + mapUrl, + requestUrl: this.requestUrl, + operation, + targets, + tileMatrixSets, + timeoutMs, + hasToken, + }); + } + + public async processTile(tile: TileWithMetadata): Promise { + const { operation } = this.shigolaConfig; + const { parent, ...baseTile } = tile; + + const attributes = { + [jobAttributes.TILE_Z]: tile.z, + [jobAttributes.TILE_X]: tile.x, + [jobAttributes.TILE_Y]: tile.y, + [jobAttributes.TILE_METATILE]: tile.metatile, + [jobAttributes.MAP_PROVIDER]: 'shigola', + [jobAttributes.SHIGOLA_OPERATION]: operation, + }; + + return this.tracer.startActiveSpan(spanName.SHIGOLA_CACHE, { attributes }, async (span) => { + const timerEnd = this.requestDurationHistogram?.startTimer({ operation, z: tile.z }); + try { + const body = this.buildRequest(baseTile); + + this.logger.debug({ msg: 'requesting shigola cache operation', tile: baseTile, parent, requestUrl: this.requestUrl, body }); + + const [response, duration] = await timerify, [string, ShigolaCacheRequest, AxiosRequestConfig]>( + this.axiosClient.post.bind(this.axiosClient), + this.requestUrl, + body, + this.requestConfig + ); + + const { seeded, purged, skipped, failed } = response.data; + const done = seeded ?? purged ?? 0; + + span.setAttributes({ + [jobAttributes.SHIGOLA_TILES_DONE]: done, + [jobAttributes.SHIGOLA_TILES_SKIPPED]: skipped, + [jobAttributes.SHIGOLA_TILES_FAILED]: failed, + }); + this.tilesCounter?.inc({ operation, result: ShigolaTileResult.DONE }, done); + this.tilesCounter?.inc({ operation, result: ShigolaTileResult.SKIPPED }, skipped); + this.tilesCounter?.inc({ operation, result: ShigolaTileResult.FAILED }, failed); + + this.recordOutcome(span, operation, ShigolaOutcome.COMPLETED, response.status); + this.logger.info({ msg: 'shigola cache operation completed', tile: baseTile, parent, operation, duration, done, skipped, failed }); + } catch (err) { + const httpStatus = (err as Partial | undefined)?.response?.status; + const error = this.classifyError(err, operation, httpStatus); + const outcome = error instanceof NonRetryableError ? ShigolaOutcome.REJECTED : ShigolaOutcome.FAILED; + + this.recordOutcome(span, operation, outcome, httpStatus); + span.recordException(error); + this.logger.error({ + msg: 'shigola cache operation failed', + err: error, + outcome, + httpStatus, + tile: baseTile, + parent, + requestUrl: this.requestUrl, + }); + + throw error; + } finally { + endMetricTimer(timerEnd); + span.end(); + } + }); + } + + private buildRequest(tile: Required): ShigolaCacheRequest { + const { operation, targets, tileMatrixSets } = this.shigolaConfig; + + // clamped, as a metatile may reach past the grid and shigola refuses bounds outside lng/lat + const { west, south, east, north } = tileToBoundingBox(tile, TILEGRID_WORLD_CRS84, true); + + return { + bounds: [west, south, east, north], + zoomRange: { min: tile.z, max: tile.z }, + targets, + ...(tileMatrixSets !== undefined && { tileMatrixSets }), + ...(operation === 'seed' && { overwrite: true }), + }; + } + + /** + * A 4xx that repeating the request cannot fix (a bad bbox, an unknown map, too many tiles) becomes a + * NonRetryableError. Anything else, including 429, an auth failure and a 404, is left for the queue to retry. + */ + private classifyError(err: unknown, operation: ShigolaOperation, httpStatus: number | undefined): Error { + const error = err instanceof Error ? err : new Error(String(err)); + + if (httpStatus === undefined || httpStatus < CLIENT_ERROR_MIN || httpStatus >= SERVER_ERROR_MIN || RETRYABLE_CLIENT_ERRORS.includes(httpStatus)) { + return error; + } + + // the body is shigola's {error}, unless a proxy in front of it answered + const reason = (err as AxiosError).response?.data?.error ?? error.message; + return new NonRetryableError(`shigola refused the ${operation} request with status ${httpStatus}: ${reason}`, { cause: err }); + } + + private recordOutcome(span: Span, operation: ShigolaOperation, outcome: ShigolaOutcome, httpStatus: number | undefined): void { + span.setAttribute(jobAttributes.SHIGOLA_OUTCOME, outcome); + if (httpStatus !== undefined) { + span.setAttribute(jobAttributes.HTTP_STATUS, httpStatus); + } + span.setStatus({ code: outcome === ShigolaOutcome.COMPLETED ? SpanStatusCode.OK : SpanStatusCode.ERROR }); + this.requestsCounter?.inc({ operation, outcome, code: httpStatus ?? NO_HTTP_STATUS }); + } +} diff --git a/src/retiler/shigola/types.ts b/src/retiler/shigola/types.ts new file mode 100644 index 0000000..d979ce4 --- /dev/null +++ b/src/retiler/shigola/types.ts @@ -0,0 +1,26 @@ +import type { VectorRetilerSchemaType } from '../../common/config'; + +export type ShigolaConfig = Extract['shigola']; + +export type ShigolaOperation = ShigolaConfig['operation']; + +export type ShigolaTarget = ShigolaConfig['targets'][number]; + +export interface ShigolaCacheRequest { + bounds: [number, number, number, number]; + zoomRange: { min: number; max: number }; + targets: ShigolaTarget[]; + tileMatrixSets?: string[]; + overwrite?: boolean; +} + +export interface ShigolaCacheResponse { + seeded?: number; + purged?: number; + skipped: number; + failed: number; +} + +export interface ShigolaErrorResponse { + error?: string; +} diff --git a/src/retiler/tileProcessor.ts b/src/retiler/tileProcessor.ts index 134deb5..601c2a5 100644 --- a/src/retiler/tileProcessor.ts +++ b/src/retiler/tileProcessor.ts @@ -19,7 +19,7 @@ import { TILES_STORAGE_PROVIDERS, } from '../common/constants'; import { endMetricTimer, MetatileStatus, ProcessKind, ProcessReason, ProcessSkipReason, SubTileStatus } from '../common/metrics'; -import { type MapProvider, type MapSplitterProvider, type TilesStorageProvider } from './interfaces'; +import { type MapProvider, type MapSplitterProvider, type MetatileProcessor, type TilesStorageProvider } from './interfaces'; import { TileWithMetadata } from './types'; interface PreProcessReult { @@ -28,7 +28,7 @@ interface PreProcessReult { } @injectable() -export class TileProcessor { +export class TileProcessor implements MetatileProcessor { private readonly project: IProjectConfig; private readonly forceProcess: boolean; private readonly shouldFilterBlankTiles: boolean; diff --git a/tests/unit/containerConfig.spec.ts b/tests/unit/containerConfig.spec.ts new file mode 100644 index 0000000..0c5a911 --- /dev/null +++ b/tests/unit/containerConfig.spec.ts @@ -0,0 +1,61 @@ +/* eslint-disable @typescript-eslint/naming-convention */ // environment variable names +import { Registry } from 'prom-client'; +import { trace } from '@opentelemetry/api'; +import { jsLogger, type Logger } from '@map-colonies/js-logger'; +import type { DependencyContainer } from 'tsyringe'; +import { getConfig, initConfig } from '../../src/common/config'; +import { registerExternalValues } from '../../src/containerConfig'; +import { + JOB_QUEUE_PROVIDER, + MAP_PROVIDER, + METATILE_PROCESSOR, + METRICS_REGISTRY, + SERVICES, + TILES_STORAGE_PROVIDERS, +} from '../../src/common/constants'; +import { ShigolaCacheProcessor } from '../../src/retiler/shigola/shigolaCacheProcessor'; +import { TileProcessor } from '../../src/retiler/tileProcessor'; + +describe('containerConfig', () => { + let logger: Logger; + const savedEnv = { ...process.env }; + + beforeAll(async () => { + logger = await jsLogger({ enabled: false }); + }); + + afterEach(() => { + process.env = { ...savedEnv }; + }); + + async function registerWithProvider(env: Record): Promise { + Object.assign(process.env, env); + await initConfig(true); + return registerExternalValues({ + override: [ + { token: SERVICES.CONFIG, provider: { useValue: getConfig() } }, + { token: SERVICES.LOGGER, provider: { useValue: logger } }, + { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('test') } }, + { token: METRICS_REGISTRY, provider: { useValue: new Registry() } }, + { token: JOB_QUEUE_PROVIDER, provider: { useValue: { startQueue: jest.fn(), stopQueue: jest.fn() } } }, + ], + useChild: true, + }); + } + + it('should process metatiles through shigola without a map provider or tiles storage', async () => { + // with no storage provider the storage factory throws, so this also proves storage is never resolved + const container = await registerWithProvider({ MAP_PROVIDER: 'shigola', MAP_SHIGOLA_TARGETS: '[{"map":"osm"}]', TILES_STORAGE_PROVIDERS: '[]' }); + + expect(container.resolve(METATILE_PROCESSOR)).toBeInstanceOf(ShigolaCacheProcessor); + expect(container.isRegistered(MAP_PROVIDER)).toBe(false); + expect(() => container.resolve(TILES_STORAGE_PROVIDERS)).toThrow('at least one provider is required'); + }); + + it('should process metatiles through the tile processor for wms', async () => { + const container = await registerWithProvider({ MAP_PROVIDER: 'wms' }); + + expect(container.resolve(METATILE_PROCESSOR)).toBeInstanceOf(TileProcessor); + expect(container.isRegistered(MAP_PROVIDER)).toBe(true); + }); +}); diff --git a/tests/unit/jobQueueProvider/pgBossJobQueue.spec.ts b/tests/unit/jobQueueProvider/pgBossJobQueue.spec.ts index 8cc1e6c..33bf22a 100644 --- a/tests/unit/jobQueueProvider/pgBossJobQueue.spec.ts +++ b/tests/unit/jobQueueProvider/pgBossJobQueue.spec.ts @@ -3,6 +3,7 @@ import { Registry } from 'prom-client'; import { jsLogger, type Logger } from '@map-colonies/js-logger'; import type { PgBoss } from 'pg-boss'; import { serializeError } from 'serialize-error'; +import { NonRetryableError } from '../../../src/common/errors'; import { PgBossJobQueueProvider } from '../../../src/retiler/jobQueueProvider/pgBossJobQueue'; import { LONG_RUNNING_TEST } from '../../integration/helpers'; @@ -17,6 +18,7 @@ describe('PgBossJobQueueProvider', () => { getQueueSize: jest.Mock; complete: jest.Mock; fail: jest.Mock; + cancel: jest.Mock; fetch: jest.Mock; }; @@ -32,6 +34,7 @@ describe('PgBossJobQueueProvider', () => { getQueueSize: jest.fn(), complete: jest.fn(), fail: jest.fn(), + cancel: jest.fn(), fetch: jest.fn(), }; }); @@ -140,5 +143,22 @@ describe('PgBossJobQueueProvider', () => { expect(pgbossMock.complete).not.toHaveBeenCalled(); expect(pgbossMock.fail).toHaveBeenCalledWith('queue-name', id, serializeError(fetchError)); }); + + it('should cancel instead of failing a job whose error cannot be fixed by a retry', async () => { + const id = 'someId'; + pgbossMock.fetch.mockResolvedValueOnce([{ id }]); + + const fnMock = jest.fn().mockRejectedValue(new NonRetryableError('refused')); + + await provider.startQueue(); + const queuePromise = provider.consumeQueue(fnMock); + await setTimeoutPromise(50); + await provider.stopQueue(); + await expect(queuePromise).resolves.not.toThrow(); + + expect(pgbossMock.complete).not.toHaveBeenCalled(); + expect(pgbossMock.fail).not.toHaveBeenCalled(); + expect(pgbossMock.cancel).toHaveBeenCalledWith('queue-name', id); + }); }); }); diff --git a/tests/unit/shigola/shigolaCacheProcessor.spec.ts b/tests/unit/shigola/shigolaCacheProcessor.spec.ts new file mode 100644 index 0000000..16cd3d7 --- /dev/null +++ b/tests/unit/shigola/shigolaCacheProcessor.spec.ts @@ -0,0 +1,185 @@ +import type { AxiosInstance } from 'axios'; +import { Registry } from 'prom-client'; +import { SpanStatusCode } from '@opentelemetry/api'; +import { jsLogger, type Logger } from '@map-colonies/js-logger'; +import { NonRetryableError } from '../../../src/common/errors'; +import { ShigolaCacheProcessor } from '../../../src/retiler/shigola/shigolaCacheProcessor'; +import type { ShigolaConfig } from '../../../src/retiler/shigola/types'; + +jest.mock('axios'); + +const MAP_URL = 'http://shigola-admin:9090'; + +let logger: Logger; + +beforeAll(async () => { + logger = await jsLogger({ enabled: false }); +}); + +function httpError(status: number | undefined, data?: unknown, code?: string): Error { + return Object.assign(new Error(status === undefined ? `failed with ${code}` : `Request failed with status code ${status}`), { + isAxiosError: true, + code, + ...(status !== undefined && { response: { status, data } }), + }); +} + +describe('shigolaCacheProcessor', () => { + describe('#processTile', () => { + const post = jest.fn(); + const mockedClient = { post } as unknown as jest.Mocked; + const span = { setAttribute: jest.fn(), setAttributes: jest.fn(), setStatus: jest.fn(), recordException: jest.fn(), end: jest.fn() }; + const tracerMock = { + startActiveSpan: jest.fn().mockImplementation((_name: string, _options: unknown, fn: (span: unknown) => unknown) => fn(span)), + startSpan: jest.fn(), + }; + let registry: Registry; + + const purgeConfig: ShigolaConfig = { operation: 'purge', targets: [{ map: 'osm' }], timeoutMs: 70000 }; + + function createProcessor(config: ShigolaConfig = purgeConfig, url = MAP_URL): ShigolaCacheProcessor { + return new ShigolaCacheProcessor(mockedClient, logger, tracerMock, url, config, registry, [1]); + } + + async function metricValue(name: string, labels: Record): Promise { + const metric = await registry.getSingleMetric(name)?.get(); + return metric?.values.find((v) => Object.entries(labels).every(([key, value]) => String(v.labels[key]) === value))?.value; + } + + beforeEach(function () { + registry = new Registry(); + }); + + afterEach(function () { + jest.clearAllMocks(); + }); + + it('should purge the metatile bbox at its zoom once, without overwrite or a token', async function () { + post.mockResolvedValue({ status: 200, data: { purged: 4, skipped: 0, failed: 0, durationMs: 3 } }); + + await createProcessor().processTile({ z: 2, x: 1, y: 0, metatile: 2, parent: 'parent' }); + + expect(post).toHaveBeenCalledTimes(1); + expect(post).toHaveBeenCalledWith( + 'http://shigola-admin:9090/cache/purge', + { bounds: [-90, 0, 0, 90], zoomRange: { min: 2, max: 2 }, targets: [{ map: 'osm' }] }, + { timeout: 70000 } + ); + await expect(metricValue('retiler_shigola_requests_count', { operation: 'purge', outcome: 'completed', code: '200' })).resolves.toBe(1); + await expect(metricValue('retiler_shigola_tiles_count', { operation: 'purge', result: 'done' })).resolves.toBe(4); + expect(span.setAttribute).toHaveBeenCalledWith('shigola.outcome', 'completed'); + expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.OK }); + expect(span.end).toHaveBeenCalled(); + }); + + it('should seed with overwrite, several targets, tile matrix sets and the bearer token', async function () { + post.mockResolvedValue({ status: 200, data: { seeded: 1, skipped: 0, failed: 0, durationMs: 1 } }); + const config: ShigolaConfig = { + operation: 'seed', + targets: [{ map: 'osm' }, { map: 'light' }], + tileMatrixSets: ['WorldCRS84Quad'], + token: 'secret', + timeoutMs: 1000, + }; + + await createProcessor(config, 'http://shigola-admin:9090/admin/').processTile({ z: 1, x: 3, y: 1, metatile: 1 }); + + expect(post).toHaveBeenCalledWith( + 'http://shigola-admin:9090/admin/cache/seed', + { + bounds: [90, -90, 180, 0], + zoomRange: { min: 1, max: 1 }, + targets: [{ map: 'osm' }, { map: 'light' }], + tileMatrixSets: ['WorldCRS84Quad'], + overwrite: true, + }, + { timeout: 1000, headers: { authorization: 'Bearer secret' } } + ); + }); + + it('should send purge targets with their layers', async function () { + post.mockResolvedValue({ status: 200, data: { purged: 1, skipped: 0, failed: 0, durationMs: 1 } }); + + await createProcessor({ ...purgeConfig, targets: [{ map: 'osm', layers: ['roads'] }] }).processTile({ z: 0, x: 0, y: 0, metatile: 1 }); + + expect(post).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ targets: [{ map: 'osm', layers: ['roads'] }] }), + expect.anything() + ); + }); + + it('should omit an empty token', async function () { + post.mockResolvedValue({ status: 200, data: { purged: 1, skipped: 0, failed: 0, durationMs: 1 } }); + + await createProcessor({ ...purgeConfig, token: '' }).processTile({ z: 0, x: 0, y: 0, metatile: 1 }); + + expect(post).toHaveBeenCalledWith(expect.any(String), expect.anything(), { timeout: 70000 }); + }); + + it('should clamp a metatile reaching past the grid to valid lng/lat bounds', async function () { + post.mockResolvedValue({ status: 200, data: { purged: 2, skipped: 0, failed: 0, durationMs: 1 } }); + + await createProcessor().processTile({ z: 0, x: 0, y: 0, metatile: 8 }); + + expect(post).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ bounds: [-180, -90, 180, 90], zoomRange: { min: 0, max: 0 } }), + expect.anything() + ); + }); + + it.each([ + ['400 bad bbox', 400, { error: 'bounds [...] are not a west, south, east, north lng/lat box' }], + ['400 from a proxy', 400, 'Bad Request'], + ['422 too many tiles', 422, { error: 'request covers more than max_tiles_per_request' }], + ])('should reject without retry on %s', async function (_name, status, body) { + post.mockRejectedValue(httpError(status, body)); + + const promise = createProcessor().processTile({ z: 0, x: 0, y: 0, metatile: 1 }); + + await expect(promise).rejects.toBeInstanceOf(NonRetryableError); + await expect(promise).rejects.toThrow(`status ${status}`); + await expect(metricValue('retiler_shigola_requests_count', { outcome: 'rejected', code: String(status) })).resolves.toBe(1); + expect(span.setAttribute).toHaveBeenCalledWith('shigola.outcome', 'rejected'); + expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR }); + }); + + it.each([ + ['401 missing token', httpError(401, { error: 'unauthorized' }), '401'], + ['403 forbidden', httpError(403, undefined), '403'], + ['404 wrong admin url', httpError(404, '404 page not found'), '404'], + ['429 busy', httpError(429, { error: 'busy' }), '429'], + ['500 failed tile', httpError(500, { purged: 1, skipped: 0, failed: 1, durationMs: 1 }), '500'], + ['503 timed out', httpError(503, { purged: 1, skipped: 0, failed: 0, durationMs: 60000 }), '503'], + ['client timeout', httpError(undefined, undefined, 'ECONNABORTED'), 'none'], + ['connection error', httpError(undefined, undefined, 'ECONNREFUSED'), 'none'], + ])('should fail for a retry on %s', async function (_name, error, status) { + post.mockRejectedValue(error); + + const promise = createProcessor().processTile({ z: 0, x: 0, y: 0, metatile: 1 }); + + await expect(promise).rejects.toBe(error); + await expect(promise).rejects.not.toBeInstanceOf(NonRetryableError); + await expect(metricValue('retiler_shigola_requests_count', { outcome: 'failed', code: status })).resolves.toBe(1); + }); + + it('should complete a 2xx that reports failed tiles, counting them', async function () { + post.mockResolvedValue({ status: 200, data: { purged: 3, skipped: 1, failed: 1, durationMs: 1 } }); + + await expect(createProcessor().processTile({ z: 0, x: 0, y: 0, metatile: 1 })).resolves.toBeUndefined(); + + await expect(metricValue('retiler_shigola_requests_count', { outcome: 'completed', code: '200' })).resolves.toBe(1); + await expect(metricValue('retiler_shigola_tiles_count', { result: 'failed' })).resolves.toBe(1); + await expect(metricValue('retiler_shigola_tiles_count', { result: 'skipped' })).resolves.toBe(1); + expect(span.setAttributes).toHaveBeenCalledWith(expect.objectContaining({ 'shigola.tiles.failed': 1 })); + }); + + it('should work without a metrics registry', async function () { + post.mockResolvedValue({ status: 200, data: { purged: 1, skipped: 0, failed: 0, durationMs: 1 } }); + const provider = new ShigolaCacheProcessor(mockedClient, logger, tracerMock, MAP_URL, purgeConfig); + + await expect(provider.processTile({ z: 0, x: 0, y: 0, metatile: 1 })).resolves.toBeUndefined(); + }); + }); +}); From 2957e80a69e0aad30f4279bf23b244571f352070 Mon Sep 17 00:00:00 2001 From: Niv Greenstein <88280771+NivGreenstein@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:21:02 +0300 Subject: [PATCH 2/5] fix: validate the config with the retiler v3 schema The shigola provider ships in a new retiler v3 schema, as released schemas are not modified. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/common/config.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common/config.ts b/src/common/config.ts index ca7ad34..e919b18 100644 --- a/src/common/config.ts +++ b/src/common/config.ts @@ -1,8 +1,8 @@ import { type ConfigInstance, config } from '@map-colonies/config'; -import { vectorRetilerV2, type vectorRetilerV2Type } from '@map-colonies/schemas'; +import { vectorRetilerV3, type vectorRetilerV3Type } from '@map-colonies/schemas'; // Choose here the type of the config instance and import this type from the entire application -type ConfigType = ConfigInstance; +type ConfigType = ConfigInstance; let configInstance: ConfigType | undefined; @@ -13,7 +13,7 @@ let configInstance: ConfigType | undefined; */ async function initConfig(offlineMode?: boolean): Promise { configInstance = await config({ - schema: vectorRetilerV2, + schema: vectorRetilerV3, offlineMode, }); } @@ -28,4 +28,4 @@ function getConfig(): ConfigType { export { getConfig, initConfig }; export type { ConfigType }; -export type VectorRetilerSchemaType = vectorRetilerV2Type; +export type VectorRetilerSchemaType = vectorRetilerV3Type; From b64b54119601a9e4c358568bcf0eafda28bf52c2 Mon Sep 17 00:00:00 2001 From: Niv Greenstein <88280771+NivGreenstein@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:24:16 +0300 Subject: [PATCH 3/5] ci: run the tests on bookworm, as bullseye's security archive no longer installs Debian 11 reached end of life, and apt in node:24-bullseye now fails with 404s from bullseye-security. The pg client repo follows the container's release instead of naming one. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/pull_request.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 92b60f7..fb738e6 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -48,7 +48,7 @@ jobs: tests: name: Run Tests runs-on: ubuntu-latest - container: node:24-bullseye + container: node:24-bookworm strategy: matrix: @@ -106,7 +106,7 @@ jobs: apt-get update && apt-get install -y lsb-release curl ca-certificates install -d /usr/share/postgresql-common/pgdg curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc - sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bullseye-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' apt-get update apt-get install -y postgresql-client From 590383b22df2af9e472a9b84064411a02680b3df Mon Sep 17 00:00:00 2001 From: Niv Greenstein <88280771+NivGreenstein@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:27:57 +0300 Subject: [PATCH 4/5] chore(deps): use the schemas PR build for the retiler v3 schema Temporary: points @map-colonies/schemas at the package built from MapColonies/schemas#181, so CI can run against the v3 schema before it is released. Replace it with the released version before merging. Co-Authored-By: Claude Opus 5.5 (1M context) --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 959f13c..b045689 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,7 @@ "@map-colonies/js-logger": "^5.0.0", "@map-colonies/prometheus": "^1.0.0", "@map-colonies/read-pkg": "^2.0.0", - "@map-colonies/schemas": "^1.21.0", + "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-361e92beea828b206d20812712578e4c0b936b97.tgz", "@map-colonies/tile-calc": "^0.1.5", "@map-colonies/tracing": "^1.0.0", "@map-colonies/tracing-utils": "^1.0.0", @@ -3427,9 +3427,9 @@ } }, "node_modules/@map-colonies/schemas": { - "version": "1.24.0", - "resolved": "https://registry.npmjs.org/@map-colonies/schemas/-/schemas-1.24.0.tgz", - "integrity": "sha512-W8FY4l5OK0iMmBsAQG5wmTjeseLmTrSlDxNt82m81MU5OFZPXJpKRUhAbs7v7zz1j9vSp5qiTA0pIOkML1wo0Q==", + "version": "1.27.0", + "resolved": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-361e92beea828b206d20812712578e4c0b936b97.tgz", + "integrity": "sha512-ZtOg9H6HsP0ZWZrBw1LXDbyEMvBDR84IRy4/AzaHM5mC07428IVKwpM78Vhrl0VQJBsiGkFCdihlBBTLM2R7Ag==", "license": "MIT" }, "node_modules/@map-colonies/tile-calc": { diff --git a/package.json b/package.json index d7a0a32..18b4964 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "@map-colonies/express-access-log-middleware": "^4.1.0", "@map-colonies/js-logger": "^5.0.0", "@map-colonies/read-pkg": "^2.0.0", - "@map-colonies/schemas": "^1.21.0", + "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-361e92beea828b206d20812712578e4c0b936b97.tgz", "@map-colonies/tracing": "^1.0.0", "@map-colonies/prometheus": "^1.0.0", "@map-colonies/tracing-utils": "^1.0.0", From 237177e58252ed55032c9d5a4e8795e856b6db24 Mon Sep 17 00:00:00 2001 From: Niv Greenstein <88280771+NivGreenstein@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:32:26 +0300 Subject: [PATCH 5/5] test: run shigola mode end to end in the integration tests Each test sends a pg-boss job through the shigola cache processor against a mocked Cache Admin API: a purge and a seed complete with one request for the metatile's bbox and zoom, a refused request cancels the job, and an unavailable or unreachable shigola fails it for a retry. The integration suite measures coverage over all of src, so without these the shigola processor failed its thresholds. Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/integration/retiler.spec.ts | 149 ++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) diff --git a/tests/integration/retiler.spec.ts b/tests/integration/retiler.spec.ts index a29bfac..397b32a 100644 --- a/tests/integration/retiler.spec.ts +++ b/tests/integration/retiler.spec.ts @@ -1558,4 +1558,153 @@ describe('retiler', function () { LONG_RUNNING_TEST ); }); + describe('shigola', function () { + const shigolaUrl = 'http://shigola-admin:9090'; + const target = { map: 'osm', layers: ['roads'] }; + let container: DependencyContainer; + + const registerShigola = async (shigolaConfig: Record): Promise => + registerExternalValues({ + override: [ + { + token: SERVICES.CONFIG, + provider: { + useValue: { + get: (key: string) => { + switch (key) { + case 'app.map.provider': + return 'shigola'; + case 'app.map.url': + return shigolaUrl; + case 'app.map.shigola': + return { targets: [target], timeoutMs: 5000, ...shigolaConfig }; + default: + return config.get(key); + } + }, + }, + }, + }, + { token: SERVICES.LOGGER, provider: { useValue: logger } }, + { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, + { token: METRICS_REGISTRY, provider: { useValue: new Registry() } }, + ], + useChild: true, + }); + + const runJob = async (data: Record): Promise>> => { + const pgBoss = container.resolve(SERVICES.PGBOSS); + const provider = container.resolve(JOB_QUEUE_PROVIDER); + const queueName = container.resolve(QUEUE_NAME); + const jobId = await pgBoss.send({ name: queueName, data }); + + const consumePromise = consumeAndProcessFactory(container)(); + const job = await waitForJobToBeResolved(pgBoss, queueName, jobId as string); + + await provider.stopQueue(); + await expect(consumePromise).resolves.not.toThrow(); + + return job; + }; + + afterEach(async () => { + const pgBoss = container.resolve(SERVICES.PGBOSS); + const queueName = container.resolve(QUEUE_NAME); + await cleanupQueue(pgBoss, queueName); + const cleanupRegistry = container.resolve(SERVICES.CLEANUP_REGISTRY); + await cleanupRegistry.trigger(); + container.reset(); + }); + + describe('Happy Path', function () { + it( + 'should complete a job with one purge request for the metatile bbox and zoom, using no detiler or tile storage', + async function () { + container = await registerShigola({ operation: 'purge', token: 'secret' }); + const shigolaScope = nock(shigolaUrl, { reqheaders: { authorization: 'Bearer secret' } }) + .post('/cache/purge', { bounds: [-90, 0, 0, 90], zoomRange: { min: 2, max: 2 }, targets: [target] }) + .reply(httpStatusCodes.OK, { purged: 4, skipped: 0, failed: 0, durationMs: 3 }); + + const job = await runJob({ z: 2, x: 1, y: 0, metatile: 2, parent: 'parent' }); + + expect(job).toHaveProperty('state', 'completed'); + shigolaScope.done(); + expect(s3SendMock).not.toHaveBeenCalled(); + expect(fsPromises.writeFile).not.toHaveBeenCalled(); + expect(detilerScope.isDone()).toBe(true); + }, + LONG_RUNNING_TEST + ); + + it( + 'should complete a job with a seed request that overwrites, sending no token when none is configured', + async function () { + container = await registerShigola({ operation: 'seed', targets: [{ map: 'osm' }], tileMatrixSets: ['WorldCRS84Quad'] }); + const shigolaScope = nock(shigolaUrl, { badheaders: ['authorization'] }) + .post('/cache/seed', { + bounds: [-180, -90, 180, 90], + zoomRange: { min: 0, max: 0 }, + targets: [{ map: 'osm' }], + tileMatrixSets: ['WorldCRS84Quad'], + overwrite: true, + }) + .reply(httpStatusCodes.OK, { seeded: 2, skipped: 0, failed: 0, durationMs: 3 }); + + const job = await runJob({ z: 0, x: 0, y: 0, metatile: 8, parent: 'parent' }); + + expect(job).toHaveProperty('state', 'completed'); + shigolaScope.done(); + }, + LONG_RUNNING_TEST + ); + }); + + describe('Bad Path', function () { + it( + 'should cancel the job without a retry if shigola refuses the request', + async function () { + container = await registerShigola({ operation: 'purge' }); + const shigolaScope = nock(shigolaUrl).post('/cache/purge').reply(httpStatusCodes.BAD_REQUEST, { error: 'targets[0]: no map named "osm"' }); + + const job = await runJob({ z: 2, x: 1, y: 0, metatile: 2, parent: 'parent' }); + + expect(job).toHaveProperty('state', 'cancelled'); + shigolaScope.done(); + }, + LONG_RUNNING_TEST + ); + + it( + 'should fail the job for a retry if shigola is unavailable', + async function () { + container = await registerShigola({ operation: 'purge' }); + const shigolaScope = nock(shigolaUrl) + .post('/cache/purge') + .reply(httpStatusCodes.SERVICE_UNAVAILABLE, { purged: 1, skipped: 0, failed: 0, durationMs: 60000 }); + + const job = await runJob({ z: 2, x: 1, y: 0, metatile: 2, parent: 'parent' }); + + expect(job).toHaveProperty('state', 'failed'); + expect(job).toHaveProperty('output.message', 'Request failed with status code 503'); + shigolaScope.done(); + }, + LONG_RUNNING_TEST + ); + + it( + 'should fail the job for a retry if shigola cannot be reached', + async function () { + container = await registerShigola({ operation: 'purge' }); + const shigolaScope = nock(shigolaUrl).post('/cache/purge').replyWithError({ message: 'connect ECONNREFUSED', code: 'ECONNREFUSED' }); + + const job = await runJob({ z: 2, x: 1, y: 0, metatile: 2, parent: 'parent' }); + + expect(job).toHaveProperty('state', 'failed'); + expect(job).toHaveProperty('output.message', 'connect ECONNREFUSED'); + shigolaScope.done(); + }, + LONG_RUNNING_TEST + ); + }); + }); });