feat: add a shigola mode that purges or seeds each metatile (MAPCO-11992)

.github/workflows/pull_request.yaml

@@ -48,7 +48,7 @@ jobs:
   tests:
     name: Run Tests
     runs-on: ubuntu-latest
-    container: node:24-bullseye
+    container: node:24-bookworm
 
     strategy:
       matrix:
@@ -106,7 +106,7 @@ jobs:
           apt-get update && apt-get install -y lsb-release curl ca-certificates
           install -d /usr/share/postgresql-common/pgdg
           curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc
-          sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bullseye-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+          sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
           apt-get update
           apt-get install -y postgresql-client
 

README.md

@@ -41,7 +41,7 @@ flowchart TD
 
 `app.jobQueue.noSupervisor`: flag for maintenance and monitoring operations on the job queue. defaults to true (meaning no supervision)
 
-`app.map.provider`: the map provider type, wms or arcgis
+`app.map.provider`: the map provider type, wms, arcgis or shigola. see [Shigola mode](#shigola-mode)
 
 `app.map.url`: the url of the map provider service to fetch the map from
 
@@ -88,6 +88,38 @@ will result in the key: "prefix/3/10/4.png"
 
 `app.tilesStorage.shouldFilterBlankTiles`: determine if tiles or their sub-tiles will not be storaged on configured providers if found as blanked
 
+## Shigola mode
+
+With `app.map.provider` set to `shigola`, retiler keeps [shigola](https://github.com/MapColonies/shigola)'s tile cache fresh instead of rendering tiles itself. Each metatile job becomes exactly one request to shigola's Cache Admin API, `POST {app.map.url}/cache/{purge|seed}`, with the metatile's lng/lat bbox and its zoom as both the min and max of the zoom range. Shigola purges or renders the covered tiles in its own cache, so no map is fetched, split or stored: `app.map.wms` and `app.tilesStorage` are unused, and so is detiler, even when enabled.
+
+`app.map.url`: shigola's Cache Admin API, served on its own admin port
+
+`app.map.shigola.operation`: `purge` (default) or `seed`. a seed always sends `"overwrite": true`, and needs a shigola with `POST /cache/seed`
+
+`app.map.shigola.targets`: the shigola maps to work on, as `[{"map": "osm", "layers": ["roads"]}]`. `layers` is optional and only valid for purge, since shigola seeds whole maps only
+
+`app.map.shigola.tileMatrixSets`: optional, work on only these tile matrix sets. omitted, every set each map serves. the zoom sent is the metatile's zoom in WorldCRS84Quad, so `["WorldCRS84Quad"]` (the helm chart's default) keeps each request to exactly the metatile's tiles, while other sets would be refreshed at that zoom number over a different extent
+
+`app.map.shigola.token`: optional, the Cache Admin API bearer token (shigola's `webserver.admin.token`). omitted or empty, no token is sent. the helm chart takes it either as `env.map.shigola.token`, stored in the chart's secret, or from a key of an existing secret with `env.map.shigola.existingSecret.name` and `.key`
+
+`app.map.shigola.timeoutMs`: the request timeout in ms, defaults to 70000. keep it above shigola's `request_timeout_ms` (60000 by default) so shigola's own timeout answer arrives first
+
+Each job ends by the request's outcome:
+
+| response | job | why |
+| --- | --- | --- |
+| 2xx | completed | shigola answers 500 when a tile fails, so a 2xx is done. any failed count it still reports goes to the tiles metric |
+| 4xx other than 401, 403, 404 and 429 (bad bbox, unknown map, too many tiles) | cancelled, never retried | repeating cannot fix it. pg-boss can only fail a fetched job into a retry, so cancel is how it ends without one: the job ends `cancelled`, with the reason in the logs, metrics and trace only |
+| 401, 403, 404, 429, 5xx, timeout or connection error | failed, retried by pg-boss | it may pass later, using the queue's retry and backoff settings. a missing or rotated token, or a wrong `app.map.url` (shigola itself never answers 404), is fixed by a deploy, so it is retried rather than losing every metatile meanwhile |
+
+Shigola answers 429 once `max_concurrent_requests` (4 by default) are in flight, so keep `app.parallelism` at or below it across retiler replicas.
+
+Shigola mode has its own metrics and traces:
+- `retiler_shigola_requests_count{operation, outcome, code}`: outcome is `completed`, `rejected` (not retried) or `failed` (retried), code the http status or `none`
+- `retiler_shigola_tiles_count{operation, result}`: the tiles shigola reported as `done` (purged or seeded), `skipped` or `failed`
+- `retiler_shigola_request_duration_seconds{operation, z}`
+- a `shigola.cache` span per job, with `shigola.operation`, `shigola.outcome`, `http.response.status_code` and the tile counts
+
 ## Run Locally
 
 Clone the project

helm/templates/configmap.yaml

@@ -64,6 +64,14 @@ data:
   MAP_WMS_VERSION: {{ .map.wms.version }}
   MAP_WMS_LAYERS: {{ .map.wms.layers | quote }}
   MAP_WMS_STYLES: {{ .map.wms.styles | quote }}
+  {{- if eq .map.provider "shigola" }}
+  MAP_SHIGOLA_OPERATION: {{ .map.shigola.operation }}
+  MAP_SHIGOLA_TARGETS: {{ .map.shigola.targets | quote }}
+  MAP_SHIGOLA_TIMEOUT_MS: {{ .map.shigola.timeoutMs | quote }}
+  {{- if .map.shigola.tileMatrixSets }}
+  MAP_SHIGOLA_TILE_MATRIX_SETS: {{ .map.shigola.tileMatrixSets | quote }}
+  {{- end }}
+  {{- end }}
   TILES_STORAGE_PROVIDERS: {{ .tilesStorage.providers | quote }}
   TILES_STORAGE_LAYOUT_FORMAT: {{ .tilesStorage.layout.format }}
   TILES_STORAGE_LAYOUT_SHOULD_FLIP_Y: {{ .tilesStorage.layout.shouldFlipY | quote }}

helm/templates/deployment.yaml

@@ -73,6 +73,15 @@ spec:
             - name: NODE_EXTRA_CA_CERTS
               value: {{ printf "[%s/%s]" .Values.caPath .Values.caKey | quote }}
             {{- end }}
+            {{- with .Values.env.map }}
+            {{- if and (eq .provider "shigola") (.shigola.existingSecret | default dict).name }}
+            - name: MAP_SHIGOLA_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .shigola.existingSecret.name | quote }}
+                  key: {{ .shigola.existingSecret.key | quote }}
+            {{- end }}
+            {{- end }}
             {{- if .Values.extraEnvVars }}
             {{- toYaml .Values.extraEnvVars | nindent 12 }}
             {{- end }}

helm/templates/secret.yaml

@@ -17,4 +17,7 @@ type: Opaque
 data:
   DB_USERNAME: {{ .Values.dbConfig.username | b64enc }}
   DB_PASSWORD: {{ .Values.dbConfig.password | b64enc }}
+  {{- if and (eq .Values.env.map.provider "shigola") .Values.env.map.shigola.token (not (.Values.env.map.shigola.existingSecret | default dict).name) }}
+  MAP_SHIGOLA_TOKEN: {{ .Values.env.map.shigola.token | b64enc }}
+  {{- end }}
 {{- end }}

helm/values.yaml

@@ -93,6 +93,20 @@ env:
       version: 1.3.0
       layers: MAP_WMS_LAYERS
       styles: MAP_WMS_STYLES
+    # used when provider is shigola, url is then shigola's cache admin api
+    shigola:
+      operation: purge # purge or seed, a seed always overwrites
+      targets: '[{"map": "osm"}]' # json array of {map, layers?}, layers for purge only
+      # json array, empty for every set each map serves. the zoom sent is the metatile's WorldCRS84Quad
+      # zoom, so other sets would be refreshed at that zoom number over a different extent
+      tileMatrixSets: '["WorldCRS84Quad"]'
+      # the admin api bearer token, empty if shigola has none. either a value stored in the chart's secret,
+      # or a key of an existing secret, which takes precedence
+      token: ''
+      existingSecret:
+        name: ''
+        key: token
+      timeoutMs: 70000 # keep above shigola's request_timeout_ms
     client:
       timeoutMs: 60000
   tilesStorage:

package.json

@@ -38,7 +38,7 @@
     "@map-colonies/express-access-log-middleware": "^4.1.0",
     "@map-colonies/js-logger": "^5.0.0",
     "@map-colonies/read-pkg": "^2.0.0",
-    "@map-colonies/schemas": "^1.21.0",
+    "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-361e92beea828b206d20812712578e4c0b936b97.tgz",
     "@map-colonies/tracing": "^1.0.0",
     "@map-colonies/prometheus": "^1.0.0",
     "@map-colonies/tracing-utils": "^1.0.0",

src/app.ts

@@ -1,16 +1,13 @@
 import type { Logger } from '@map-colonies/js-logger';
 import type { FactoryFunction } from 'tsyringe';
-import { JOB_QUEUE_PROVIDER, SERVICES, TILES_STORAGE_PROVIDERS } from './common/constants';
+import { JOB_QUEUE_PROVIDER, METATILE_PROCESSOR, SERVICES } from './common/constants';
 import { timerify } from './common/util';
-import type { JobQueueProvider, TilesStorageProvider } from './retiler/interfaces';
-import { TileProcessor } from './retiler/tileProcessor';
+import type { JobQueueProvider, MetatileProcessor } from './retiler/interfaces';
 import type { TileWithMetadata } from './retiler/types';
 import type { ConfigType } from './common/config';
 
 export const consumeAndProcessFactory: FactoryFunction<() => Promise<void>> = (container) => {
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars -- used for tiles storage providers factory initialization before the tiles processor
-  const tilesStorageProviders = container.resolve<TilesStorageProvider[]>(TILES_STORAGE_PROVIDERS);
-  const processor = container.resolve(TileProcessor);
+  const processor = container.resolve<MetatileProcessor>(METATILE_PROCESSOR);
   const queueProv = container.resolve<JobQueueProvider>(JOB_QUEUE_PROVIDER);
   const logger = container.resolve<Logger>(SERVICES.LOGGER);
   const config = container.resolve<ConfigType>(SERVICES.CONFIG);

src/common/config.ts

@@ -1,8 +1,8 @@
 import { type ConfigInstance, config } from '@map-colonies/config';
-import { vectorRetilerV2, type vectorRetilerV2Type } from '@map-colonies/schemas';
+import { vectorRetilerV3, type vectorRetilerV3Type } from '@map-colonies/schemas';
 
 // Choose here the type of the config instance and import this type from the entire application
-type ConfigType = ConfigInstance<vectorRetilerV2Type>;
+type ConfigType = ConfigInstance<vectorRetilerV3Type>;
 
 let configInstance: ConfigType | undefined;
 
@@ -13,7 +13,7 @@ let configInstance: ConfigType | undefined;
  */
 async function initConfig(offlineMode?: boolean): Promise<void> {
   configInstance = await config({
-    schema: vectorRetilerV2,
+    schema: vectorRetilerV3,
     offlineMode,
   });
 }
@@ -28,4 +28,4 @@ function getConfig(): ConfigType {
 export { getConfig, initConfig };
 export type { ConfigType };
 
-export type VectorRetilerSchemaType = vectorRetilerV2Type;
+export type VectorRetilerSchemaType = vectorRetilerV3Type;

src/common/constants.ts

@@ -24,6 +24,8 @@ export const METRICS_REGISTRY = Symbol('MetricsRegistry');
 export const MAP_URL = Symbol('MapURL');
 export const MAP_FORMAT = Symbol('MapFormat');
 export const MAP_PROVIDER_CONFIG = Symbol('MapProviderConfig');
+export const SHIGOLA_CONFIG = Symbol('ShigolaConfig');
+export const METATILE_PROCESSOR = Symbol('MetatileProcessor');
 
 export const S3_BUCKET = Symbol('S3Bucket');
 export const TILES_STORAGE_LAYOUT = Symbol('TilesStorageLayout');

src/common/errors.ts

@@ -0,0 +1,10 @@
+/**
+ * An error that repeating the job cannot fix, such as a request the server refused as invalid.
+ * The job queue ends such a job instead of retrying it.
+ */
+export class NonRetryableError extends Error {
+  public constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
+    this.name = 'NonRetryableError';
+  }
+}

src/common/metrics.ts

@@ -34,6 +34,24 @@ export const enum SubTileStatus {
   OUT_OF_BOUNDS = 'out_of_bounds',
 }
 
+/** How a Shigola Cache Admin API request ended, and so what becomes of its job. */
+export const enum ShigolaOutcome {
+  /** 2xx, the job completes */
+  COMPLETED = 'completed',
+  /** a 4xx that repeating cannot fix, the job ends without a retry */
+  REJECTED = 'rejected',
+  /** 401, 403, 404, 429, 5xx, timeout or connection error, the job is retried */
+  FAILED = 'failed',
+}
+
+/** The tiles a Shigola cache operation reported. */
+export const enum ShigolaTileResult {
+  /** purged or seeded */
+  DONE = 'done',
+  SKIPPED = 'skipped',
+  FAILED = 'failed',
+}
+
 export const endMetricTimer = (timer?: ReturnType<Histogram['startTimer']>): void => {
   if (timer) {
     timer();

src/common/tracing/job.ts

@@ -13,6 +13,12 @@ export const jobAttributes = {
   TILES_BLANK_COUNT: 'tiles.blank_count',
   TILES_OUT_OF_BOUNDS_COUNT: 'tiles.out_of_bounds_count',
   MAP_PROVIDER: 'map.provider',
+  HTTP_STATUS: 'http.response.status_code',
+  SHIGOLA_OPERATION: 'shigola.operation',
+  SHIGOLA_OUTCOME: 'shigola.outcome',
+  SHIGOLA_TILES_DONE: 'shigola.tiles.done',
+  SHIGOLA_TILES_SKIPPED: 'shigola.tiles.skipped',
+  SHIGOLA_TILES_FAILED: 'shigola.tiles.failed',
 } satisfies Record<string, string>;
 
 export const spanName = {
@@ -24,4 +30,5 @@ export const spanName = {
   TILE_DELETE: 'tile.delete',
   TILE_POSTPROCESS: 'tile.post_process',
   JOB_PROCESS: 'job.process',
+  SHIGOLA_CACHE: 'shigola.cache',
 } satisfies Record<string, string>;

src/containerConfig.ts

@@ -28,6 +28,8 @@ import {
   METRICS_REGISTRY,
   ON_SIGNAL,
   HEALTHCHECK,
+  METATILE_PROCESSOR,
+  SHIGOLA_CONFIG,
 } from './common/constants';
 import type { InjectionObject } from './common/dependencyRegistration';
 import { registerDependencies } from './common/dependencyRegistration';
@@ -39,6 +41,8 @@ import { ArcgisMapProvider } from './retiler/mapProvider/arcgis/arcgisMapProvide
 import { SharpMapSplitter } from './retiler/mapSplitterProvider/sharp';
 import { consumeAndProcessFactory } from './app';
 import { WmsMapProvider } from './retiler/mapProvider/wms/wmsMapProvider';
+import { ShigolaCacheProcessor } from './retiler/shigola/shigolaCacheProcessor';
+import { TileProcessor } from './retiler/tileProcessor';
 import { tilesStorageProvidersFactory } from './retiler/tilesStorageProvider/factory';
 import type { ConfigType } from './common/config';
 import { getConfig } from './common/config';
@@ -206,11 +210,22 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise
           const config = container.resolve<ConfigType>(SERVICES.CONFIG);
           const mapProviderType = config.get('app.map.provider');
 
+          if (mapProviderType === 'shigola') {
+            // shigola caches the tiles itself, so no map provider, splitter, tiles storage or detiler is used
+            if (config.get('detiler.enabled')) {
+              container.resolve<Logger>(SERVICES.LOGGER).warn({ msg: 'detiler is enabled but not used with the shigola map provider' });
+            }
+            container.register(SHIGOLA_CONFIG, { useValue: config.get('app.map.shigola') });
+            container.register(METATILE_PROCESSOR, { useClass: ShigolaCacheProcessor }, { lifecycle: Lifecycle.Singleton });
+            return Promise.resolve();
+          }
+
           if (mapProviderType === 'wms') {
             container.register(MAP_PROVIDER, { useClass: WmsMapProvider });
           } else {
             container.register(MAP_PROVIDER, { useClass: ArcgisMapProvider });
           }
+          container.register(METATILE_PROCESSOR, { useClass: TileProcessor }, { lifecycle: Lifecycle.Singleton });
           return Promise.resolve();
         },
       },

src/retiler/interfaces.ts

@@ -7,6 +7,10 @@ export interface JobQueueProvider {
   stopQueue: () => Promise<void>;
 }
 
+export interface MetatileProcessor {
+  processTile: (tile: TileWithMetadata) => Promise<void>;
+}
+
 export interface MapProvider {
   getMap: (tile: TileWithMetadata) => Promise<Buffer>;
 }

src/retiler/jobQueueProvider/pgBossJobQueue.ts

@@ -8,6 +8,7 @@ import { serializeError } from 'serialize-error';
 import { type Tracer, SpanStatusCode } from '@opentelemetry/api';
 import { jobAttributes, spanName } from '@src/common/tracing/job';
 import { METRICS_REGISTRY, QUEUE_EMPTY_TIMEOUT, QUEUE_NAME, SERVICES } from '../../common/constants';
+import { NonRetryableError } from '../../common/errors';
 import { JobQueueProvider } from '../interfaces';
 
 @injectable()
@@ -123,8 +124,14 @@ export class PgBossJobQueueProvider implements JobQueueProvider {
         span.setStatus({ code: SpanStatusCode.OK });
       } catch (err) {
         const error = err as Error;
-        this.logger.error({ err: error, jobId: job.id, job });
-        await this.pgBoss.fail(this.queueName, job.id, serializeError(error));
+        if (error instanceof NonRetryableError) {
+          // pg-boss fail() retries while retries remain, cancel() is its only terminal state without a retry
+          this.logger.error({ msg: 'job cannot succeed on retry, cancelling it', err: error, jobId: job.id, job });
+          await this.pgBoss.cancel(this.queueName, job.id);
+        } else {
+          this.logger.error({ err: error, jobId: job.id, job });
+          await this.pgBoss.fail(this.queueName, job.id, serializeError(error));
+        }
         span.setStatus({ code: SpanStatusCode.ERROR });
         span.recordException(error);
       } finally {