From 41f2d65613c445490c2b4b16b6f2b4a622c2b87a Mon Sep 17 00:00:00 2001 From: shlomiko Date: Thu, 15 Oct 2026 16:20:41 +0300 Subject: [PATCH 1/8] fix: delete only cache tiles that can touch the previous footprint (MAPCO-12035) The update cache deletion job walked the whole update area from zoom 0 to the configured max zoom, so an update that adds a new area next to an existing layer produced millions of tiles above zoom 16 for ground that was never served and never cached. The update job now records the catalog footprint as previousFootprint at init, before the polygon parts are merged. The cache deletion creator clips the update area per zoom to the previous footprint buffered by one tile diagonal, and stops the walk at the first zoom whose clip is empty, since a deeper tile cannot touch both areas when its parent does not. When the footprint cannot be recorded the whole update area is walked as before. The buffer is taken in grid degrees: turf measures along the globe, and a buffer of many degrees around a footprint away from the equator bends around the sphere and misses the footprint, which ended the walk after zoom 0 for any real layer. --- .../ingestion/cacheDeletionJobCreator.ts | 102 ++++++++- src/job/models/ingestion/updateJobHandler.ts | 34 +++ src/utils/zod/schemas/jobParameters.schema.ts | 11 +- .../cacheDeletionJobCreator.spec.ts | 204 +++++++++++++++++- .../updateJobHandler/updateJobHandler.spec.ts | 49 +++++ .../updateJobHandler/updateJobHandlerSetup.ts | 2 +- 6 files changed, 394 insertions(+), 8 deletions(-) diff --git a/src/job/models/ingestion/cacheDeletionJobCreator.ts b/src/job/models/ingestion/cacheDeletionJobCreator.ts index 2df4c43c..77e691e9 100644 --- a/src/job/models/ingestion/cacheDeletionJobCreator.ts +++ b/src/job/models/ingestion/cacheDeletionJobCreator.ts @@ -1,5 +1,8 @@ import type { Logger } from '@map-colonies/js-logger'; -import { footprintToTileRanges } from '@map-colonies/mc-utils'; +import type { Feature, MultiPolygon, Polygon } from 'geojson'; +import { bbox, buffer, clone, coordEach, feature, featureCollection, intersect } from '@turf/turf'; +import { degreesPerTile, footprintToTileRanges } from '@map-colonies/mc-utils'; +import type { ITileRange } from '@map-colonies/mc-utils'; import type { ICreateJobBody, ICreateTaskBody } from '@map-colonies/mc-priority-queue'; import { OperationStatus, TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; import type { CacheDeletionJobParams, LayerName } from '@map-colonies/raster-shared'; @@ -30,6 +33,50 @@ interface CacheDeletionStrategy { * - swap-update: wipes entire prefix (layer redirects to new tiles path) * - update: deletes specific tile ranges over ingested footprint across all zooms */ +/** the globe is flat for practical purposes across this many degrees around the equator */ +const FLAT_EXTENT_DEGREES = 1; +/** + * keeps the buffer a superset of the grid distance: turf draws the round ends as polygons inscribed in the + * circle, which fall short of the radius by up to half a percent, and the flat patch has a little curvature + */ +const FLAT_BUFFER_MARGIN = 1.01; +const HALF = 2; + +const midpoint = (a: number, b: number): number => a + (b - a) / HALF; + +const mapCoordinates = (geometry: G, map: (x: number, y: number) => [number, number]): G => { + const mapped = clone(geometry); + coordEach(mapped, (coord) => { + const [x, y] = map(coord[0] as number, coord[1] as number); + coord[0] = x; + coord[1] = y; + }); + return mapped; +}; + +/** + * Buffers a footprint by a distance in grid degrees, the metric of the WorldCRS84 tile grid. + * + * turf measures a buffer along the globe, so a buffer of many degrees around a footprint away from the + * equator bends around the sphere and can even leave the footprint itself outside the result. The footprint + * is therefore moved to the equator and scaled down to a patch small enough to be flat, buffered there, and + * mapped back, which makes the buffer planar in degrees as the grid requires. + */ +const bufferInGridDegrees = (footprint: Feature, distance: number): Feature | undefined => { + const [minX, minY, maxX, maxY] = bbox(footprint); + const centerX = midpoint(minX, maxX); + const centerY = midpoint(minY, maxY); + // the flat patch has to hold the footprint plus the buffer on both sides + const scale = FLAT_EXTENT_DEGREES / (Math.max(maxX - minX, maxY - minY) + distance + distance); + + const flat = mapCoordinates(footprint.geometry, (x, y) => [(x - centerX) * scale, (y - centerY) * scale]); + const buffered = buffer(flat, distance * scale * FLAT_BUFFER_MARGIN, { units: 'degrees' }); + if (buffered === undefined) { + return undefined; + } + return feature(mapCoordinates(buffered.geometry, (x, y) => [x / scale + centerX, y / scale + centerY])); +}; + @injectable() export class CacheDeletionJobCreator { private readonly taskConfig: CacheDeletionTaskConfig; @@ -141,11 +188,22 @@ export class CacheDeletionJobCreator { private async *buildRangeTasks(job: IngestionFinalizeJob, prefix: string, logger: Logger): AsyncGenerator { const { maxZoom, tileBatchSize, maxRangesPerTask } = this.taskConfig; + const { previousFootprint } = job.parameters.additionalParams; + + const updateArea = await this.readProductGeometry(job.parameters.inputFiles.productShapefilePath); + logger.info({ + msg: 'Computing tile ranges over the updated footprint', + maxZoom, + tileBatchSize, + maxRangesPerTask, + restrictedToPreviousFootprint: previousFootprint !== undefined, + }); - const geometry = await this.readProductGeometry(job.parameters.inputFiles.productShapefilePath); - logger.info({ msg: 'Computing tile ranges over the updated footprint', maxZoom, tileBatchSize, maxRangesPerTask }); - - const ranges = footprintToTileRanges(geometry, { minZoom: 0, maxZoom }); + // without the previous footprint (jobs created before init recorded it) every tile of the update area is covered + const ranges = + previousFootprint === undefined + ? footprintToTileRanges(updateArea, { minZoom: 0, maxZoom }) + : this.rangesTouchingPreviousFootprint(updateArea, previousFootprint, maxZoom, logger); for await (const batch of limitedTileBatchGenerator(tileBatchSize, maxRangesPerTask, ranges)) { yield { @@ -156,6 +214,40 @@ export class CacheDeletionJobCreator { } } + /** + * Walks only the tiles that can hold a cached tile of the replaced data. + * + * Only the previous footprint ever had tiles to cache, so a tile of the update area that does not also + * touch it was never cached and needs no deletion. A tile of side `s` that touches the previous footprint + * lies within `s * sqrt(2)` of it, so clipping the update area to a buffer of that reach, per zoom, gives a + * tight superset of the tiles touching both: at most a rim about two tiles wide along the previous + * boundary. Touching is monotone down the pyramid (a child touching both implies its parent does), so the + * first zoom with an empty clip ends the walk - for an update outside the previous footprint that happens + * at a low zoom, leaving just the coarse tiles both areas share. + */ + private *rangesTouchingPreviousFootprint( + updateArea: Polygon | MultiPolygon, + previousFootprint: Polygon | MultiPolygon, + maxZoom: number, + logger: Logger + ): Generator { + const update = feature(updateArea); + const previous = feature(previousFootprint); + + for (let zoom = 0; zoom <= maxZoom; zoom++) { + const reach = degreesPerTile(zoom) * Math.SQRT2; + const nearPrevious = bufferInGridDegrees(previous, reach); + const target = nearPrevious === undefined ? null : intersect(featureCollection([update, nearPrevious])); + + if (target === null) { + logger.info({ msg: 'Update area stops touching the previous footprint, ending the range walk', lastZoom: zoom - 1, maxZoom }); + return; + } + + yield* footprintToTileRanges(target, { minZoom: zoom, maxZoom: zoom }); + } + } + private async createJobWithStreamedTasks( job: IngestionFinalizeJob, jobType: string, diff --git a/src/job/models/ingestion/updateJobHandler.ts b/src/job/models/ingestion/updateJobHandler.ts index b414a0d6..73e9b477 100644 --- a/src/job/models/ingestion/updateJobHandler.ts +++ b/src/job/models/ingestion/updateJobHandler.ts @@ -10,6 +10,7 @@ import { IngestionUpdateFinalizeTask, IngestionUpdateCreateTasksJob, } from '../../../utils/zod/schemas/job.schema'; +import { previousFootprintSchema } from '../../../utils/zod/schemas/jobParameters.schema'; import { PolygonPartsMangerClient } from '../../../httpClients/polygonPartsMangerClient'; import { CatalogClient } from '../../../httpClients/catalogClient'; import type { ReadProductGeometry } from '../../../utils/storage/productReader'; @@ -66,6 +67,8 @@ export class UpdateJobHandler const layerRelativePath = join(job.internalId, additionalParams.displayPath); + await this.recordPreviousFootprint(job, job.internalId, logger); + logger.info({ msg: 'building tasks' }); const { polygonPartsEntityName } = this.validateAndGenerateLayerNameFormats(job); @@ -133,4 +136,35 @@ export class UpdateJobHandler } ); } + + /** + * Records the layer's catalog footprint, as it stands before this update lands, on the job's additional + * params. The cache deletion job created at finalize uses it to leave out the part of the update area that + * never had tiles to cache, and by then the catalog already holds the merged footprint. Best effort: without + * it the cache deletion covers the whole update area, so a failure here must not fail the ingestion. + */ + private async recordPreviousFootprint(job: IngestionUpdateCreateTasksJob, catalogId: string, logger: Logger): Promise { + const activeSpan = trace.getActiveSpan(); + + try { + const { metadata } = await this.catalogClient.findLayer(catalogId); + const previousFootprint = previousFootprintSchema.parse(metadata.footprint); + + await this.queueClient.jobManagerClient.updateJob(job.id, { + parameters: { ...job.parameters, additionalParams: { ...job.parameters.additionalParams, previousFootprint } }, + }); + + logger.info({ msg: 'Recorded the previous layer footprint on the job for the cache deletion', catalogId }); + activeSpan?.addEvent('recordPreviousFootprint.success', { catalogId }); + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + logger.warn({ + msg: 'Could not record the previous layer footprint, the cache deletion will cover the whole update area', + catalogId, + reason, + err, + }); + activeSpan?.addEvent('recordPreviousFootprint.skipped', { catalogId, reason }); + } + } } diff --git a/src/utils/zod/schemas/jobParameters.schema.ts b/src/utils/zod/schemas/jobParameters.schema.ts index d7cde528..14ff4242 100644 --- a/src/utils/zod/schemas/jobParameters.schema.ts +++ b/src/utils/zod/schemas/jobParameters.schema.ts @@ -7,6 +7,8 @@ import { taskBlockDuplicationParamSchema, ingestionBaseJobParamsSchema, ingestionUpdateJobParamsSchema, + multiPolygonSchema, + polygonSchema, } from '@map-colonies/raster-shared'; import { z } from 'zod'; import { Grid } from '../../../common/interfaces'; @@ -39,7 +41,14 @@ export const internalIdSchema = z.object({ }); //#region AdditionalParams -export const updateFinalizeAdditionalParamsSchema = updateAdditionalParamsSchema; +/** the layer's catalog footprint as it stood before the update, recorded by the update job's init step */ +export const previousFootprintSchema = polygonSchema.or(multiPolygonSchema); +export type PreviousFootprint = z.infer; + +export const updateFinalizeAdditionalParamsSchema = updateAdditionalParamsSchema.extend({ + // optional so jobs created before overseer recorded it still finalize; cache deletion then covers the whole update area + previousFootprint: previousFootprintSchema.optional(), +}); export const swapUpdateFinalizeAdditionalParamsSchema = updateFinalizeAdditionalParamsSchema.extend({ displayPath: updateAdditionalParamsSchema.shape.displayPath.optional(), diff --git a/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts b/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts index b6cd4ba6..0262ae6d 100644 --- a/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts +++ b/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts @@ -3,7 +3,9 @@ import { randomUUID } from 'node:crypto'; import type { MultiPolygon, Polygon } from 'geojson'; import { OperationStatus } from '@map-colonies/mc-priority-queue'; import { StorageProvider, type CacheDeletionJobParams } from '@map-colonies/raster-shared'; +import { degreesPerTile } from '@map-colonies/mc-utils'; import type { CreateCacheDeletionJobParams, CacheDeletionTaskConfig, GetMapproxyCacheResponse } from '../../../../src/common/interfaces'; +import type { IngestionUpdateFinalizeJob } from '../../../../src/utils/zod/schemas/job.schema'; import { registerDefaultConfig, configMock, setValue } from '../../mocks/configMock'; import { createFakePolygonalGeometry } from '../../mocks/geometryMockData'; import { LayerCacheType } from '../../../../src/common/constants'; @@ -151,7 +153,7 @@ describe('CacheDeletionJobCreator', () => { } }); - it('should cover every zoom from 0 to maxZoom', async () => { + it('should cover the whole update area at every zoom from 0 to maxZoom when no previous footprint is recorded', async () => { const { cacheDeletionJobCreator, jobManagerClientMock, mapproxyClientMock, readProductGeometryMock } = ctx; const taskConfig = configMock.get('jobManagement.ingestion.tasks.cacheDeletion'); @@ -172,6 +174,206 @@ describe('CacheDeletionJobCreator', () => { expect(Math.max(...zooms)).toBe(taskConfig.maxZoom); }); + describe('with the previous layer footprint recorded on the job', () => { + interface Range { + zoom: number; + minX: number; + maxX: number; + minY: number; + maxY: number; + } + interface Tile { + zoom: number; + x: number; + y: number; + } + interface Bounds { + minLon: number; + maxLon: number; + minLat: number; + maxLat: number; + } + + // axis-aligned squares: tile/footprint contact reduces to interval overlap, which keeps the exhaustive + // oracle below trivial + const square = (minLon: number, minLat: number, size: number): Polygon => ({ + type: 'Polygon', + coordinates: [ + [ + [minLon, minLat], + [minLon + size, minLat], + [minLon + size, minLat + size], + [minLon, minLat + size], + [minLon, minLat], + ], + ], + }); + const PREVIOUS = square(0, 0, 1); + + const withPreviousFootprint = (previousFootprint: Polygon): IngestionUpdateFinalizeJob => ({ + ...ingestionUpdateFinalizeJob, + parameters: { + ...ingestionUpdateFinalizeJob.parameters, + additionalParams: { ...ingestionUpdateFinalizeJob.parameters.additionalParams, previousFootprint }, + }, + }); + + const emittedRanges = (jobManagerClientMock: CacheDeletionJobCreatorTestContext['jobManagerClientMock']): Range[] => { + const streamed = jobManagerClientMock.createTaskForJob.mock.calls.flatMap((call) => call[1] as { parameters: unknown }[]); + const tasks = [...jobManagerClientMock.createJob.mock.calls[0]![0].tasks!, ...streamed]; + return tasks.flatMap((task) => (task.parameters as { ranges: Range[] }).ranges); + }; + + const covers = (ranges: Range[], tile: Tile): boolean => + ranges.some( + (range) => range.zoom === tile.zoom && tile.x >= range.minX && tile.x <= range.maxX && tile.y >= range.minY && tile.y <= range.maxY + ); + + // west edge of a tile column on the geodetic grid + const tileMinLon = (zoom: number, x: number): number => -180 + x * degreesPerTile(zoom); + + const boundsOf = (polygon: Polygon): Bounds => { + const ring = polygon.coordinates[0]!; + return { + minLon: Math.min(...ring.map((point) => point[0]!)), + maxLon: Math.max(...ring.map((point) => point[0]!)), + minLat: Math.min(...ring.map((point) => point[1]!)), + maxLat: Math.max(...ring.map((point) => point[1]!)), + }; + }; + + // exhaustive oracle: every tile at `zoom` whose square touches both footprints, with closed bounds + // since footprintToTileRanges counts a shared edge as touching + const tilesTouchingBoth = (a: Polygon, b: Polygon, zoom: number): Tile[] => { + const side = degreesPerTile(zoom); + const [boundsA, boundsB] = [boundsOf(a), boundsOf(b)]; + const touches = (bounds: Bounds, minLon: number, minLat: number): boolean => + minLon <= bounds.maxLon && minLon + side >= bounds.minLon && minLat <= bounds.maxLat && minLat + side >= bounds.minLat; + + const tiles: Tile[] = []; + for (let x = Math.floor((boundsA.minLon + 180) / side) - 1; x <= Math.floor((boundsA.maxLon + 180) / side) + 1; x++) { + for (let y = Math.floor((boundsA.minLat + 90) / side) - 1; y <= Math.floor((boundsA.maxLat + 90) / side) + 1; y++) { + const minLon = tileMinLon(zoom, x); + const minLat = -90 + y * side; + if (touches(boundsA, minLon, minLat) && touches(boundsB, minLon, minLat)) { + tiles.push({ zoom, x, y }); + } + } + } + return tiles; + }; + + it('should walk no deeper than the last zoom whose tiles can still span both areas when the update lies outside the previous footprint', async () => { + const { cacheDeletionJobCreator, jobManagerClientMock, mapproxyClientMock, readProductGeometryMock } = ctx; + const taskConfig = configMock.get('jobManagement.ingestion.tasks.cacheDeletion'); + // 1.5 degrees east of PREVIOUS. A zoom 7 tile is 1.41 degrees wide (1.99 across its diagonal), so the + // clip still reaches the update area; a zoom 8 tile is 0.70 wide (0.99 across) and cannot bridge the gap + const UPDATE = square(2.5, 0, 1); + const LAST_BRIDGING_ZOOM = 7; + + mapproxyClientMock.getRedisCache.mockResolvedValue(redisCache('layer-Orthophoto')); + readProductGeometryMock.mockResolvedValue(UPDATE); + jobManagerClientMock.createJob.mockResolvedValue({ id: randomUUID(), taskIds: [randomUUID()] }); + jobManagerClientMock.createTaskForJob.mockResolvedValue(undefined); + + await cacheDeletionJobCreator.create({ layerName: 'layer-Orthophoto', ingestionJob: withPreviousFootprint(PREVIOUS) }); + + const ranges = emittedRanges(jobManagerClientMock); + const zooms = ranges.map((range) => range.zoom); + + expect(Math.min(...zooms)).toBe(0); + expect(Math.max(...zooms)).toBe(LAST_BRIDGING_ZOOM); + // the full walk would have gone on to the configured ceiling + expect(LAST_BRIDGING_ZOOM).toBeLessThan(taskConfig.maxZoom); + + // nothing that genuinely touches both areas is dropped + for (let zoom = 0; zoom <= LAST_BRIDGING_ZOOM; zoom++) { + for (const tile of tilesTouchingBoth(UPDATE, PREVIOUS, zoom)) { + expect(covers(ranges, tile)).toBe(true); + } + } + + // and that check is not vacuous: at zoom 6 a single 2.8-degree tile spans the gap + expect(tilesTouchingBoth(UPDATE, PREVIOUS, 6).length).toBeGreaterThan(0); + }); + + it('should reach the configured max zoom over the overlap but stay out of the new-only ground beyond it', async () => { + // a shallow ceiling keeps the walk over the overlap small + const MAX_ZOOM = 10; + setValue('jobManagement.ingestion.tasks.cacheDeletion.maxZoom', MAX_ZOOM); + const { cacheDeletionJobCreator, jobManagerClientMock, mapproxyClientMock, readProductGeometryMock } = + await setupCacheDeletionJobCreatorTest(); + // overlaps PREVIOUS on lon 0.5..1 and extends onto new ground up to lon 1.5 + const UPDATE = square(0.5, 0, 1); + const NEW_GROUND_EDGE = 1; + + mapproxyClientMock.getRedisCache.mockResolvedValue(redisCache('layer-Orthophoto')); + readProductGeometryMock.mockResolvedValue(UPDATE); + jobManagerClientMock.createJob.mockResolvedValue({ id: randomUUID(), taskIds: [randomUUID()] }); + jobManagerClientMock.createTaskForJob.mockResolvedValue(undefined); + + await cacheDeletionJobCreator.create({ layerName: 'layer-Orthophoto', ingestionJob: withPreviousFootprint(PREVIOUS) }); + + const ranges = emittedRanges(jobManagerClientMock); + + expect(Math.max(...ranges.map((range) => range.zoom))).toBe(MAX_ZOOM); + + for (let zoom = 0; zoom <= MAX_ZOOM; zoom++) { + for (const tile of tilesTouchingBoth(UPDATE, PREVIOUS, zoom)) { + expect(covers(ranges, tile)).toBe(true); + } + } + + // at the deepest zoom nothing starts farther from PREVIOUS than one tile diagonal: the new-only ground + // beyond that was never cached and is left alone + const reach = degreesPerTile(MAX_ZOOM) * Math.SQRT2; + const deepest = ranges.filter((range) => range.zoom === MAX_ZOOM); + + expect(deepest.length).toBeGreaterThan(0); + + for (const range of deepest) { + expect(tileMinLon(MAX_ZOOM, range.maxX)).toBeLessThanOrEqual(NEW_GROUND_EDGE + reach); + } + // whereas the update area itself does extend past that reach, so the full walk would have gone there + const farEdgeTile = Math.floor((boundsOf(UPDATE).maxLon + 180) / degreesPerTile(MAX_ZOOM)); + + expect(tileMinLon(MAX_ZOOM, farEdgeTile)).toBeGreaterThan(NEW_GROUND_EDGE + reach); + }); + + it('should measure the reach in grid degrees for a layer far from the equator', async () => { + const { cacheDeletionJobCreator, jobManagerClientMock, mapproxyClientMock, readProductGeometryMock } = ctx; + // a layer near 35E 32N with an update 0.05 degrees east of it. A zoom 12 tile is 0.044 wide (0.062 + // across its diagonal) so its clip still reaches the update; a zoom 13 tile (0.031 across) cannot. + // An arc-length buffer of 127 degrees at zoom 1 bends around the globe and misses this latitude + // altogether, which used to end the walk after zoom 0 + const PREVIOUS_NORTH = square(34.9, 32, 0.2); + const UPDATE = square(35.15, 32.05, 0.1); + const LAST_BRIDGING_ZOOM = 12; + + mapproxyClientMock.getRedisCache.mockResolvedValue(redisCache('layer-Orthophoto')); + readProductGeometryMock.mockResolvedValue(UPDATE); + jobManagerClientMock.createJob.mockResolvedValue({ id: randomUUID(), taskIds: [randomUUID()] }); + jobManagerClientMock.createTaskForJob.mockResolvedValue(undefined); + + await cacheDeletionJobCreator.create({ layerName: 'layer-Orthophoto', ingestionJob: withPreviousFootprint(PREVIOUS_NORTH) }); + + const ranges = emittedRanges(jobManagerClientMock); + const zooms = ranges.map((range) => range.zoom); + + expect(Math.min(...zooms)).toBe(0); + expect(Math.max(...zooms)).toBe(LAST_BRIDGING_ZOOM); + + for (let zoom = 0; zoom <= LAST_BRIDGING_ZOOM; zoom++) { + for (const tile of tilesTouchingBoth(UPDATE, PREVIOUS_NORTH, zoom)) { + expect(covers(ranges, tile)).toBe(true); + } + } + + // not vacuous: at zoom 11 a 0.088-degree tile still spans the gap + expect(tilesTouchingBoth(UPDATE, PREVIOUS_NORTH, 11).length).toBeGreaterThan(0); + }); + }); + it('should stream the remaining tasks with createTaskForJob once the first batch fills the job', async () => { const jobId = randomUUID(); diff --git a/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts b/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts index beecbaba..c7525749 100644 --- a/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts +++ b/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts @@ -2,6 +2,7 @@ import { updateAdditionalParamsSchema } from '@map-colonies/raster-shared'; import { finalizeTaskForIngestionUpdate, createTasksTaskForIngestionUpdate } from '../../mocks/tasksMockData'; import { createFakePolygonalGeometry } from '../../mocks/geometryMockData'; +import { layerRecord } from '../../mocks/catalogClientMockData'; import { registerDefaultConfig } from '../../mocks/configMock'; import { ingestionUpdateFinalizeJob, ingestionUpdateJob } from '../../mocks/jobsMockData'; import { setupUpdateJobHandlerTest, type UpdateJobHandlerTestContext } from './updateJobHandlerSetup'; @@ -42,6 +43,8 @@ describe('updateJobHandler', () => { const polygonPartsEntityName = `${job.resourceId}_${String(job.productType).toLowerCase()}`; readProductGeometryMock.mockResolvedValue(productGeometry); + catalogClientMock.findLayer.mockResolvedValue(layerRecord); + jobManagerClientMock.updateJob.mockResolvedValue(undefined); tileDeletionTaskManagerMock.buildAndPushTasks.mockResolvedValue(undefined); taskBuilderMock.buildAndPushTasks.mockResolvedValue(undefined); queueClientMock.ack.mockResolvedValue(undefined); @@ -53,6 +56,52 @@ describe('updateJobHandler', () => { expect(queueClientMock.ack).toHaveBeenCalledWith(job.id, task.id); }); + it('should record the previous layer footprint on the job before building tasks', async () => { + const job = structuredClone(ingestionUpdateJob); + const task = createTasksTaskForIngestionUpdate; + + readProductGeometryMock.mockResolvedValue(createFakePolygonalGeometry()); + catalogClientMock.findLayer.mockResolvedValue(layerRecord); + jobManagerClientMock.updateJob.mockResolvedValue(undefined); + tileDeletionTaskManagerMock.buildAndPushTasks.mockResolvedValue(undefined); + taskBuilderMock.buildAndPushTasks.mockResolvedValue(undefined); + queueClientMock.ack.mockResolvedValue(undefined); + + await updateJobHandler.handleJobInit(job, task); + + expect(catalogClientMock.findLayer).toHaveBeenCalledWith(job.internalId); + // finalize reads it back from the job, by which time the catalog already holds the merged footprint + expect(jobManagerClientMock.updateJob).toHaveBeenCalledWith(job.id, { + parameters: { + ...job.parameters, + additionalParams: { ...job.parameters.additionalParams, previousFootprint: layerRecord.metadata.footprint }, + }, + }); + expect(jobManagerClientMock.updateJob.mock.invocationCallOrder[0]).toBeLessThan(taskBuilderMock.buildAndPushTasks.mock.invocationCallOrder[0]!); + }); + + it('should still build the tasks when the previous footprint cannot be recorded', async () => { + const job = structuredClone(ingestionUpdateJob); + const task = createTasksTaskForIngestionUpdate; + + readProductGeometryMock.mockResolvedValue(createFakePolygonalGeometry()); + catalogClientMock.findLayer.mockRejectedValue(new Error('catalog unavailable')); + tileDeletionTaskManagerMock.buildAndPushTasks.mockResolvedValue(undefined); + taskBuilderMock.buildAndPushTasks.mockResolvedValue(undefined); + queueClientMock.ack.mockResolvedValue(undefined); + + await updateJobHandler.handleJobInit(job, task); + + const recordedFootprint = jobManagerClientMock.updateJob.mock.calls.some( + ([, body]) => + (body as { parameters?: { additionalParams?: Record } }).parameters?.additionalParams?.previousFootprint !== undefined + ); + + expect(recordedFootprint).toBe(false); + expect(taskBuilderMock.buildAndPushTasks).toHaveBeenCalledTimes(1); + expect(queueClientMock.ack).toHaveBeenCalledWith(job.id, task.id); + }); + it('should reject task when mergeTaskManager.buildAndPushTasks throws', async () => { const job = structuredClone(ingestionUpdateJob); const task = createTasksTaskForIngestionUpdate; diff --git a/tests/unit/job/updateJobHandler/updateJobHandlerSetup.ts b/tests/unit/job/updateJobHandler/updateJobHandlerSetup.ts index 9acdf266..5ab7ca52 100644 --- a/tests/unit/job/updateJobHandler/updateJobHandlerSetup.ts +++ b/tests/unit/job/updateJobHandler/updateJobHandlerSetup.ts @@ -40,7 +40,7 @@ export const setupUpdateJobHandlerTest = async (): Promise; const mapproxyClientMock = { publish: vi.fn() } as unknown as Mocked; - const catalogClientMock = { publish: vi.fn(), update: vi.fn() } as unknown as Mocked; + const catalogClientMock = { publish: vi.fn(), update: vi.fn(), findLayer: vi.fn() } as unknown as Mocked; const cacheDeletionJobCreatorMock = { create: vi.fn() } as unknown as Mocked; const updateJobHandler = new UpdateJobHandler( From fd9d42f6c31ce9c36238c02b2246f1d10a962f7d Mon Sep 17 00:00:00 2001 From: shlomiko Date: Sun, 18 Oct 2026 10:51:58 +0300 Subject: [PATCH 2/8] refactor: buffer the previous footprint directly and skip the clip at coarse zooms (MAPCO-12035) turf measures a buffer along the globe, which is a superset of the grid distance as long as the buffer is not wide enough to bend around the sphere. The coarse zooms whose reach exceeds ten degrees are therefore walked over the whole update area, a handful of tiles that are nearly always shared, and the deeper zooms buffer the footprint as it is. This removes the flattening of the footprint onto an equatorial patch and the coordinate mapping that came with it. --- .../ingestion/cacheDeletionJobCreator.ts | 59 +++++-------------- .../cacheDeletionJobCreator.spec.ts | 4 +- 2 files changed, 17 insertions(+), 46 deletions(-) diff --git a/src/job/models/ingestion/cacheDeletionJobCreator.ts b/src/job/models/ingestion/cacheDeletionJobCreator.ts index 77e691e9..1d026dd4 100644 --- a/src/job/models/ingestion/cacheDeletionJobCreator.ts +++ b/src/job/models/ingestion/cacheDeletionJobCreator.ts @@ -1,6 +1,6 @@ import type { Logger } from '@map-colonies/js-logger'; -import type { Feature, MultiPolygon, Polygon } from 'geojson'; -import { bbox, buffer, clone, coordEach, feature, featureCollection, intersect } from '@turf/turf'; +import type { MultiPolygon, Polygon } from 'geojson'; +import { buffer, feature, featureCollection, intersect } from '@turf/turf'; import { degreesPerTile, footprintToTileRanges } from '@map-colonies/mc-utils'; import type { ITileRange } from '@map-colonies/mc-utils'; import type { ICreateJobBody, ICreateTaskBody } from '@map-colonies/mc-priority-queue'; @@ -33,49 +33,14 @@ interface CacheDeletionStrategy { * - swap-update: wipes entire prefix (layer redirects to new tiles path) * - update: deletes specific tile ranges over ingested footprint across all zooms */ -/** the globe is flat for practical purposes across this many degrees around the equator */ -const FLAT_EXTENT_DEGREES = 1; /** - * keeps the buffer a superset of the grid distance: turf draws the round ends as polygons inscribed in the - * circle, which fall short of the radius by up to half a percent, and the flat patch has a little curvature + * turf measures a buffer along the globe. Up to this reach the result holds every point within that many grid + * degrees of the footprint (a degree of latitude is the same distance everywhere and a degree of longitude is + * shorter away from the equator), while a wider buffer bends around the sphere and falls apart */ -const FLAT_BUFFER_MARGIN = 1.01; -const HALF = 2; - -const midpoint = (a: number, b: number): number => a + (b - a) / HALF; - -const mapCoordinates = (geometry: G, map: (x: number, y: number) => [number, number]): G => { - const mapped = clone(geometry); - coordEach(mapped, (coord) => { - const [x, y] = map(coord[0] as number, coord[1] as number); - coord[0] = x; - coord[1] = y; - }); - return mapped; -}; - -/** - * Buffers a footprint by a distance in grid degrees, the metric of the WorldCRS84 tile grid. - * - * turf measures a buffer along the globe, so a buffer of many degrees around a footprint away from the - * equator bends around the sphere and can even leave the footprint itself outside the result. The footprint - * is therefore moved to the equator and scaled down to a patch small enough to be flat, buffered there, and - * mapped back, which makes the buffer planar in degrees as the grid requires. - */ -const bufferInGridDegrees = (footprint: Feature, distance: number): Feature | undefined => { - const [minX, minY, maxX, maxY] = bbox(footprint); - const centerX = midpoint(minX, maxX); - const centerY = midpoint(minY, maxY); - // the flat patch has to hold the footprint plus the buffer on both sides - const scale = FLAT_EXTENT_DEGREES / (Math.max(maxX - minX, maxY - minY) + distance + distance); - - const flat = mapCoordinates(footprint.geometry, (x, y) => [(x - centerX) * scale, (y - centerY) * scale]); - const buffered = buffer(flat, distance * scale * FLAT_BUFFER_MARGIN, { units: 'degrees' }); - if (buffered === undefined) { - return undefined; - } - return feature(mapCoordinates(buffered.geometry, (x, y) => [x / scale + centerX, y / scale + centerY])); -}; +const MAX_BUFFERED_REACH_DEGREES = 10; +/** turf draws the round ends of a buffer as polygons inscribed in the circle, which fall short of the radius by up to half a percent */ +const REACH_MARGIN = 1.01; @injectable() export class CacheDeletionJobCreator { @@ -236,7 +201,13 @@ export class CacheDeletionJobCreator { for (let zoom = 0; zoom <= maxZoom; zoom++) { const reach = degreesPerTile(zoom) * Math.SQRT2; - const nearPrevious = bufferInGridDegrees(previous, reach); + if (reach > MAX_BUFFERED_REACH_DEGREES) { + // tiles this coarse are a handful and nearly always shared, so the whole update area is walked + yield* footprintToTileRanges(update, { minZoom: zoom, maxZoom: zoom }); + continue; + } + + const nearPrevious = buffer(previous, reach * REACH_MARGIN, { units: 'degrees' }); const target = nearPrevious === undefined ? null : intersect(featureCollection([update, nearPrevious])); if (target === null) { diff --git a/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts b/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts index 0262ae6d..ecf0349b 100644 --- a/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts +++ b/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts @@ -344,8 +344,8 @@ describe('CacheDeletionJobCreator', () => { const { cacheDeletionJobCreator, jobManagerClientMock, mapproxyClientMock, readProductGeometryMock } = ctx; // a layer near 35E 32N with an update 0.05 degrees east of it. A zoom 12 tile is 0.044 wide (0.062 // across its diagonal) so its clip still reaches the update; a zoom 13 tile (0.031 across) cannot. - // An arc-length buffer of 127 degrees at zoom 1 bends around the globe and misses this latitude - // altogether, which used to end the walk after zoom 0 + // A buffer of 127 degrees at zoom 1 would bend around the globe and miss this latitude altogether, + // which is why the coarse zooms are walked without a clip const PREVIOUS_NORTH = square(34.9, 32, 0.2); const UPDATE = square(35.15, 32.05, 0.1); const LAST_BRIDGING_ZOOM = 12; From ca7b2e767333af9d683da13c97e37d2dd3a23722 Mon Sep 17 00:00:00 2001 From: shlomiko Date: Sun, 18 Oct 2026 13:14:15 +0300 Subject: [PATCH 3/8] Revert "refactor: buffer the previous footprint directly and skip the clip at coarse zooms (MAPCO-12035)" This reverts commit fd9d42f6c31ce9c36238c02b2246f1d10a962f7d. --- .../ingestion/cacheDeletionJobCreator.ts | 59 ++++++++++++++----- .../cacheDeletionJobCreator.spec.ts | 4 +- 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/src/job/models/ingestion/cacheDeletionJobCreator.ts b/src/job/models/ingestion/cacheDeletionJobCreator.ts index 1d026dd4..77e691e9 100644 --- a/src/job/models/ingestion/cacheDeletionJobCreator.ts +++ b/src/job/models/ingestion/cacheDeletionJobCreator.ts @@ -1,6 +1,6 @@ import type { Logger } from '@map-colonies/js-logger'; -import type { MultiPolygon, Polygon } from 'geojson'; -import { buffer, feature, featureCollection, intersect } from '@turf/turf'; +import type { Feature, MultiPolygon, Polygon } from 'geojson'; +import { bbox, buffer, clone, coordEach, feature, featureCollection, intersect } from '@turf/turf'; import { degreesPerTile, footprintToTileRanges } from '@map-colonies/mc-utils'; import type { ITileRange } from '@map-colonies/mc-utils'; import type { ICreateJobBody, ICreateTaskBody } from '@map-colonies/mc-priority-queue'; @@ -33,14 +33,49 @@ interface CacheDeletionStrategy { * - swap-update: wipes entire prefix (layer redirects to new tiles path) * - update: deletes specific tile ranges over ingested footprint across all zooms */ +/** the globe is flat for practical purposes across this many degrees around the equator */ +const FLAT_EXTENT_DEGREES = 1; /** - * turf measures a buffer along the globe. Up to this reach the result holds every point within that many grid - * degrees of the footprint (a degree of latitude is the same distance everywhere and a degree of longitude is - * shorter away from the equator), while a wider buffer bends around the sphere and falls apart + * keeps the buffer a superset of the grid distance: turf draws the round ends as polygons inscribed in the + * circle, which fall short of the radius by up to half a percent, and the flat patch has a little curvature */ -const MAX_BUFFERED_REACH_DEGREES = 10; -/** turf draws the round ends of a buffer as polygons inscribed in the circle, which fall short of the radius by up to half a percent */ -const REACH_MARGIN = 1.01; +const FLAT_BUFFER_MARGIN = 1.01; +const HALF = 2; + +const midpoint = (a: number, b: number): number => a + (b - a) / HALF; + +const mapCoordinates = (geometry: G, map: (x: number, y: number) => [number, number]): G => { + const mapped = clone(geometry); + coordEach(mapped, (coord) => { + const [x, y] = map(coord[0] as number, coord[1] as number); + coord[0] = x; + coord[1] = y; + }); + return mapped; +}; + +/** + * Buffers a footprint by a distance in grid degrees, the metric of the WorldCRS84 tile grid. + * + * turf measures a buffer along the globe, so a buffer of many degrees around a footprint away from the + * equator bends around the sphere and can even leave the footprint itself outside the result. The footprint + * is therefore moved to the equator and scaled down to a patch small enough to be flat, buffered there, and + * mapped back, which makes the buffer planar in degrees as the grid requires. + */ +const bufferInGridDegrees = (footprint: Feature, distance: number): Feature | undefined => { + const [minX, minY, maxX, maxY] = bbox(footprint); + const centerX = midpoint(minX, maxX); + const centerY = midpoint(minY, maxY); + // the flat patch has to hold the footprint plus the buffer on both sides + const scale = FLAT_EXTENT_DEGREES / (Math.max(maxX - minX, maxY - minY) + distance + distance); + + const flat = mapCoordinates(footprint.geometry, (x, y) => [(x - centerX) * scale, (y - centerY) * scale]); + const buffered = buffer(flat, distance * scale * FLAT_BUFFER_MARGIN, { units: 'degrees' }); + if (buffered === undefined) { + return undefined; + } + return feature(mapCoordinates(buffered.geometry, (x, y) => [x / scale + centerX, y / scale + centerY])); +}; @injectable() export class CacheDeletionJobCreator { @@ -201,13 +236,7 @@ export class CacheDeletionJobCreator { for (let zoom = 0; zoom <= maxZoom; zoom++) { const reach = degreesPerTile(zoom) * Math.SQRT2; - if (reach > MAX_BUFFERED_REACH_DEGREES) { - // tiles this coarse are a handful and nearly always shared, so the whole update area is walked - yield* footprintToTileRanges(update, { minZoom: zoom, maxZoom: zoom }); - continue; - } - - const nearPrevious = buffer(previous, reach * REACH_MARGIN, { units: 'degrees' }); + const nearPrevious = bufferInGridDegrees(previous, reach); const target = nearPrevious === undefined ? null : intersect(featureCollection([update, nearPrevious])); if (target === null) { diff --git a/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts b/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts index ecf0349b..0262ae6d 100644 --- a/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts +++ b/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts @@ -344,8 +344,8 @@ describe('CacheDeletionJobCreator', () => { const { cacheDeletionJobCreator, jobManagerClientMock, mapproxyClientMock, readProductGeometryMock } = ctx; // a layer near 35E 32N with an update 0.05 degrees east of it. A zoom 12 tile is 0.044 wide (0.062 // across its diagonal) so its clip still reaches the update; a zoom 13 tile (0.031 across) cannot. - // A buffer of 127 degrees at zoom 1 would bend around the globe and miss this latitude altogether, - // which is why the coarse zooms are walked without a clip + // An arc-length buffer of 127 degrees at zoom 1 bends around the globe and misses this latitude + // altogether, which used to end the walk after zoom 0 const PREVIOUS_NORTH = square(34.9, 32, 0.2); const UPDATE = square(35.15, 32.05, 0.1); const LAST_BRIDGING_ZOOM = 12; From 3e3924a8075a2243705ed2f352ca62d766d38421 Mon Sep 17 00:00:00 2001 From: shlomiko Date: Sun, 18 Oct 2026 14:06:17 +0300 Subject: [PATCH 4/8] refactor: clip update tile ranges to the previous footprint's tile box (MAPCO-12035) --- .../ingestion/cacheDeletionJobCreator.ts | 96 ++++++------------- .../cacheDeletionJobCreator.spec.ts | 23 +++-- 2 files changed, 42 insertions(+), 77 deletions(-) diff --git a/src/job/models/ingestion/cacheDeletionJobCreator.ts b/src/job/models/ingestion/cacheDeletionJobCreator.ts index 77e691e9..688fa3c3 100644 --- a/src/job/models/ingestion/cacheDeletionJobCreator.ts +++ b/src/job/models/ingestion/cacheDeletionJobCreator.ts @@ -1,8 +1,8 @@ import type { Logger } from '@map-colonies/js-logger'; -import type { Feature, MultiPolygon, Polygon } from 'geojson'; -import { bbox, buffer, clone, coordEach, feature, featureCollection, intersect } from '@turf/turf'; -import { degreesPerTile, footprintToTileRanges } from '@map-colonies/mc-utils'; -import type { ITileRange } from '@map-colonies/mc-utils'; +import type { MultiPolygon, Polygon } from 'geojson'; +import { bbox } from '@turf/turf'; +import { bboxToTileRange, footprintToTileRanges } from '@map-colonies/mc-utils'; +import type { BBox2d, ITileRange } from '@map-colonies/mc-utils'; import type { ICreateJobBody, ICreateTaskBody } from '@map-colonies/mc-priority-queue'; import { OperationStatus, TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; import type { CacheDeletionJobParams, LayerName } from '@map-colonies/raster-shared'; @@ -33,50 +33,6 @@ interface CacheDeletionStrategy { * - swap-update: wipes entire prefix (layer redirects to new tiles path) * - update: deletes specific tile ranges over ingested footprint across all zooms */ -/** the globe is flat for practical purposes across this many degrees around the equator */ -const FLAT_EXTENT_DEGREES = 1; -/** - * keeps the buffer a superset of the grid distance: turf draws the round ends as polygons inscribed in the - * circle, which fall short of the radius by up to half a percent, and the flat patch has a little curvature - */ -const FLAT_BUFFER_MARGIN = 1.01; -const HALF = 2; - -const midpoint = (a: number, b: number): number => a + (b - a) / HALF; - -const mapCoordinates = (geometry: G, map: (x: number, y: number) => [number, number]): G => { - const mapped = clone(geometry); - coordEach(mapped, (coord) => { - const [x, y] = map(coord[0] as number, coord[1] as number); - coord[0] = x; - coord[1] = y; - }); - return mapped; -}; - -/** - * Buffers a footprint by a distance in grid degrees, the metric of the WorldCRS84 tile grid. - * - * turf measures a buffer along the globe, so a buffer of many degrees around a footprint away from the - * equator bends around the sphere and can even leave the footprint itself outside the result. The footprint - * is therefore moved to the equator and scaled down to a patch small enough to be flat, buffered there, and - * mapped back, which makes the buffer planar in degrees as the grid requires. - */ -const bufferInGridDegrees = (footprint: Feature, distance: number): Feature | undefined => { - const [minX, minY, maxX, maxY] = bbox(footprint); - const centerX = midpoint(minX, maxX); - const centerY = midpoint(minY, maxY); - // the flat patch has to hold the footprint plus the buffer on both sides - const scale = FLAT_EXTENT_DEGREES / (Math.max(maxX - minX, maxY - minY) + distance + distance); - - const flat = mapCoordinates(footprint.geometry, (x, y) => [(x - centerX) * scale, (y - centerY) * scale]); - const buffered = buffer(flat, distance * scale * FLAT_BUFFER_MARGIN, { units: 'degrees' }); - if (buffered === undefined) { - return undefined; - } - return feature(mapCoordinates(buffered.geometry, (x, y) => [x / scale + centerX, y / scale + centerY])); -}; - @injectable() export class CacheDeletionJobCreator { private readonly taskConfig: CacheDeletionTaskConfig; @@ -215,15 +171,14 @@ export class CacheDeletionJobCreator { } /** - * Walks only the tiles that can hold a cached tile of the replaced data. + * Keeps only the update tiles that can also hold a cached tile of the replaced data. * - * Only the previous footprint ever had tiles to cache, so a tile of the update area that does not also - * touch it was never cached and needs no deletion. A tile of side `s` that touches the previous footprint - * lies within `s * sqrt(2)` of it, so clipping the update area to a buffer of that reach, per zoom, gives a - * tight superset of the tiles touching both: at most a rim about two tiles wide along the previous - * boundary. Touching is monotone down the pyramid (a child touching both implies its parent does), so the - * first zoom with an empty clip ends the walk - for an update outside the previous footprint that happens - * at a low zoom, leaving just the coarse tiles both areas share. + * Only the previous footprint ever had tiles to cache, so an update tile that lies outside the tile box + * around it was never cached and needs no deletion. Each zoom clips the update ranges to that box with + * plain range arithmetic, padded by one tile so tiles that merely touch the previous footprint's edge are + * kept too. Sharing a tile is monotone down the pyramid (a child shared by both areas has a shared parent), + * so the first zoom with no shared tile ends the walk - for an update on new ground that happens at a low + * zoom, leaving just the coarse tiles both areas share. */ private *rangesTouchingPreviousFootprint( updateArea: Polygon | MultiPolygon, @@ -231,20 +186,31 @@ export class CacheDeletionJobCreator { maxZoom: number, logger: Logger ): Generator { - const update = feature(updateArea); - const previous = feature(previousFootprint); + const [minLon, minLat, maxLon, maxLat] = bbox(previousFootprint); + const previousBbox: BBox2d = [minLon, minLat, maxLon, maxLat]; for (let zoom = 0; zoom <= maxZoom; zoom++) { - const reach = degreesPerTile(zoom) * Math.SQRT2; - const nearPrevious = bufferInGridDegrees(previous, reach); - const target = nearPrevious === undefined ? null : intersect(featureCollection([update, nearPrevious])); + const previous = bboxToTileRange(previousBbox, zoom); + let shared = false; + + for (const range of footprintToTileRanges(updateArea, { minZoom: zoom, maxZoom: zoom })) { + const clipped: ITileRange = { + zoom, + minX: Math.max(range.minX, previous.minX - 1), + minY: Math.max(range.minY, previous.minY - 1), + maxX: Math.min(range.maxX, previous.maxX + 1), + maxY: Math.min(range.maxY, previous.maxY + 1), + }; + if (clipped.minX <= clipped.maxX && clipped.minY <= clipped.maxY) { + shared = true; + yield clipped; + } + } - if (target === null) { - logger.info({ msg: 'Update area stops touching the previous footprint, ending the range walk', lastZoom: zoom - 1, maxZoom }); + if (!shared) { + logger.info({ msg: 'Update area stops sharing tiles with the previous footprint, ending the range walk', lastZoom: zoom - 1, maxZoom }); return; } - - yield* footprintToTileRanges(target, { minZoom: zoom, maxZoom: zoom }); } } diff --git a/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts b/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts index 0262ae6d..b5ba2480 100644 --- a/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts +++ b/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts @@ -266,8 +266,9 @@ describe('CacheDeletionJobCreator', () => { it('should walk no deeper than the last zoom whose tiles can still span both areas when the update lies outside the previous footprint', async () => { const { cacheDeletionJobCreator, jobManagerClientMock, mapproxyClientMock, readProductGeometryMock } = ctx; const taskConfig = configMock.get('jobManagement.ingestion.tasks.cacheDeletion'); - // 1.5 degrees east of PREVIOUS. A zoom 7 tile is 1.41 degrees wide (1.99 across its diagonal), so the - // clip still reaches the update area; a zoom 8 tile is 0.70 wide (0.99 across) and cannot bridge the gap + // 1.5 degrees east of PREVIOUS. A zoom 7 tile is 1.41 degrees wide, so the tile box around PREVIOUS padded + // by one tile still reaches the update area's first column; at zoom 8 (0.70 wide) whole tiles of new + // ground separate the two const UPDATE = square(2.5, 0, 1); const LAST_BRIDGING_ZOOM = 7; @@ -324,28 +325,26 @@ describe('CacheDeletionJobCreator', () => { } } - // at the deepest zoom nothing starts farther from PREVIOUS than one tile diagonal: the new-only ground + // at the deepest zoom nothing starts farther from PREVIOUS than the one-tile pad: the new-only ground // beyond that was never cached and is left alone - const reach = degreesPerTile(MAX_ZOOM) * Math.SQRT2; + const pad = degreesPerTile(MAX_ZOOM); const deepest = ranges.filter((range) => range.zoom === MAX_ZOOM); expect(deepest.length).toBeGreaterThan(0); for (const range of deepest) { - expect(tileMinLon(MAX_ZOOM, range.maxX)).toBeLessThanOrEqual(NEW_GROUND_EDGE + reach); + expect(tileMinLon(MAX_ZOOM, range.maxX)).toBeLessThanOrEqual(NEW_GROUND_EDGE + pad); } - // whereas the update area itself does extend past that reach, so the full walk would have gone there + // whereas the update area itself does extend past the pad, so the full walk would have gone there const farEdgeTile = Math.floor((boundsOf(UPDATE).maxLon + 180) / degreesPerTile(MAX_ZOOM)); - expect(tileMinLon(MAX_ZOOM, farEdgeTile)).toBeGreaterThan(NEW_GROUND_EDGE + reach); + expect(tileMinLon(MAX_ZOOM, farEdgeTile)).toBeGreaterThan(NEW_GROUND_EDGE + pad); }); - it('should measure the reach in grid degrees for a layer far from the equator', async () => { + it('should clip in grid tiles, unaffected by latitude, for a layer far from the equator', async () => { const { cacheDeletionJobCreator, jobManagerClientMock, mapproxyClientMock, readProductGeometryMock } = ctx; - // a layer near 35E 32N with an update 0.05 degrees east of it. A zoom 12 tile is 0.044 wide (0.062 - // across its diagonal) so its clip still reaches the update; a zoom 13 tile (0.031 across) cannot. - // An arc-length buffer of 127 degrees at zoom 1 bends around the globe and misses this latitude - // altogether, which used to end the walk after zoom 0 + // a layer near 35E 32N with an update 0.05 degrees east of it. A zoom 12 tile is 0.044 wide, so the + // one-tile pad around the layer still reaches the update; a zoom 13 tile (0.022 wide) does not const PREVIOUS_NORTH = square(34.9, 32, 0.2); const UPDATE = square(35.15, 32.05, 0.1); const LAST_BRIDGING_ZOOM = 12; From 81cfe44e71b7e4b3d1b95b6b6096c983d925269b Mon Sep 17 00:00:00 2001 From: shlomiko Date: Sun, 18 Oct 2026 15:50:44 +0300 Subject: [PATCH 5/8] fix: drop clipped ranges that do not touch the previous footprint (MAPCO-12035) --- .../ingestion/cacheDeletionJobCreator.ts | 45 ++++++++++---- .../cacheDeletionJobCreator.spec.ts | 58 +++++++++++++++++-- 2 files changed, 88 insertions(+), 15 deletions(-) diff --git a/src/job/models/ingestion/cacheDeletionJobCreator.ts b/src/job/models/ingestion/cacheDeletionJobCreator.ts index 688fa3c3..1cf8b26b 100644 --- a/src/job/models/ingestion/cacheDeletionJobCreator.ts +++ b/src/job/models/ingestion/cacheDeletionJobCreator.ts @@ -1,7 +1,7 @@ import type { Logger } from '@map-colonies/js-logger'; -import type { MultiPolygon, Polygon } from 'geojson'; -import { bbox } from '@turf/turf'; -import { bboxToTileRange, footprintToTileRanges } from '@map-colonies/mc-utils'; +import type { Feature, MultiPolygon, Polygon } from 'geojson'; +import { bbox, bboxPolygon, booleanIntersects } from '@turf/turf'; +import { bboxToTileRange, degreesPerTile, footprintToTileRanges } from '@map-colonies/mc-utils'; import type { BBox2d, ITileRange } from '@map-colonies/mc-utils'; import type { ICreateJobBody, ICreateTaskBody } from '@map-colonies/mc-priority-queue'; import { OperationStatus, TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; @@ -33,6 +33,22 @@ interface CacheDeletionStrategy { * - swap-update: wipes entire prefix (layer redirects to new tiles path) * - update: deletes specific tile ranges over ingested footprint across all zooms */ +/** tiles kept around the previous footprint, so tiles that only touch its edge are deleted as well */ +const PAD_TILES = 1; +const GRID_MIN_LON = -180; +const GRID_MIN_LAT = -90; + +/** the ground a tile range covers, grown by `pad` tiles on every side */ +const tileRangeToPolygon = (range: ITileRange, pad: number): Feature => { + const side = degreesPerTile(range.zoom); + return bboxPolygon([ + GRID_MIN_LON + (range.minX - pad) * side, + GRID_MIN_LAT + (range.minY - pad) * side, + GRID_MIN_LON + (range.maxX + 1 + pad) * side, + GRID_MIN_LAT + (range.maxY + 1 + pad) * side, + ]); +}; + @injectable() export class CacheDeletionJobCreator { private readonly taskConfig: CacheDeletionTaskConfig; @@ -176,9 +192,12 @@ export class CacheDeletionJobCreator { * Only the previous footprint ever had tiles to cache, so an update tile that lies outside the tile box * around it was never cached and needs no deletion. Each zoom clips the update ranges to that box with * plain range arithmetic, padded by one tile so tiles that merely touch the previous footprint's edge are - * kept too. Sharing a tile is monotone down the pyramid (a child shared by both areas has a shared parent), - * so the first zoom with no shared tile ends the walk - for an update on new ground that happens at a low - * zoom, leaving just the coarse tiles both areas share. + * kept too. The box is the footprint's bbox, which for an irregular footprint also holds ground the layer + * never covered, so each clipped range is then checked against the footprint itself: a range that, grown + * by one tile, does not touch the previous footprint is dropped. Sharing a tile is monotone down the + * pyramid (a child shared by both areas has a shared parent), so the first zoom with no shared tile ends + * the walk - for an update on new ground that happens at a low zoom, leaving just the coarse tiles both + * areas share. */ private *rangesTouchingPreviousFootprint( updateArea: Polygon | MultiPolygon, @@ -196,12 +215,16 @@ export class CacheDeletionJobCreator { for (const range of footprintToTileRanges(updateArea, { minZoom: zoom, maxZoom: zoom })) { const clipped: ITileRange = { zoom, - minX: Math.max(range.minX, previous.minX - 1), - minY: Math.max(range.minY, previous.minY - 1), - maxX: Math.min(range.maxX, previous.maxX + 1), - maxY: Math.min(range.maxY, previous.maxY + 1), + minX: Math.max(range.minX, previous.minX - PAD_TILES), + minY: Math.max(range.minY, previous.minY - PAD_TILES), + maxX: Math.min(range.maxX, previous.maxX + PAD_TILES), + maxY: Math.min(range.maxY, previous.maxY + PAD_TILES), }; - if (clipped.minX <= clipped.maxX && clipped.minY <= clipped.maxY) { + if ( + clipped.minX <= clipped.maxX && + clipped.minY <= clipped.maxY && + booleanIntersects(tileRangeToPolygon(clipped, PAD_TILES), previousFootprint) + ) { shared = true; yield clipped; } diff --git a/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts b/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts index b5ba2480..6afaa032 100644 --- a/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts +++ b/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts @@ -196,14 +196,14 @@ describe('CacheDeletionJobCreator', () => { // axis-aligned squares: tile/footprint contact reduces to interval overlap, which keeps the exhaustive // oracle below trivial - const square = (minLon: number, minLat: number, size: number): Polygon => ({ + const square = (minLon: number, minLat: number, width: number, height = width): Polygon => ({ type: 'Polygon', coordinates: [ [ [minLon, minLat], - [minLon + size, minLat], - [minLon + size, minLat + size], - [minLon, minLat + size], + [minLon + width, minLat], + [minLon + width, minLat + height], + [minLon, minLat + height], [minLon, minLat], ], ], @@ -341,6 +341,56 @@ describe('CacheDeletionJobCreator', () => { expect(tileMinLon(MAX_ZOOM, farEdgeTile)).toBeGreaterThan(NEW_GROUND_EDGE + pad); }); + it('should stop early when the update sits inside the previous bbox but outside the previous footprint', async () => { + const { cacheDeletionJobCreator, jobManagerClientMock, mapproxyClientMock, readProductGeometryMock } = ctx; + const taskConfig = configMock.get('jobManagement.ingestion.tasks.cacheDeletion'); + // an L-shaped layer whose bbox is the unit square, with the update in the notch: the bbox clip alone would + // keep every update tile to max zoom, the footprint check drops them once a tile no longer reaches the L. + // The notch starts 0.35 away from both arms; a zoom 9 tile is 0.35 wide, so its one-tile pad still reaches, + // a zoom 10 tile (0.18) does not + const PREVIOUS_L: Polygon = { + type: 'Polygon', + coordinates: [ + [ + [0, 0], + [1, 0], + [1, 0.3], + [0.3, 0.3], + [0.3, 1], + [0, 1], + [0, 0], + ], + ], + }; + const UPDATE = square(0.65, 0.65, 0.2); + const LAST_REACHING_ZOOM = 9; + + mapproxyClientMock.getRedisCache.mockResolvedValue(redisCache('layer-Orthophoto')); + readProductGeometryMock.mockResolvedValue(UPDATE); + jobManagerClientMock.createJob.mockResolvedValue({ id: randomUUID(), taskIds: [randomUUID()] }); + jobManagerClientMock.createTaskForJob.mockResolvedValue(undefined); + + await cacheDeletionJobCreator.create({ layerName: 'layer-Orthophoto', ingestionJob: withPreviousFootprint(PREVIOUS_L) }); + + const ranges = emittedRanges(jobManagerClientMock); + const zooms = ranges.map((range) => range.zoom); + + expect(Math.min(...zooms)).toBe(0); + expect(Math.max(...zooms)).toBe(LAST_REACHING_ZOOM); + expect(LAST_REACHING_ZOOM).toBeLessThan(taskConfig.maxZoom); + + // the coarse tiles a square-based oracle cannot judge for an L are checked by brute force instead: + // every tile touching both the update and either arm of the L is covered + const ARMS = [square(0, 0, 1, 0.3), square(0, 0, 0.3, 1)]; + for (let zoom = 0; zoom <= LAST_REACHING_ZOOM; zoom++) { + for (const arm of ARMS) { + for (const tile of tilesTouchingBoth(UPDATE, arm, zoom)) { + expect(covers(ranges, tile)).toBe(true); + } + } + } + }); + it('should clip in grid tiles, unaffected by latitude, for a layer far from the equator', async () => { const { cacheDeletionJobCreator, jobManagerClientMock, mapproxyClientMock, readProductGeometryMock } = ctx; // a layer near 35E 32N with an update 0.05 degrees east of it. A zoom 12 tile is 0.044 wide, so the From 19879f7fb5b06adb50c6f4c8f5e5ece9dd5eb6f7 Mon Sep 17 00:00:00 2001 From: shlomiko Date: Sun, 18 Oct 2026 16:33:21 +0300 Subject: [PATCH 6/8] refactor: clip to the tiles touching the previous bbox, without a pad ring (MAPCO-12035) --- .../ingestion/cacheDeletionJobCreator.ts | 63 +++++++++++-------- .../cacheDeletionJobCreator.spec.ts | 30 +++++---- 2 files changed, 50 insertions(+), 43 deletions(-) diff --git a/src/job/models/ingestion/cacheDeletionJobCreator.ts b/src/job/models/ingestion/cacheDeletionJobCreator.ts index 1cf8b26b..57eebff4 100644 --- a/src/job/models/ingestion/cacheDeletionJobCreator.ts +++ b/src/job/models/ingestion/cacheDeletionJobCreator.ts @@ -1,7 +1,7 @@ import type { Logger } from '@map-colonies/js-logger'; import type { Feature, MultiPolygon, Polygon } from 'geojson'; import { bbox, bboxPolygon, booleanIntersects } from '@turf/turf'; -import { bboxToTileRange, degreesPerTile, footprintToTileRanges } from '@map-colonies/mc-utils'; +import { degreesPerTile, footprintToTileRanges } from '@map-colonies/mc-utils'; import type { BBox2d, ITileRange } from '@map-colonies/mc-utils'; import type { ICreateJobBody, ICreateTaskBody } from '@map-colonies/mc-priority-queue'; import { OperationStatus, TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; @@ -33,19 +33,34 @@ interface CacheDeletionStrategy { * - swap-update: wipes entire prefix (layer redirects to new tiles path) * - update: deletes specific tile ranges over ingested footprint across all zooms */ -/** tiles kept around the previous footprint, so tiles that only touch its edge are deleted as well */ -const PAD_TILES = 1; const GRID_MIN_LON = -180; const GRID_MIN_LAT = -90; -/** the ground a tile range covers, grown by `pad` tiles on every side */ -const tileRangeToPolygon = (range: ITileRange, pad: number): Feature => { +/** + * The tiles whose squares touch the bbox, bounds included: an edge lying exactly on a grid line also claims the + * tile beyond it on either side (ceil - 1 and floor agree everywhere else), as footprintToTileRanges does for + * footprints. Indices may fall outside the grid at the world's edge; they are only ever intersected with ranges + * that lie within it + */ +const tilesTouchingBbox = ([minLon, minLat, maxLon, maxLat]: BBox2d, zoom: number): ITileRange => { + const side = degreesPerTile(zoom); + return { + zoom, + minX: Math.ceil((minLon - GRID_MIN_LON) / side) - 1, + minY: Math.ceil((minLat - GRID_MIN_LAT) / side) - 1, + maxX: Math.floor((maxLon - GRID_MIN_LON) / side), + maxY: Math.floor((maxLat - GRID_MIN_LAT) / side), + }; +}; + +/** the ground a tile range covers */ +const tileRangeToPolygon = (range: ITileRange): Feature => { const side = degreesPerTile(range.zoom); return bboxPolygon([ - GRID_MIN_LON + (range.minX - pad) * side, - GRID_MIN_LAT + (range.minY - pad) * side, - GRID_MIN_LON + (range.maxX + 1 + pad) * side, - GRID_MIN_LAT + (range.maxY + 1 + pad) * side, + GRID_MIN_LON + range.minX * side, + GRID_MIN_LAT + range.minY * side, + GRID_MIN_LON + (range.maxX + 1) * side, + GRID_MIN_LAT + (range.maxY + 1) * side, ]); }; @@ -190,14 +205,12 @@ export class CacheDeletionJobCreator { * Keeps only the update tiles that can also hold a cached tile of the replaced data. * * Only the previous footprint ever had tiles to cache, so an update tile that lies outside the tile box - * around it was never cached and needs no deletion. Each zoom clips the update ranges to that box with - * plain range arithmetic, padded by one tile so tiles that merely touch the previous footprint's edge are - * kept too. The box is the footprint's bbox, which for an irregular footprint also holds ground the layer - * never covered, so each clipped range is then checked against the footprint itself: a range that, grown - * by one tile, does not touch the previous footprint is dropped. Sharing a tile is monotone down the - * pyramid (a child shared by both areas has a shared parent), so the first zoom with no shared tile ends - * the walk - for an update on new ground that happens at a low zoom, leaving just the coarse tiles both - * areas share. + * around it was never cached and needs no deletion. Each zoom clips the update ranges to the tiles touching + * the previous bbox with plain range arithmetic. The bbox of an irregular footprint also holds ground the + * layer never covered, so each clipped range is then checked against the footprint itself and dropped when + * its ground does not touch it. Sharing a tile is monotone down the pyramid (a child shared by both areas + * has a shared parent), so the first zoom with no shared tile ends the walk - for an update on new ground + * that happens at a low zoom, leaving just the coarse tiles both areas share. */ private *rangesTouchingPreviousFootprint( updateArea: Polygon | MultiPolygon, @@ -209,22 +222,18 @@ export class CacheDeletionJobCreator { const previousBbox: BBox2d = [minLon, minLat, maxLon, maxLat]; for (let zoom = 0; zoom <= maxZoom; zoom++) { - const previous = bboxToTileRange(previousBbox, zoom); + const previous = tilesTouchingBbox(previousBbox, zoom); let shared = false; for (const range of footprintToTileRanges(updateArea, { minZoom: zoom, maxZoom: zoom })) { const clipped: ITileRange = { zoom, - minX: Math.max(range.minX, previous.minX - PAD_TILES), - minY: Math.max(range.minY, previous.minY - PAD_TILES), - maxX: Math.min(range.maxX, previous.maxX + PAD_TILES), - maxY: Math.min(range.maxY, previous.maxY + PAD_TILES), + minX: Math.max(range.minX, previous.minX), + minY: Math.max(range.minY, previous.minY), + maxX: Math.min(range.maxX, previous.maxX), + maxY: Math.min(range.maxY, previous.maxY), }; - if ( - clipped.minX <= clipped.maxX && - clipped.minY <= clipped.maxY && - booleanIntersects(tileRangeToPolygon(clipped, PAD_TILES), previousFootprint) - ) { + if (clipped.minX <= clipped.maxX && clipped.minY <= clipped.maxY && booleanIntersects(tileRangeToPolygon(clipped), previousFootprint)) { shared = true; yield clipped; } diff --git a/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts b/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts index 6afaa032..3afa34c5 100644 --- a/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts +++ b/tests/unit/job/cacheDeletionJobCreator/cacheDeletionJobCreator.spec.ts @@ -266,11 +266,10 @@ describe('CacheDeletionJobCreator', () => { it('should walk no deeper than the last zoom whose tiles can still span both areas when the update lies outside the previous footprint', async () => { const { cacheDeletionJobCreator, jobManagerClientMock, mapproxyClientMock, readProductGeometryMock } = ctx; const taskConfig = configMock.get('jobManagement.ingestion.tasks.cacheDeletion'); - // 1.5 degrees east of PREVIOUS. A zoom 7 tile is 1.41 degrees wide, so the tile box around PREVIOUS padded - // by one tile still reaches the update area's first column; at zoom 8 (0.70 wide) whole tiles of new - // ground separate the two + // 1.5 degrees east of PREVIOUS. A zoom 6 tile is 2.81 degrees wide and can span the gap; a zoom 7 tile is + // 1.41 wide and cannot, so no tile touches both areas from there on const UPDATE = square(2.5, 0, 1); - const LAST_BRIDGING_ZOOM = 7; + const LAST_BRIDGING_ZOOM = 6; mapproxyClientMock.getRedisCache.mockResolvedValue(redisCache('layer-Orthophoto')); readProductGeometryMock.mockResolvedValue(UPDATE); @@ -325,29 +324,28 @@ describe('CacheDeletionJobCreator', () => { } } - // at the deepest zoom nothing starts farther from PREVIOUS than the one-tile pad: the new-only ground + // at the deepest zoom nothing starts east of the tile holding PREVIOUS's east edge: the new-only ground // beyond that was never cached and is left alone - const pad = degreesPerTile(MAX_ZOOM); const deepest = ranges.filter((range) => range.zoom === MAX_ZOOM); expect(deepest.length).toBeGreaterThan(0); for (const range of deepest) { - expect(tileMinLon(MAX_ZOOM, range.maxX)).toBeLessThanOrEqual(NEW_GROUND_EDGE + pad); + expect(tileMinLon(MAX_ZOOM, range.maxX)).toBeLessThanOrEqual(NEW_GROUND_EDGE); } - // whereas the update area itself does extend past the pad, so the full walk would have gone there + // whereas the update area itself does extend past that tile, so the full walk would have gone there const farEdgeTile = Math.floor((boundsOf(UPDATE).maxLon + 180) / degreesPerTile(MAX_ZOOM)); - expect(tileMinLon(MAX_ZOOM, farEdgeTile)).toBeGreaterThan(NEW_GROUND_EDGE + pad); + expect(tileMinLon(MAX_ZOOM, farEdgeTile)).toBeGreaterThan(NEW_GROUND_EDGE); }); it('should stop early when the update sits inside the previous bbox but outside the previous footprint', async () => { const { cacheDeletionJobCreator, jobManagerClientMock, mapproxyClientMock, readProductGeometryMock } = ctx; const taskConfig = configMock.get('jobManagement.ingestion.tasks.cacheDeletion'); // an L-shaped layer whose bbox is the unit square, with the update in the notch: the bbox clip alone would - // keep every update tile to max zoom, the footprint check drops them once a tile no longer reaches the L. - // The notch starts 0.35 away from both arms; a zoom 9 tile is 0.35 wide, so its one-tile pad still reaches, - // a zoom 10 tile (0.18) does not + // keep every update tile to max zoom, the footprint check drops them once no tile touches both. The notch + // starts 0.35 away from both arms; a zoom 8 tile is 0.70 wide and spans the gap, a zoom 9 tile (0.35 wide, + // grid-aligned at 0.35) stops just short of the arms const PREVIOUS_L: Polygon = { type: 'Polygon', coordinates: [ @@ -363,7 +361,7 @@ describe('CacheDeletionJobCreator', () => { ], }; const UPDATE = square(0.65, 0.65, 0.2); - const LAST_REACHING_ZOOM = 9; + const LAST_REACHING_ZOOM = 8; mapproxyClientMock.getRedisCache.mockResolvedValue(redisCache('layer-Orthophoto')); readProductGeometryMock.mockResolvedValue(UPDATE); @@ -393,11 +391,11 @@ describe('CacheDeletionJobCreator', () => { it('should clip in grid tiles, unaffected by latitude, for a layer far from the equator', async () => { const { cacheDeletionJobCreator, jobManagerClientMock, mapproxyClientMock, readProductGeometryMock } = ctx; - // a layer near 35E 32N with an update 0.05 degrees east of it. A zoom 12 tile is 0.044 wide, so the - // one-tile pad around the layer still reaches the update; a zoom 13 tile (0.022 wide) does not + // a layer near 35E 32N with an update 0.05 degrees east of it. A zoom 11 tile is 0.088 wide and spans + // the gap; a zoom 12 tile (0.044 wide) does not const PREVIOUS_NORTH = square(34.9, 32, 0.2); const UPDATE = square(35.15, 32.05, 0.1); - const LAST_BRIDGING_ZOOM = 12; + const LAST_BRIDGING_ZOOM = 11; mapproxyClientMock.getRedisCache.mockResolvedValue(redisCache('layer-Orthophoto')); readProductGeometryMock.mockResolvedValue(UPDATE); From 4838942f8b90766d9b6fcf7d1cea8eb1a08caf10 Mon Sep 17 00:00:00 2001 From: shlomiko Date: Mon, 19 Oct 2026 17:20:48 +0300 Subject: [PATCH 7/8] docs: shorten rangesTouchingPreviousFootprint comment --- src/job/models/ingestion/cacheDeletionJobCreator.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/job/models/ingestion/cacheDeletionJobCreator.ts b/src/job/models/ingestion/cacheDeletionJobCreator.ts index 57eebff4..9fc183db 100644 --- a/src/job/models/ingestion/cacheDeletionJobCreator.ts +++ b/src/job/models/ingestion/cacheDeletionJobCreator.ts @@ -202,15 +202,8 @@ export class CacheDeletionJobCreator { } /** - * Keeps only the update tiles that can also hold a cached tile of the replaced data. - * - * Only the previous footprint ever had tiles to cache, so an update tile that lies outside the tile box - * around it was never cached and needs no deletion. Each zoom clips the update ranges to the tiles touching - * the previous bbox with plain range arithmetic. The bbox of an irregular footprint also holds ground the - * layer never covered, so each clipped range is then checked against the footprint itself and dropped when - * its ground does not touch it. Sharing a tile is monotone down the pyramid (a child shared by both areas - * has a shared parent), so the first zoom with no shared tile ends the walk - for an update on new ground - * that happens at a low zoom, leaving just the coarse tiles both areas share. + * Per zoom, clips the update ranges to the tile box of the previous bbox and keeps those whose ground touches the + * previous footprint; stops at the first zoom with no shared tile, since deeper zooms cannot share any either. */ private *rangesTouchingPreviousFootprint( updateArea: Polygon | MultiPolygon, From 7ea6969d489c251ef0b86f6896b70aa1bd92a787 Mon Sep 17 00:00:00 2001 From: shlomiko Date: Mon, 19 Oct 2026 17:21:51 +0300 Subject: [PATCH 8/8] docs: shorten recordPreviousFootprint comment --- src/job/models/ingestion/updateJobHandler.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/job/models/ingestion/updateJobHandler.ts b/src/job/models/ingestion/updateJobHandler.ts index 73e9b477..e8033d0a 100644 --- a/src/job/models/ingestion/updateJobHandler.ts +++ b/src/job/models/ingestion/updateJobHandler.ts @@ -138,10 +138,8 @@ export class UpdateJobHandler } /** - * Records the layer's catalog footprint, as it stands before this update lands, on the job's additional - * params. The cache deletion job created at finalize uses it to leave out the part of the update area that - * never had tiles to cache, and by then the catalog already holds the merged footprint. Best effort: without - * it the cache deletion covers the whole update area, so a failure here must not fail the ingestion. + * Saves the catalog footprint from before this update on the job, so the cache deletion at finalize can limit itself + * to the previously covered ground. Best effort: without it the cache deletion covers the whole update area. */ private async recordPreviousFootprint(job: IngestionUpdateCreateTasksJob, catalogId: string, logger: Logger): Promise { const activeSpan = trace.getActiveSpan();