build-tiles.mjs

#!/usr/bin/env node
/**
 * Builds public/tiles/iugnorge.pmtiles — everything the group put on the map,
 * and nothing else.
 *
 * The archive holds only features whose current version belongs to one of our
 * #iugnorge changesets, plus the Tasking Manager task squares that contain at
 * least one of them. Surrounding context comes from an ordinary basemap at
 * runtime, so this stays small enough to live in the repository and the app
 * never has to query Overpass.
 *
 * Layers: buildings, roads, waterways, tasks.
 *
 * Attribution is by the changeset that last touched a way, which is exact for
 * anything nobody has edited since and an undercount otherwise — a building we
 * drew and someone else later squared off now belongs to their changeset. It
 * undercounts; it never claims someone else's work.
 *
 * Runs in CI: a full build makes a few hundred Overpass queries, and Overpass
 * firewalls clients that do that from a shared address.
 *
 * Extracted features are cached per project under extracts/, so a rebuild after
 * a mapathon only re-queries what changed and re-tiles the rest from disk —
 * minutes instead of an hour. A project is re-queried when --only names it, when
 * it has no cached extract, or when its changeset count has moved since the
 * cache was written. The archive is always rebuilt from every cached project, so
 * --only narrows the querying without narrowing the output.
 *
 * Usage:
 *   node scripts/build-tiles.mjs                    # everything, cache-aware
 *   node scripts/build-tiles.mjs --only 49638,63366 # re-query just these
 */

import { readFile, writeFile, mkdir, rm } from 'node:fs/promises'
import { dirname, resolve, join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { spawn } from 'node:child_process'

const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..')
/** The full project store, with the changeset ids the app never sees. */
const DATA_FILE = resolve(ROOT, 'data/projects.json')
/** What the app needs to know about the archive: counts and extents. */
const SUMMARY_FILE = resolve(ROOT, 'public/data/tiles.json')

/**
 * The build only ever asks for the current state — no historical `[date:]`
 * queries — so it is not tied to overpass-api.de the way the browser once was,
 * and CORS is irrelevant server-side. The French instance is minutely current,
 * answers these queries in about three seconds, and does not throttle us; the
 * main instance is the fallback, where the same query took twenty-two seconds
 * once queueing is counted. Tried in order, sticking with whichever answers.
 */
const ENDPOINTS = [
  'https://overpass.openstreetmap.fr/api/interpreter',
  'https://overpass-api.de/api/interpreter',
]
let preferred = 0
const host = (url) => new URL(url).host
const TM_API = 'https://tasking-manager-production-api.hotosm.org/api/v2'
const UA = 'missingmaps-iug/1.0 (+https://github.com/MasterMaps/missingmaps)'

/**
 * Query areas follow the edits, not the project boundary. Tasking Manager areas
 * of interest can span whole border regions — one of ours needs 1248 quarter-
 * degree tiles to cover, to find edits that sit in two of them. Binning the
 * changeset locations instead takes the whole run from 1784 queries to 518.
 */
const CELL_DEG = 0.05
/** Changeset centres are points; this covers the area around each one. */
const MARGIN_DEG = 0.03
/** Fallback only, for a project with no recorded changeset locations. */
const MAX_SPAN_DEG = 0.25
/**
 * Give up on a box after this many refusals. Generous, because a skipped
 * project leaves a hole in the archive until someone notices — and a run that
 * takes longer is cheaper than a rebuild that has to happen twice.
 */
const MAX_ATTEMPTS = 14

const args = process.argv.slice(2)
const flag = (name, fallback) => {
  const i = args.indexOf(`--${name}`)
  return i === -1 ? fallback : args[i + 1]
}
const OUT_DIR = resolve(ROOT, flag('outdir', 'public/tiles'))
const WORK_DIR = resolve(ROOT, '.tiles-work')
/**
 * One file of extracted features per project, kept in the repository so a
 * rebuild after a mapathon only has to re-query the project that changed.
 * Not under public/, so it is never published.
 */
const EXTRACT_DIR = resolve(ROOT, 'extracts')
const only = (flag('only', '') || '').split(',').map((s) => s.trim()).filter(Boolean).map(Number)

const sleep = (ms) => new Promise((r) => setTimeout(r, ms))

/* --------------------------------------------------------------- overpass */

const query = (bbox) => {
  const [w, s, e, n] = bbox
  const box = `${s},${w},${n},${e}`
  // Only what humanitarian mapping actually produces. Landuse and natural
  // features are left to the basemap.
  return `[out:json][timeout:600];
(
  way["building"](${box});
  way["highway"](${box});
  way["waterway"](${box});
);
out meta geom;`
}

/**
 * How long to wait for a query slot, straight from Overpass.
 *
 * Guessing is expensive: a blind 30/60/90s ladder spent two thirds of an
 * earlier run asleep. Overpass publishes exactly when the next slot frees, and
 * it is usually seconds away.
 */
async function slotWait(endpoint, round = 0) {
  // A reported free slot is not a reserved one. overpass-api.de load-balances
  // across backends and CI runners share an address with other tenants, so the
  // slot is often gone by the time we ask for it. Retrying two seconds later
  // just burns an attempt, hence a floor that grows with each refusal.
  const floor = Math.min(5000 * (round + 1), 90_000)
  let reported = 15_000
  try {
    const res = await fetch(endpoint.replace('/interpreter', '/status'), {
      headers: { 'User-Agent': UA },
    })
    const text = await res.text()
    if (/slots? available now/i.test(text)) reported = 0
    else {
      const seconds = [...text.matchAll(/in (-?\d+) seconds/g)].map((m) => Math.max(Number(m[1]), 0))
      if (seconds.length) reported = Math.min(Math.max(...seconds) + 1, 180) * 1000
    }
  } catch {
    // keep the default
  }
  return Math.max(reported, floor)
}

const RETRYABLE = [429, 500, 502, 503, 504]

/**
 * Endpoints whose data stopped updating, skipped for the rest of the run.
 *
 * A stale instance does not fail either. In November 2026 the French instance
 * sat frozen at 23 October for weeks, answering every query promptly with HTTP
 * 200 and the map as it was before our mapathons — so every project mapped
 * since came out empty and fell off the app without a single error logged.
 */
const stale = new Set()

/** `notBefore`: the newest edit the answer must already include. */
async function overpass(data, label, notBefore) {
  let last = 'unknown'
  for (let round = 0; round <= MAX_ATTEMPTS; round++) {
    // Rounds are over the live endpoints only. Counting a stale one as an
    // attempt once let the loop hammer the other without ever pausing, until
    // overpass-api.de stopped accepting connections from the runner.
    const live = ENDPOINTS.filter((e) => !stale.has(e))
    // Whichever answered last goes first.
    live.sort((a, b) => (b === ENDPOINTS[preferred]) - (a === ENDPOINTS[preferred]))
    for (const endpoint of live) {
      try {
        const res = await fetch(endpoint, {
          method: 'POST',
          headers: { 'User-Agent': UA },
          body: new URLSearchParams({ data }),
        })
        if (res.ok) {
          const body = await res.json()
          const base = body.osm3s?.timestamp_osm_base
          if (notBefore && base && new Date(base) < new Date(notBefore)) {
            console.log(`    ${host(endpoint)} has data only up to ${base}, older than ${notBefore}; not using it`)
            stale.add(endpoint)
            continue
          }
          preferred = ENDPOINTS.indexOf(endpoint)
          return body
        }
        if (!RETRYABLE.includes(res.status)) throw new Error(`HTTP ${res.status} from ${host(endpoint)}`)
        last = `HTTP ${res.status} from ${host(endpoint)}`
      } catch (err) {
        if (!/HTTP \d+/.test(err.message)) last = `${host(endpoint)}: ${err.message}`
        else throw new Error(`${label}: ${err.message}`)
      }
    }
    if (stale.size === ENDPOINTS.length) throw new Error(`${label}: every Overpass instance is behind ${notBefore}`)

    // Only sleep once every endpoint has refused this round.
    const endpoint = ENDPOINTS.find((e) => !stale.has(e))
    const wait = await slotWait(endpoint, round)
    console.log(`    no slot (${last}), waiting ${Math.round(wait / 1000)}s`)
    await sleep(wait)
  }
  throw new Error(`${label}: ${last}`)
}

/**
 * Every way in one box — and all of them, not just as many as Overpass felt
 * like returning.
 *
 * An oversized query does not fail. It comes back HTTP 200 with a partial
 * element list and a `remark` saying it gave up, and accepting that silently is
 * how an earlier build undercounted by a factor of three while looking healthy.
 * So a remark means the box was too big: quarter it and ask again.
 */
async function fetchArea(bbox, label, notBefore, depth = 0) {
  const body = await overpass(query(bbox), label, notBefore)
  if (!body.remark) return body.elements || []
  if (depth >= 3) throw new Error(`${label}: ${body.remark}`)

  console.log(`    partial result, splitting: ${body.remark.slice(0, 70)}`)
  const [w, s, e, n] = bbox
  const midX = (w + e) / 2
  const midY = (s + n) / 2
  const elements = []
  for (const quarter of [
    [w, s, midX, midY],
    [midX, s, e, midY],
    [w, midY, midX, n],
    [midX, midY, e, n],
  ]) {
    elements.push(...(await fetchArea(quarter, label, notBefore, depth + 1)))
    await sleep(1000)
  }
  return elements
}

/**
 * The boxes to ask Overpass about: one per populated cell of the grid the
 * group's changesets fall into, grown by a margin so features near a cell edge
 * are not clipped off.
 */
function queryAreas(project) {
  const points = project.editPoints ?? []
  if (!points.length) return splitBbox(project.bbox ?? project.editBbox)

  // Same reason as the stray filter: a changeset that wandered leaves an edit
  // point far outside the project, and querying around it is wasted work.
  const aoi = project.bbox ? grow(project.bbox, AOI_MARGIN_DEG) : null
  const cells = new Map()
  for (const [lon, lat] of points) {
    if (aoi && !covers(aoi, [lon, lat])) continue
    const x = Math.floor(lon / CELL_DEG)
    const y = Math.floor(lat / CELL_DEG)
    const key = `${x}/${y}`
    if (cells.has(key)) continue
    cells.set(key, [
      x * CELL_DEG - MARGIN_DEG,
      y * CELL_DEG - MARGIN_DEG,
      (x + 1) * CELL_DEG + MARGIN_DEG,
      (y + 1) * CELL_DEG + MARGIN_DEG,
    ])
  }
  // If the area of interest disagrees with every recorded edit location, trust
  // the edits: an empty query list would silently drop the whole project.
  return cells.size ? [...cells.values()] : queryAreas({ ...project, bbox: null })
}

/** Splits a bbox into pieces no larger than MAX_SPAN_DEG on a side. */
function splitBbox([w, s, e, n]) {
  const cols = Math.max(1, Math.ceil((e - w) / MAX_SPAN_DEG))
  const rows = Math.max(1, Math.ceil((n - s) / MAX_SPAN_DEG))
  const out = []
  for (let x = 0; x < cols; x++) {
    for (let y = 0; y < rows; y++) {
      out.push([
        w + ((e - w) * x) / cols,
        s + ((n - s) * y) / rows,
        w + ((e - w) * (x + 1)) / cols,
        s + ((n - s) * (y + 1)) / rows,
      ])
    }
  }
  return out
}

/* -------------------------------------------------------------- geometry */

const round = (v) => Math.round(v * 1e6) / 1e6

/** Turns one of our ways into a GeoJSON feature, or null if it is not drawable. */
function toFeature(el, project) {
  const tags = el.tags ?? {}
  const coords = el.geometry.map(({ lon, lat }) => [round(lon), round(lat)])
  const closed =
    coords.length > 3 &&
    coords[0][0] === coords[coords.length - 1][0] &&
    coords[0][1] === coords[coords.length - 1][1]

  const properties = { project, year: Number((el.timestamp || '').slice(0, 4)) || null }

  if (tags.building && tags.building !== 'no' && closed) {
    return ['buildings', properties, { type: 'Polygon', coordinates: [coords] }]
  }
  if (tags.highway) {
    return ['roads', { ...properties, kind: tags.highway }, { type: 'LineString', coordinates: coords }]
  }
  if (tags.waterway) {
    return ['waterways', { ...properties, kind: tags.waterway }, { type: 'LineString', coordinates: coords }]
  }
  return null
}

/**
 * A Tasking Manager project is its area of interest, so anything outside it is
 * not that project's work — however the changeset ids say otherwise.
 *
 * One changeset in the Accra project also touched features in Bangladesh, which
 * put 1 of 9972 coordinates 10000 km from the rest, stretched the project extent
 * across two continents and left its globe dot sitting in Yemen. Others are
 * worse: a Bangladesh project whose features are in Peru, a Ukrainian one whose
 * features are in Nigeria — mappers who kept a stale #hotosm-project hashtag
 * while mapping somewhere else. The hashtag is wrong; the area of interest is
 * right.
 *
 * The margin is 3 km because work legitimately spills past a task boundary —
 * one project's features sit 2 km outside a small area of interest. Nothing
 * mis-attributed lands within 1000 km, so the tolerance costs nothing.
 */
const AOI_MARGIN_DEG = 0.03

const grow = ([w, s, e, n], by) => [w - by, s - by, e + by, n + by]

const covers = ([w, s, e, n], [lon, lat]) => lon >= w && lon <= e && lat >= s && lat <= n

const centroid = (coords) => {
  let x = 0
  let y = 0
  for (const [lon, lat] of coords) {
    x += lon
    y += lat
  }
  return [x / coords.length, y / coords.length]
}

const centroidOf = (geometry) =>
  centroid(geometry.type === 'Polygon' ? geometry.coordinates[0] : geometry.coordinates)

const bboxOf = (geometry) => {
  const lons = []
  const lats = []
  const walk = (c) => (typeof c[0] === 'number' ? (lons.push(c[0]), lats.push(c[1])) : c.forEach(walk))
  walk(geometry.coordinates)
  return [Math.min(...lons), Math.min(...lats), Math.max(...lons), Math.max(...lats)]
}

/**
 * The task squares the group actually worked in. A square counts only if one of
 * our features falls inside it — a square someone merely marked done does not.
 */
async function ourTaskSquares(projectId, points) {
  let body
  try {
    const res = await fetch(`${TM_API}/projects/${projectId}/tasks/`, { headers: { 'User-Agent': UA } })
    if (!res.ok) return []
    body = await res.json()
  } catch {
    return []
  }

  const squares = []
  for (const feature of body.features || []) {
    if (!feature.geometry) continue
    const [w, s, e, n] = bboxOf(feature.geometry)
    let edits = 0
    for (const [lon, lat] of points) {
      if (lon >= w && lon <= e && lat >= s && lat <= n) edits++
    }
    if (!edits) continue
    squares.push({
      type: 'Feature',
      properties: { project: projectId, task: feature.properties?.taskId ?? null, edits },
      geometry: feature.geometry,
    })
  }
  return squares
}

/* ------------------------------------------------------------------ main */

const raw = JSON.parse(await readFile(DATA_FILE, 'utf8'))
const projects = raw.projects.filter((p) => (p.bbox || p.editBbox) && p.changesets?.length)

await rm(WORK_DIR, { recursive: true, force: true })
await mkdir(WORK_DIR, { recursive: true })
await mkdir(EXTRACT_DIR, { recursive: true })

const layers = ['buildings', 'roads', 'waterways', 'tasks']
const streams = Object.fromEntries(layers.map((l) => [l, []]))
const perProject = {}

/** Re-query a project only when asked to, or when we have never extracted it. */
async function extract(project, label) {
  const ourChangesets = new Set(project.changesets)
  const pieces = queryAreas(project)
  // Keyed by way id: the margin makes neighbouring boxes overlap by more than
  // the stride between them, so most ways come back from several of them.
  const ours = new Map()

  for (const [i, piece] of pieces.entries()) {
    const suffix = pieces.length > 1 ? ` [${i + 1}/${pieces.length}]` : ''
    const started = Date.now()
    const elements = await fetchArea(piece, `${label}${suffix}`, project.lastEdit)
    let mine = 0
    for (const el of elements) {
      if (el.type !== 'way' || !el.geometry || el.geometry.length < 2) continue
      if (!ourChangesets.has(el.changeset)) continue
      ours.set(el.id, el)
      mine++
    }
    // A successful query is otherwise silent, and a long run of them looks
    // exactly like a hung one.
    const seconds = Math.round((Date.now() - started) / 1000)
    console.log(`    [${i + 1}/${pieces.length}] ${elements.length} ways, ${mine} ours, ${seconds}s`)
    await sleep(500) // one query at a time, with a pause, is a polite client.
  }

  const features = Object.fromEntries(layers.map((l) => [l, []]))
  const points = []
  for (const el of ours.values()) {
    const converted = toFeature(el, project.id)
    if (!converted) continue
    const [layer, properties, geometry] = converted
    features[layer].push({ type: 'Feature', properties, geometry })
    points.push(centroid(geometry.type === 'Polygon' ? geometry.coordinates[0] : geometry.coordinates))
  }
  features.tasks = await ourTaskSquares(project.id, points)

  return { id: project.id, changesets: project.changesets.length, features }
}

const extractPath = (id) => join(EXTRACT_DIR, `${id}.json`)

for (const [index, project] of projects.entries()) {
  const label = `#${project.id} ${(project.name || '').slice(0, 40)}`
  const path = extractPath(project.id)

  let cached = null
  if (!only.includes(project.id)) {
    try {
      cached = JSON.parse(await readFile(path, 'utf8'))
      // A project that has gained changesets since we last looked needs redoing.
      if (cached.changesets !== project.changesets.length) cached = null
    } catch {
      cached = null
    }
  }

  let extracted = cached
  if (extracted) {
    console.log(`[${index + 1}/${projects.length}] ${label} — cached`)
  } else {
    console.log(`[${index + 1}/${projects.length}] ${label}`)
    try {
      extracted = await extract(project, label)
      await writeFile(path, JSON.stringify(extracted))
    } catch (err) {
      console.warn(`  failed: ${err.message}`)
      process.exitCode = 1
      // Out of date beats absent: dropping the project would take its work off
      // the live map until the next good run.
      extracted = await readFile(path, 'utf8').then(JSON.parse, () => null)
      if (!extracted) continue
      console.warn('  keeping the previous extract')
    }
  }

  let kept = 0
  let strays = 0
  // Applied here rather than during extraction so that already-cached extracts
  // are cleaned up too, without re-querying anything.
  const aoi = project.bbox ? grow(project.bbox, AOI_MARGIN_DEG) : null
  // Extent of the group's own work, so the app can frame the whole of it
  // rather than guessing from changeset centres.
  let bounds = null
  const stretch = (coords) => {
    if (typeof coords[0] === 'number') {
      const [lon, lat] = coords
      bounds = bounds
        ? [Math.min(bounds[0], lon), Math.min(bounds[1], lat), Math.max(bounds[2], lon), Math.max(bounds[3], lat)]
        : [lon, lat, lon, lat]
    } else coords.forEach(stretch)
  }

  for (const layer of layers) {
    for (const feature of extracted.features[layer] ?? []) {
      if (layer !== 'tasks' && aoi && !covers(aoi, centroidOf(feature.geometry))) {
        strays++
        continue
      }
      streams[layer].push(JSON.stringify(feature))
      if (layer !== 'tasks') {
        kept++
        stretch(feature.geometry.coordinates)
      }
    }
  }
  const squares = extracted.features.tasks?.length ?? 0
  // A project with no features of its own still has squares worth framing.
  if (!bounds) for (const sq of extracted.features.tasks ?? []) stretch(sq.geometry.coordinates)

  perProject[project.id] = { features: kept, squares, bbox: bounds }
  if (!cached) console.log(`  ${kept} features by us, ${squares} task squares`)
  if (strays) console.log(`  dropped ${strays} feature(s) outside the project area`)
}

const totals = Object.values(perProject).reduce(
  (acc, p) => ({ features: acc.features + p.features, squares: acc.squares + p.squares }),
  { features: 0, squares: 0 },
)
console.log(`\ntiles: ${totals.features} features, ${totals.squares} task squares`)

const inputs = []
for (const layer of layers) {
  if (!streams[layer].length) continue
  const file = join(WORK_DIR, `${layer}.geojsonl`)
  await writeFile(file, streams[layer].join('\n') + '\n')
  inputs.push('-L', `${layer}:${file}`)
  console.log(`  ${layer}: ${streams[layer].length}`)
}
if (!inputs.length) {
  console.error('tiles: nothing to tile')
  process.exit(1)
}

await mkdir(OUT_DIR, { recursive: true })
const out = join(OUT_DIR, 'iugnorge.pmtiles')

// z16 is plenty for buildings and the app overzooms past it. Dropping is
// allowed only at the overview zooms, where the task squares carry the story
// anyway; from z12 in, every feature is present.
const tippecanoe = [
  '-o', out,
  '--force',
  '-Z4',
  '-z16',
  '--drop-densest-as-needed',
  '--extend-zooms-if-still-dropping',
  '--no-tile-size-limit',
  '--simplification=2',
  ...inputs,
]
console.log(`\ntippecanoe ${tippecanoe.join(' ')}\n`)
await new Promise((done, reject) => {
  const proc = spawn('tippecanoe', tippecanoe, { stdio: 'inherit' })
  proc.on('close', (code) => (code === 0 ? done() : reject(new Error(`tippecanoe exited ${code}`))))
  proc.on('error', reject)
})

await writeFile(
  SUMMARY_FILE,
  JSON.stringify({ generated: new Date().toISOString(), projects: perProject }, null, 1) + '\n',
)

await rm(WORK_DIR, { recursive: true, force: true })
console.log(`\nwrote ${out}`)
console.log(`wrote ${SUMMARY_FILE}`)