From 5952fe301d9bb9588f069e864fcb35c3fc4f61be Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 7 Feb 2025 21:19:13 +0000 Subject: [PATCH 1/2] refactor(backend): remove basemapper.py mbtile/pmtile generator, use go binaries --- compose.yaml | 1 - deploy/compose.development.yaml | 1 - deploy/compose.main.yaml | 1 - src/backend/Dockerfile | 5 +- src/backend/app/projects/project_crud.py | 128 +++++++++++++++++------ 5 files changed, 96 insertions(+), 40 deletions(-) diff --git a/compose.yaml b/compose.yaml index 60cd5726f5..b5b840cc84 100644 --- a/compose.yaml +++ b/compose.yaml @@ -76,7 +76,6 @@ services: # tty: true volumes: - fmtm_logs:/opt/logs - - fmtm_tiles:/opt/tiles - ./src/backend/pyproject.toml:/opt/pyproject.toml:ro - ./src/backend/app:/opt/app - ./src/backend/tests:/opt/tests:ro diff --git a/deploy/compose.development.yaml b/deploy/compose.development.yaml index 92a448f619..190d31de2f 100644 --- a/deploy/compose.development.yaml +++ b/deploy/compose.development.yaml @@ -83,7 +83,6 @@ services: image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" volumes: - fmtm_logs:/opt/logs - - fmtm_tiles:/opt/tiles depends_on: fmtm-db: condition: service_healthy diff --git a/deploy/compose.main.yaml b/deploy/compose.main.yaml index 604930c2d3..c01aea0bc8 100644 --- a/deploy/compose.main.yaml +++ b/deploy/compose.main.yaml @@ -73,7 +73,6 @@ services: image: "ghcr.io/hotosm/fmtm/backend:main" volumes: - fmtm_logs:/opt/logs - - fmtm_tiles:/opt/tiles depends_on: fmtm-db: condition: service_healthy diff --git a/src/backend/Dockerfile b/src/backend/Dockerfile index 04da14d68f..2d3091cc33 100644 --- a/src/backend/Dockerfile +++ b/src/backend/Dockerfile @@ -131,12 +131,11 @@ COPY app/ /opt/app/ COPY migrations/ /opt/migrations/ # Add non-root user, permissions RUN useradd -u 1001 -m -c "fmtm account" -d /home/appuser -s /bin/false appuser \ - && mkdir -p /opt/logs /opt/tiles \ + && mkdir -p /opt/logs \ && chown -R appuser:appuser /opt /home/appuser \ && chmod +x /app-entrypoint.sh /migrate-entrypoint.sh /backup-entrypoint.sh -# Add volumes for persistence +# Add log volume for persistence VOLUME /opt/logs -VOLUME /opt/tiles # Change to non-root user USER appuser diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index a2bd669486..d4e09b2bee 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -18,7 +18,7 @@ """Logic for FMTM project routes.""" import json -import shutil +import subprocess import uuid from io import BytesIO from pathlib import Path @@ -32,7 +32,6 @@ from asgiref.sync import async_to_sync from fastapi import HTTPException, Request from loguru import logger as log -from osm_fieldwork.basemapper import create_basemap_file from osm_login_python.core import Auth from osm_rawdata.postgres import PostgresClient from psycopg import Connection @@ -56,8 +55,6 @@ from app.projects import project_deps, project_schemas from app.s3 import add_file_to_bucket, add_obj_to_bucket -TILESDIR = "/opt/tiles" - async def get_projects_featcol( db: Connection, @@ -727,15 +724,49 @@ def generate_project_basemap( tms (str, optional): Default None. Custom TMS provider URL. """ new_basemap = None + mbtiles_file = "" + final_tile_file = "" # TODO update this for user input or automatic # maxzoom can be determined from OAM: https://tiles.openaerialmap.org/663 # c76196049ef00013b8494/0/663c76196049ef00013b8495 - # TODO xy should also be user configurable + # TODO should inverted_y be user configurable? + # NOTE mbtile max supported zoom level is 22 (in GDAL at least) - zooms = "12-22" if tms else "12-19" - tiles_dir = f"{TILESDIR}/{project_id}" - outfile = f"{tiles_dir}/{project_id}_{source}tiles.{output_format}" + if tms: + zooms = "12-22" + # While typically satellite imagery TMS only goes to zoom 19 + else: + zooms = "12-19" + + mbtiles_file = Path(f"/tmp/{project_id}_{source}tiles.mbtiles") + + # Set URL based on source (previously in osm-fieldwork) + if source == "esri": + # ESRI uses inverted zyx convention + # the ordering is extracted automatically from the URL, else use + # -inverted-y param + tms_url = "http://services.arcgisonline.com/arcgis/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}.png" + tile_format = "png" + elif source == "bing": + # FIXME this probably doesn't work + tms_url = "http://ecn.t0.tiles.virtualearth.net/tiles/h{z}/{x}/{y}.jpg?g=129&mkt=en&stl=H" + tile_format = "jpg" + elif source == "google": + tms_url = "https://mt0.google.com/vt?lyrs=s&x={x}&y={y}&z={z}" + tile_format = "jpg" + elif source == "custom" and tms: + tms_url = tms + if not (tile_format := Path(tms_url).suffix.lstrip(".")): + # Default to png if suffix not included in URL + tile_format = "png" + else: + raise ValueError("Must select a source from: esri,bing,google,custom") + + # Invert zxy --> zyx for OAM provider + # inverted_y = True if tms and "openaerialmap" in tms else False + # NOTE the xy ordering is determined from the URL placeholders, by tilepack! + inverted_y = False # NOTE here we put the connection in autocommit mode to ensure we get # background task db entries if there is an exception. @@ -757,36 +788,65 @@ def generate_project_basemap( min_lon, min_lat, max_lon, max_lat = new_basemap.bbox - # Overwrite source with OAM provider - if tms and "openaerialmap" in tms: - # NOTE the 'xy' param is set automatically by source=oam - source = "oam" - + tilepack_cmd = [ + # "prlimit", f"--as={500 * 1000}", "--", + "tilepack", + "-dsn", + f"{str(mbtiles_file)}", + "-url-template", + f"{tms_url}", + # tilepack requires format: south,west,north,east + "-bounds", + f"{min_lat},{min_lon},{max_lat},{max_lon}", + "-zooms", + f"{zooms}", + "-output-mode", + "mbtiles", # options: mbtiles or disk + "-mbtiles-format", + f"{tile_format}", + "-ensure-gzip=false", # gzip is only used for pbf vector tiles + "-tileset-name", + f"fmtm_{project_id}_{source}tiles", + ] + # Add '-inverted-y' only if needed + if inverted_y: + tilepack_cmd.append("-inverted-y") log.debug( - "Creating basemap with params: " - f"boundary={min_lon},{min_lat},{max_lon},{max_lat} | " - f"outfile={outfile} | " - f"zooms={zooms} | " - f"outdir={tiles_dir} | " - f"source={source} | " - f"tms={tms}" + "Creating basemap mbtiles using tilepack with command: " + f"{' '.join(tilepack_cmd)}" ) - - create_basemap_file( - boundary=f"{min_lon},{min_lat},{max_lon},{max_lat}", - outfile=outfile, - zooms=zooms, - outdir=tiles_dir, - source=source, - tms=tms, + subprocess.run(tilepack_cmd, check=True) + log.info( + f"MBTile basemap created for project ID {project_id}: {str(mbtiles_file)}" ) - log.info(f"Basemap created for project ID {project_id}: {outfile}") + # write to another var so we upload either mbtiles OR pmtiles override below + final_tile_file = str(mbtiles_file) + + if output_format == "pmtiles": + pmtiles_file = mbtiles_file.with_suffix(".pmtiles") + pmtile_command = [ + # "prlimit", f"--as={500 * 1000}", "--", + "pmtiles", + "convert", + f"{str(mbtiles_file)}", + f"{str(pmtiles_file)}", + ] + log.debug( + "Converting mbtiles --> pmtiles file with command: " + f"{' '.join(pmtile_command)}" + ) + subprocess.run(pmtile_command, check=True) + log.info( + f"PMTile basemap created for project ID {project_id}: " + f"{str(pmtiles_file)}" + ) + final_tile_file = str(pmtiles_file) # Generate S3 urls # We parse as BasemapOut to calculated computed fields (format, mimetype) basemap_out = project_schemas.BasemapOut( **new_basemap.model_dump(exclude=["url"]), - url=outfile, + url=final_tile_file, ) basemap_s3_path = ( f"{org_id}/{project_id}/basemaps/{basemap_out.id}.{basemap_out.format}" @@ -795,7 +855,7 @@ def generate_project_basemap( add_file_to_bucket( settings.S3_BUCKET_NAME, basemap_s3_path, - outfile, + final_tile_file, content_type=basemap_out.mimetype, ) basemap_external_s3_url = ( @@ -845,9 +905,9 @@ def generate_project_basemap( ), ) finally: - log.info(f"Cleaned up tiles directory: {tiles_dir}") - Path(outfile).unlink(missing_ok=True) - shutil.rmtree(tiles_dir) + Path(mbtiles_file).unlink(missing_ok=True) + Path(final_tile_file).unlink(missing_ok=True) + log.debug("Cleaning up tile archives on disk") # async def convert_geojson_to_osm(geojson_file: str): From ee6e6b6a982974c1e7e4408fee01ab982f776e8d Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 7 Feb 2025 21:21:06 +0000 Subject: [PATCH 2/2] build: add go-tilepacks & go-pmtiles binaries to backend for basemap gen --- contrib/basemaps/Dockerfile | 52 +++++++++++++++++++++++++++++++++++++ contrib/basemaps/README.md | 14 ++++++++++ contrib/basemaps/build.sh | 10 +++++++ src/backend/Dockerfile | 4 +++ 4 files changed, 80 insertions(+) create mode 100644 contrib/basemaps/Dockerfile create mode 100644 contrib/basemaps/README.md create mode 100644 contrib/basemaps/build.sh diff --git a/contrib/basemaps/Dockerfile b/contrib/basemaps/Dockerfile new file mode 100644 index 0000000000..a87207f1c2 --- /dev/null +++ b/contrib/basemaps/Dockerfile @@ -0,0 +1,52 @@ +FROM docker.io/alpine/curl:8.11.1 AS base + + +# Download and verify tilepack binary +# FROM base AS tilepack +# ENV GO_TILEPACK_URL=https://github.com/tilezen/go-tilepacks/releases/download/v1.0.0-pre1/tilepack_1.0.0-pre1_linux_amd64.tar.gz \ +# TILEPACK_SHA1SUM=1f235fd3da7f9c8a2710a1a8d44a27c2c98df939 +# RUN curl -fsSLO "$GO_TILEPACK_URL" \ +# && tar -xvzf tilepack_1.0.0-pre1_linux_amd64.tar.gz \ +# && echo "${TILEPACK_SHA1SUM} tilepack" | sha1sum -c - \ +# && chmod +x tilepack \ +# && mv tilepack /tilepack +# FIXME temp workaround using fork until PRs merged +# - https://github.com/tilezen/go-tilepacks/pull/36 +# - https://github.com/tilezen/go-tilepacks/pull/38 +FROM base AS tilepack +ENV GO_TILEPACK_URL=https://github.com/spwoodcock/go-tilepacks/releases/download/0.3.0/tilepack_0.3.0_linux_amd64.tar.gz \ + TILEPACK_SHA1SUM=6b7269735e9fb431de3457c8e2b6aa6d3f3ee49e +RUN curl -fsSLO "$GO_TILEPACK_URL" \ + && tar -xvzf tilepack_0.3.0_linux_amd64.tar.gz \ + && echo "${TILEPACK_SHA1SUM} tilepack" | sha1sum -c - \ + && chmod +x tilepack \ + && mv tilepack /tilepack + + + +# Download and verify pmtiles binary +FROM base AS pmtiles +ENV GO_PMTILES_URL=https://github.com/protomaps/go-pmtiles/releases/download/v1.25.0/go-pmtiles_1.25.0_Linux_x86_64.tar.gz \ + PMTILES_SHA1SUM=ed4795e24bfcccc4fd07a54dfc7926e15cc835de +RUN curl -fsSLO "$GO_PMTILES_URL" \ + && tar -xvzf go-pmtiles_1.25.0_Linux_x86_64.tar.gz \ + && echo "${PMTILES_SHA1SUM} pmtiles" | sha1sum -c - \ + && chmod +x pmtiles \ + && mv pmtiles /pmtiles + + +# Add a non-root user to passwd file +FROM base AS useradd +RUN addgroup -g 1000 nonroot +RUN adduser -D -u 1000 -G nonroot nonroot + + +# Deploy the application binary into scratch image +FROM scratch AS release +WORKDIR /app +COPY --from=useradd /etc/group /etc/group +COPY --from=useradd /etc/passwd /etc/passwd +COPY --from=tilepack /tilepack /usr/bin/tilepack +COPY --from=pmtiles /pmtiles /usr/bin/pmtiles +USER nonroot:nonroot +ENTRYPOINT ["tilepack"] diff --git a/contrib/basemaps/README.md b/contrib/basemaps/README.md new file mode 100644 index 0000000000..7ad1cdbcf1 --- /dev/null +++ b/contrib/basemaps/README.md @@ -0,0 +1,14 @@ +# Basemap Generator Image + +This container image contains binaries for: + +- +- + +With these, we can generate an mbtiles file from TMS, then also convert to PMTiles. + +## Usage + +```bash +docker run --rm ghcr.io/hotosm/fmtm/basemap-generator:0.3.0 --help +``` diff --git a/contrib/basemaps/build.sh b/contrib/basemaps/build.sh new file mode 100644 index 0000000000..fdd006b35f --- /dev/null +++ b/contrib/basemaps/build.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +IMAGE_NAME=ghcr.io/hotosm/fmtm/basemap-generator:0.3.0 + +echo "Building ${IMAGE_NAME}" +docker build . --tag "${IMAGE_NAME}" + +if [[ -n "$PUSH_IMG" ]]; then + docker push "${IMAGE_NAME}" +fi diff --git a/src/backend/Dockerfile b/src/backend/Dockerfile index 2d3091cc33..70437be04d 100644 --- a/src/backend/Dockerfile +++ b/src/backend/Dockerfile @@ -19,6 +19,7 @@ ARG UV_IMG_TAG=0.5.2 ARG MINIO_TAG=${MINIO_TAG:-RELEASE.2025-01-20T14-49-07Z} FROM ghcr.io/astral-sh/uv:${UV_IMG_TAG} AS uv FROM docker.io/minio/minio:${MINIO_TAG} AS minio +FROM ghcr.io/hotosm/fmtm/basemap-generator:0.3.0 AS basemap-bins # Includes all labels and timezone info to extend from @@ -121,6 +122,9 @@ RUN apt-get update --quiet \ && rm -rf /var/lib/apt/lists/* # Copy minio mc client COPY --from=minio /usr/bin/mc /usr/local/bin/ +# Copy basemap generation binaries +COPY --from=basemap-bins /usr/bin/tilepack /usr/local/bin/ +COPY --from=basemap-bins /usr/bin/pmtiles /usr/local/bin/ COPY *-entrypoint.sh / ENTRYPOINT ["/app-entrypoint.sh"] # Copy Python deps from build to runtime