From 748c1e8365c5c549782a25bbd0cdde8063729cca Mon Sep 17 00:00:00 2001 From: Valmo Date: Fri, 24 Jul 2026 11:16:19 -0300 Subject: [PATCH] Made project use a callback provider to avoid letting almost the whole world on blank --- nginx/00-callback-provider-cache.conf | 2 + nginx/default.conf | 19 ++++- nginx/render-callback-provider.sh | 4 + scripts/build-tiles.py | 50 +++++++++--- scripts/callback_provider.py | 111 ++++++++++++++++++++++++++ scripts/smoke-test.sh | 4 +- tests/test_build_tiles.py | 39 +++++++++ 7 files changed, 213 insertions(+), 16 deletions(-) create mode 100644 nginx/00-callback-provider-cache.conf create mode 100644 nginx/render-callback-provider.sh create mode 100644 scripts/callback_provider.py diff --git a/nginx/00-callback-provider-cache.conf b/nginx/00-callback-provider-cache.conf new file mode 100644 index 0000000..81063a3 --- /dev/null +++ b/nginx/00-callback-provider-cache.conf @@ -0,0 +1,2 @@ +# Included inside nginx's http block by the stock nginx.conf. +proxy_cache_path /var/cache/nginx/callback-provider levels=1:2 keys_zone=callback_provider:10m inactive=7d max_size=2g use_temp_path=off; diff --git a/nginx/default.conf b/nginx/default.conf index c09037b..428925e 100644 --- a/nginx/default.conf +++ b/nginx/default.conf @@ -8,10 +8,27 @@ server { # Public API is x/y/z. try_files performs an internal lookup at z/x/y.png. location ~ ^/tiles/(?[0-9]+)/(?[0-9]+)/(?[0-9]+)$ { - try_files /tiles/$tile_z/$tile_x/$tile_y.png =404; + if ($request_method = OPTIONS) { + add_header Access-Control-Allow-Origin "*" always; + add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always; + add_header Access-Control-Allow-Headers "Origin, Range, Accept" always; + add_header Content-Length 0; + return 204; + } + try_files /tiles/$tile_z/$tile_x/$tile_y.png @callback_provider_tile; add_header Cache-Control "public, max-age=86400, stale-while-revalidate=604800" always; + add_header Access-Control-Allow-Origin "*" always; + add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always; + add_header Access-Control-Allow-Headers "Origin, Range, Accept" always; } + # A callback-provider base layer fills transparent edges when configured. + location ~ ^/base/(?[0-9]+)/(?[0-9]+)/(?[0-9]+)$ { + try_files /__callback_provider_base__ @callback_provider_base; + } + + include /etc/nginx/includes/callback-provider.conf; + # metadata.json and internal z/x/y.png paths are intentionally not public. location / { return 404; diff --git a/nginx/render-callback-provider.sh b/nginx/render-callback-provider.sh new file mode 100644 index 0000000..2d71bda --- /dev/null +++ b/nginx/render-callback-provider.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu + +python3 /usr/local/bin/callback-provider.py --nginx-output /etc/nginx/includes/callback-provider.conf diff --git a/scripts/build-tiles.py b/scripts/build-tiles.py index cc6bfa4..5e56567 100755 --- a/scripts/build-tiles.py +++ b/scripts/build-tiles.py @@ -21,6 +21,11 @@ from xml.etree import ElementTree as ET from PIL import Image +SCRIPT_DIRECTORY = Path(__file__).resolve().parent +if str(SCRIPT_DIRECTORY) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIRECTORY)) +from callback_provider import ProviderValidationError, validate_callback_provider + PIPELINE_VERSION = "1.0.0" WORLD_WIDTH_METERS = 40075016.68557849 @@ -147,7 +152,8 @@ def read_config(path: Path) -> dict: def effective_zooms(auto_max: int, relative_source: str, config: dict) -> tuple[int, int, bool]: - auto_min = max(6, min(10, auto_max - 7)) + # Low zooms are cheap (usually one tile per source) and make isolated maps discoverable from a world view. + auto_min = 0 override = config["sources"].get(relative_source, {}) minimum = override.get("minzoom") maximum = override.get("maxzoom") @@ -364,6 +370,10 @@ def compose_layer(source_tiles: Path, final_tiles: Path) -> None: result.save(destination, format="PNG", optimize=False) +def write_empty_tile(tiles_dir: Path) -> None: + Image.new("RGBA", (256, 256), (255, 255, 255, 255)).save(tiles_dir / "empty.png", format="PNG", optimize=False) + + def tile_overlay(overlay: dict, work_root: Path, final_tiles: Path, processes: int) -> None: layer_root = work_root / f"layer-{overlay['priority']:04d}-{overlay['index']}" help_result = subprocess.run(["gdal2tiles.py", "--help"], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -376,16 +386,21 @@ def tile_overlay(overlay: dict, work_root: Path, final_tiles: Path, processes: i shutil.rmtree(layer_root, ignore_errors=True) -def publish(stage: Path, data_dir: Path) -> None: - current = data_dir / "tiles" - backup = data_dir / ".tiles-previous" - if backup.exists(): - shutil.rmtree(backup) - if current.exists(): - current.rename(backup) - stage.rename(current) - if backup.exists(): - shutil.rmtree(backup) +def remove_generated_path(path: Path) -> None: + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink() + + +def publish(stage: Path, tiles_dir: Path) -> None: + """Publish staged tiles into a directory which may be a bind mount.""" + tiles_dir.mkdir(parents=True, exist_ok=True) + for existing in tiles_dir.iterdir(): + remove_generated_path(existing) + for generated in stage.iterdir(): + shutil.move(str(generated), str(tiles_dir / generated.name)) + stage.rmdir() def source_record(descriptor: dict, config: dict) -> dict: @@ -415,8 +430,11 @@ def source_record(descriptor: dict, config: dict) -> dict: def build(arguments: argparse.Namespace) -> None: + # Validate before reading or warping any source, so a malformed provider never wastes a full build. + validate_callback_provider(os.environ.get("CALLBACK_PROVIDER")) source_dir = Path(arguments.source_dir).resolve() data_dir = Path(arguments.data_dir).resolve() + tiles_dir = Path(arguments.tiles_dir).resolve() config = read_config(Path(arguments.config).resolve()) if not source_dir.is_dir(): raise RuntimeError(f"diretório source inexistente: {source_dir}") @@ -468,13 +486,15 @@ def build(arguments: argparse.Namespace) -> None: "generated_at": datetime.now(UTC).isoformat(), "minzoom": global_min, "maxzoom": global_max, + "tile_background": "transparent", "processed_sources": records, "sources_with_error": errors, "source_count": len(records), "error_count": len(errors), } + write_empty_tile(stage) (stage / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - publish(stage, data_dir) + publish(stage, tiles_dir) except Exception: shutil.rmtree(stage, ignore_errors=True) raise @@ -486,10 +506,14 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--source-dir", default=os.environ.get("SOURCE_DIR", "source")) parser.add_argument("--data-dir", default=os.environ.get("DATA_DIR", ".")) + parser.add_argument("--tiles-dir", default=os.environ.get("TILES_DIR")) parser.add_argument("--config", default=os.environ.get("MAPS_CONFIG", "config/maps.json")) parser.add_argument("--source-filter", action="append", help="caminho relativo exato sob source/; repetível") parser.add_argument("--processes", type=int, default=max(1, int(os.environ.get("GDAL2TILES_PROCESSES", "1")))) - return parser.parse_args() + arguments = parser.parse_args() + if arguments.tiles_dir is None: + arguments.tiles_dir = str(Path(arguments.data_dir) / "tiles") + return arguments if __name__ == "__main__": diff --git a/scripts/callback_provider.py b/scripts/callback_provider.py new file mode 100644 index 0000000..7c74333 --- /dev/null +++ b/scripts/callback_provider.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Validate CALLBACK_PROVIDER and render its safe Nginx proxy locations.""" + +from __future__ import annotations + +import argparse +import ipaddress +import re +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlsplit + + +class ProviderValidationError(ValueError): + pass + + +@dataclass(frozen=True) +class CallbackProvider: + origin: str + path_template: str + + +def validate_callback_provider(value: str | None) -> CallbackProvider | None: + if value is None or value == "": + return None + if value != value.strip() or len(value) > 2048 or any(character.isspace() for character in value): + raise ProviderValidationError("CALLBACK_PROVIDER não pode conter espaços e deve ter até 2048 caracteres") + parsed = urlsplit(value) + if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password: + raise ProviderValidationError("CALLBACK_PROVIDER deve ser uma URL HTTPS pública, sem usuário ou senha") + if parsed.query or parsed.fragment or not parsed.path.startswith("/"): + raise ProviderValidationError("CALLBACK_PROVIDER não aceita query, fragmento e exige um caminho absoluto") + try: + port = parsed.port + except ValueError as error: + raise ProviderValidationError("CALLBACK_PROVIDER possui porta inválida") from error + host = parsed.hostname + if not host or not re.fullmatch(r"[A-Za-z0-9.-]+", host) or ".." in host or host.startswith(".") or host.endswith("."): + raise ProviderValidationError("CALLBACK_PROVIDER possui host inválido") + try: + address = ipaddress.ip_address(host) + except ValueError: + address = None + if address is not None and not address.is_global: + raise ProviderValidationError("CALLBACK_PROVIDER não pode apontar para endereço IP privado ou reservado") + if port is not None and not 1 <= port <= 65535: + raise ProviderValidationError("CALLBACK_PROVIDER possui porta inválida") + if not re.fullmatch(r"/[A-Za-z0-9._~/%{}-]+", parsed.path) or "/../" in f"{parsed.path}/" or any(token in parsed.path for token in ("$", ";", "\\")): + raise ProviderValidationError("CALLBACK_PROVIDER possui caminho não permitido") + for token in ("{x}", "{y}", "{z}"): + if parsed.path.count(token) != 1: + raise ProviderValidationError("CALLBACK_PROVIDER deve conter exatamente uma ocorrência de {x}, {y} e {z}") + remaining_placeholders = parsed.path + for token in ("{x}", "{y}", "{z}"): + remaining_placeholders = remaining_placeholders.replace(token, "") + if "{" in remaining_placeholders or "}" in remaining_placeholders: + raise ProviderValidationError("CALLBACK_PROVIDER contém placeholder desconhecido") + return CallbackProvider(origin=f"https://{parsed.netloc}", path_template=parsed.path) + + +def render_location(name: str, provider: CallbackProvider | None, x: str, y: str, z: str, label: str) -> str: + headers = """ add_header Cache-Control \"public, max-age=86400, stale-while-revalidate=604800\" always; + add_header Access-Control-Allow-Origin \"*\" always; + add_header Access-Control-Allow-Methods \"GET, HEAD, OPTIONS\" always; + add_header Access-Control-Allow-Headers \"Origin, Range, Accept\" always;""" + if provider is None: + return f"""location @{name} {{ + try_files /tiles/empty.png =500; +{headers} + }} +""" + rendered_path = provider.path_template.replace("{x}", f"${x}").replace("{y}", f"${y}").replace("{z}", f"${z}") + return f"""location @{name} {{ + rewrite ^ {rendered_path} break; + proxy_pass {provider.origin}; + proxy_set_header Host $proxy_host; + proxy_ssl_server_name on; + proxy_cache callback_provider; + proxy_cache_valid 200 7d; + proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504; + add_header X-Map-Source \"Callback provider {label}\" always; +{headers} + }} +""" + + +def render_nginx(provider: CallbackProvider | None) -> str: + return ( + "# Generated at container startup. Do not edit.\n" + + render_location("callback_provider_tile", provider, "tile_x", "tile_y", "tile_z", "fallback") + + "\n" + + render_location("callback_provider_base", provider, "base_x", "base_y", "base_z", "base") + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--nginx-output", type=Path) + arguments = parser.parse_args() + provider = validate_callback_provider(__import__("os").environ.get("CALLBACK_PROVIDER")) + if arguments.nginx_output: + arguments.nginx_output.parent.mkdir(parents=True, exist_ok=True) + arguments.nginx_output.write_text(render_nginx(provider), encoding="utf-8") + + +if __name__ == "__main__": + try: + main() + except ProviderValidationError as error: + raise SystemExit(f"CALLBACK_PROVIDER inválido: {error}") diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index a3e4afd..6fccb01 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -35,6 +35,6 @@ done headers="$(curl -sS -D - -o /dev/null "$base_url/tiles/$x/$y/$z")" printf '%s\n' "$headers" | grep -qi '^HTTP/.* 200' printf '%s\n' "$headers" | grep -qi '^Content-Type: image/png' -test "$(curl -sS -o /dev/null -w '%{http_code}' "$base_url/tiles/0/0/0")" = "404" +printf '%s\n' "$headers" | grep -qi '^Access-Control-Allow-Origin: \*' -printf 'Smoke test passou: /tiles/%s/%s/%s retornou PNG e tile descoberta retornou 404.\n' "$x" "$y" "$z" +printf 'Smoke test passou: /tiles/%s/%s/%s retornou PNG/CORS.\n' "$x" "$y" "$z" diff --git a/tests/test_build_tiles.py b/tests/test_build_tiles.py index c3b961e..a3fbec8 100644 --- a/tests/test_build_tiles.py +++ b/tests/test_build_tiles.py @@ -26,12 +26,51 @@ class BuildTilesTests(unittest.TestCase): config = {"global": {"minzoom": 7, "maxzoom": 12}, "sources": {"Altis.kmz": {"minzoom": 9, "maxzoom": 14}}} self.assertEqual(build_tiles.effective_zooms(11, "Altis.kmz", config), (9, 14, True)) + def test_callback_provider_accepts_safe_https_xyz_template(self): + provider = build_tiles.validate_callback_provider("https://tiles.example.org/maps/{z}/{x}/{y}.png") + self.assertEqual(provider.origin, "https://tiles.example.org") + + def test_callback_provider_rejects_missing_or_unsafe_template(self): + with self.assertRaises(build_tiles.ProviderValidationError): + build_tiles.validate_callback_provider("http://tiles.example.org/{z}/{x}/{y}.png") + with self.assertRaises(build_tiles.ProviderValidationError): + build_tiles.validate_callback_provider("https://127.0.0.1/{z}/{x}/{y}.png") + with self.assertRaises(build_tiles.ProviderValidationError): + build_tiles.validate_callback_provider("https://tiles.example.org/{z}/{x}.png") + def test_config_normalizes_source_key(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "maps.json" path.write_text(json.dumps({"global": {}, "sources": {"nested\\map.kmz": {"minzoom": 8}}})) self.assertIn("nested/map.kmz", build_tiles.read_config(path)["sources"]) + def test_publish_replaces_generated_output_directory(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + stage = root / "stage" + output = root / "tiles" + stage.mkdir() + output.mkdir() + (stage / "metadata.json").write_text("new") + (stage / "8").mkdir() + (stage / "8" / "tile.png").write_text("tile") + (output / "stale.png").write_text("stale") + build_tiles.publish(stage, output) + self.assertFalse((output / "stale.png").exists()) + self.assertEqual((output / "metadata.json").read_text(), "new") + self.assertTrue((output / "8" / "tile.png").is_file()) + + def test_empty_tile_is_opaque_white_png(self): + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) + build_tiles.write_empty_tile(target) + with build_tiles.Image.open(target / "empty.png") as image: + self.assertEqual(image.mode, "RGBA") + self.assertEqual(image.size, (256, 256)) + self.assertEqual(image.getpixel((0, 0)), (255, 255, 255, 255)) + + + if __name__ == "__main__": unittest.main()