diff --git a/README.md b/README.md index e247cb9..22ed397 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,17 @@ Or download manually into `data/landcover` and `data/canopy`. If the canopy tile index is unavailable, pass a newline-separated list of COG URLs with `scripts/bootstrap_canopy.py --urls-file urls.txt --output-dir data/canopy`. +If direct download is blocked, use a SOCKS5 proxy (`pip install PySocks`): + +```bash +python scripts/bootstrap_canopy.py \ + --bbox 27.3,58.4,35.8,61.4 \ + --output-dir data/canopy \ + --proxy socks5://user:pass@host:port +``` + +The same URL can be passed via `CANOPY_PROXY` env var instead of `--proxy`. + The `/api/v1/landcover/path` endpoint samples all `.tif`/`.tiff` files under `LANDCOVER_PATH` recursively. Canopy data is optional; when it is missing, `canopy_height_m` is returned as `null`. diff --git a/api/tests/test_viewshed_and_canopy.py b/api/tests/test_viewshed_and_canopy.py index 74b93e5..fb7046d 100644 --- a/api/tests/test_viewshed_and_canopy.py +++ b/api/tests/test_viewshed_and_canopy.py @@ -65,6 +65,17 @@ def test_canopy_selects_intersecting_tiles_from_geojson_index() -> None: assert urls[0].endswith("/tile_a.tif") +def test_canopy_parse_socks5_proxy_url() -> None: + proxy = bootstrap_canopy.parse_socks5_proxy( + "socks5://proxy_user:secret@194.33.35.46:38599" + ) + assert proxy.host == "194.33.35.46" + assert proxy.port == 38599 + assert proxy.username == "proxy_user" + assert proxy.password == "secret" + assert proxy.endpoint == "socks5://194.33.35.46:38599" + + def test_canopy_urls_file_ignores_comments(tmp_path: Path) -> None: urls_file = tmp_path / "urls.txt" urls_file.write_text( diff --git a/scripts/bootstrap_canopy.py b/scripts/bootstrap_canopy.py index c8a68e6..997975b 100644 --- a/scripts/bootstrap_canopy.py +++ b/scripts/bootstrap_canopy.py @@ -4,11 +4,15 @@ from __future__ import annotations import argparse import json +import os +import socket import time from collections.abc import Iterable +from dataclasses import dataclass from pathlib import Path from typing import Any from urllib.error import HTTPError, URLError +from urllib.parse import unquote, urlparse from urllib.request import Request, urlopen META_CHM_V2_INDEX_BASE_URL = ( @@ -20,6 +24,56 @@ META_CHM_V2_INDEX_URL = f"{META_CHM_V2_INDEX_BASE_URL}/tiles.geojson" USER_AGENT = "RadioApi canopy bootstrap/0.1" +@dataclass(frozen=True) +class Socks5Proxy: + scheme: str + host: str + port: int + username: str | None + password: str | None + + @property + def endpoint(self) -> str: + return f"{self.scheme}://{self.host}:{self.port}" + + +def parse_socks5_proxy(proxy_url: str) -> Socks5Proxy: + parsed = urlparse(proxy_url) + if parsed.scheme not in {"socks5", "socks5h"}: + raise SystemExit( + f"Unsupported proxy scheme: {parsed.scheme!r}. Use socks5:// or socks5h://" + ) + if not parsed.hostname: + raise SystemExit("Proxy URL must include hostname, e.g. socks5://user:pass@host:port") + return Socks5Proxy( + scheme=parsed.scheme, + host=parsed.hostname, + port=parsed.port or 1080, + username=unquote(parsed.username) if parsed.username else None, + password=unquote(parsed.password) if parsed.password else None, + ) + + +def configure_socks5_proxy(proxy_url: str) -> Socks5Proxy: + try: + import socks + except ImportError as exc: + raise SystemExit( + "SOCKS5 proxy requires PySocks. Install with: pip install PySocks" + ) from exc + + proxy = parse_socks5_proxy(proxy_url) + socks.set_default_proxy( + socks.SOCKS5, + proxy.host, + proxy.port, + username=proxy.username, + password=proxy.password, + ) + socket.socket = socks.socksocket + return proxy + + def parse_bbox(raw: str) -> tuple[float, float, float, float]: parts = [float(part.strip()) for part in raw.split(",")] if len(parts) != 4: @@ -218,6 +272,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--urls-file", type=Path) parser.add_argument("--skip-url-check", action="store_true") parser.add_argument("--timeout-s", default=60, type=int) + parser.add_argument( + "--proxy", + default=os.environ.get("CANOPY_PROXY"), + help="SOCKS5 proxy URL, e.g. socks5://user:pass@host:port. " + "Default: CANOPY_PROXY env var", + ) parser.add_argument("--overwrite", action="store_true") return parser.parse_args() @@ -227,6 +287,10 @@ def main() -> None: if args.urls_file is None and args.bbox is None: raise SystemExit("--bbox is required unless --urls-file is provided") + if args.proxy: + proxy = configure_socks5_proxy(args.proxy) + print(f"using proxy {proxy.endpoint}") + if args.urls_file is not None: urls = urls_from_file(args.urls_file) else: