diff --git a/README.md b/README.md index 22ed397..c3fe91e 100644 --- a/README.md +++ b/README.md @@ -96,9 +96,12 @@ If direct download is blocked, use a SOCKS5 proxy (`pip install PySocks`): python scripts/bootstrap_canopy.py \ --bbox 27.3,58.4,35.8,61.4 \ --output-dir data/canopy \ - --proxy socks5://user:pass@host:port + --proxy socks5h://user:pass@host:port ``` +Use `socks5h://` (DNS via proxy), same as `curl --proxy socks5h://`. Plain `socks5://` +resolves hostnames locally and may hang if local DNS is blocked. + The same URL can be passed via `CANOPY_PROXY` env var instead of `--proxy`. The `/api/v1/landcover/path` endpoint samples all `.tif`/`.tiff` files under diff --git a/api/tests/test_viewshed_and_canopy.py b/api/tests/test_viewshed_and_canopy.py index fb7046d..87d353e 100644 --- a/api/tests/test_viewshed_and_canopy.py +++ b/api/tests/test_viewshed_and_canopy.py @@ -67,13 +67,14 @@ def test_canopy_selects_intersecting_tiles_from_geojson_index() -> None: def test_canopy_parse_socks5_proxy_url() -> None: proxy = bootstrap_canopy.parse_socks5_proxy( - "socks5://proxy_user:secret@194.33.35.46:38599" + "socks5h://proxy_user:secret@194.33.35.46:38599" ) + assert proxy.scheme == "socks5h" assert proxy.host == "194.33.35.46" assert proxy.port == 38599 assert proxy.username == "proxy_user" assert proxy.password == "secret" - assert proxy.endpoint == "socks5://194.33.35.46:38599" + assert proxy.endpoint == "socks5h://194.33.35.46:38599" def test_canopy_urls_file_ignores_comments(tmp_path: Path) -> None: diff --git a/scripts/bootstrap_canopy.py b/scripts/bootstrap_canopy.py index 997975b..50ab70c 100644 --- a/scripts/bootstrap_canopy.py +++ b/scripts/bootstrap_canopy.py @@ -63,10 +63,13 @@ def configure_socks5_proxy(proxy_url: str) -> Socks5Proxy: ) from exc proxy = parse_socks5_proxy(proxy_url) + # socks5h resolves hostnames on the proxy (like curl --proxy socks5h://). + rdns = proxy.scheme == "socks5h" socks.set_default_proxy( socks.SOCKS5, proxy.host, proxy.port, + rdns=rdns, username=proxy.username, password=proxy.password, ) @@ -140,11 +143,12 @@ def _asset_url(feature: dict[str, Any], base_url: str = META_CHM_V2_BASE_URL) -> return None -def validate_urls(urls: list[str], sample_size: int = 2) -> None: +def validate_urls(urls: list[str], sample_size: int = 2, *, timeout_s: int = 60) -> None: + print(f"validating {min(sample_size, len(urls))} sample URL(s)") for url in urls[:sample_size]: request = Request(url, headers={"Range": "bytes=0-0", "User-Agent": USER_AGENT}) try: - with urlopen(request) as response: + with urlopen(request, timeout=timeout_s) as response: if response.status >= 400: raise SystemExit(f"Canopy URL is not reachable: {url} ({response.status})") except HTTPError as exc: @@ -156,13 +160,17 @@ def validate_urls(urls: list[str], sample_size: int = 2) -> None: raise SystemExit(f"Canopy URL validation failed: {url}: {exc}") from exc -def load_index(index_url: str) -> dict[str, Any]: +def load_index(index_url: str, *, timeout_s: int = 120) -> dict[str, Any]: path = Path(index_url) if path.exists(): + print(f"loading index from {path}") return json.loads(path.read_text(encoding="utf-8")) + print(f"loading index from {index_url}") request = Request(index_url, headers={"User-Agent": USER_AGENT}) - with urlopen(request) as response: - return json.loads(response.read().decode("utf-8")) + with urlopen(request, timeout=timeout_s) as response: + payload = response.read() + print(f"index loaded ({_format_mb(len(payload))})") + return json.loads(payload.decode("utf-8")) def select_tile_urls( @@ -294,14 +302,14 @@ def main() -> None: if args.urls_file is not None: urls = urls_from_file(args.urls_file) else: - index = load_index(args.index_url) + index = load_index(args.index_url, timeout_s=max(args.timeout_s, 120)) urls = select_tile_urls(index, args.bbox) if not urls: raise SystemExit("No canopy tiles intersect requested AOI") if not args.skip_url_check: - validate_urls(urls) + validate_urls(urls, timeout_s=args.timeout_s) print(f"selected_tiles={len(urls)}") downloaded, skipped, failed = download_urls(