diff --git a/scripts/bootstrap_canopy.py b/scripts/bootstrap_canopy.py index d498cf6..491f7c0 100644 --- a/scripts/bootstrap_canopy.py +++ b/scripts/bootstrap_canopy.py @@ -8,13 +8,15 @@ from collections.abc import Iterable from pathlib import Path from typing import Any from urllib.error import HTTPError, URLError -from urllib.request import urlopen, urlretrieve +from urllib.request import Request, urlopen -META_CHM_V2_BASE_URL = ( +META_CHM_V2_INDEX_BASE_URL = ( "https://dataforgood-fb-data.s3.amazonaws.com/" "forests/v2/global/dinov3_global_chm_v2_ml3" ) -META_CHM_V2_INDEX_URL = f"{META_CHM_V2_BASE_URL}/tiles.geojson" +META_CHM_V2_BASE_URL = "https://data.source.coop/tge-labs/meta-chm-v2/chm" +META_CHM_V2_INDEX_URL = f"{META_CHM_V2_INDEX_BASE_URL}/tiles.geojson" +USER_AGENT = "RadioApi canopy bootstrap/0.1" def parse_bbox(raw: str) -> tuple[float, float, float, float]: @@ -83,11 +85,28 @@ def _asset_url(feature: dict[str, Any], base_url: str = META_CHM_V2_BASE_URL) -> return None +def validate_urls(urls: list[str], sample_size: int = 2) -> None: + for url in urls[:sample_size]: + request = Request(url, headers={"Range": "bytes=0-0", "User-Agent": USER_AGENT}) + try: + with urlopen(request) as response: + if response.status >= 400: + raise SystemExit(f"Canopy URL is not reachable: {url} ({response.status})") + except HTTPError as exc: + raise SystemExit( + f"Canopy URL is not reachable: {url} ({exc.code}). " + "Check --index-url/--urls-file or the dataset URL template." + ) from exc + except URLError as exc: + raise SystemExit(f"Canopy URL validation failed: {url}: {exc}") from exc + + def load_index(index_url: str) -> dict[str, Any]: path = Path(index_url) if path.exists(): return json.loads(path.read_text(encoding="utf-8")) - with urlopen(index_url) as response: + request = Request(index_url, headers={"User-Agent": USER_AGENT}) + with urlopen(request) as response: return json.loads(response.read().decode("utf-8")) @@ -138,7 +157,9 @@ def download_urls(urls: list[str], output_dir: Path, overwrite: bool) -> tuple[i print(f"download {url}") try: - urlretrieve(url, destination) + request = Request(url, headers={"User-Agent": USER_AGENT}) + with urlopen(request) as response, destination.open("wb") as output: + output.write(response.read()) except (HTTPError, URLError) as exc: failed.append(f"{url}: {exc}") if destination.exists(): @@ -156,6 +177,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--source", default="meta-chm-v2", choices=["meta-chm-v2"]) parser.add_argument("--index-url", default=META_CHM_V2_INDEX_URL) parser.add_argument("--urls-file", type=Path) + parser.add_argument("--skip-url-check", action="store_true") parser.add_argument("--overwrite", action="store_true") return parser.parse_args() @@ -174,6 +196,9 @@ def main() -> None: if not urls: raise SystemExit("No canopy tiles intersect requested AOI") + if not args.skip_url_check: + validate_urls(urls) + downloaded, skipped, failed = download_urls(urls, args.output_dir, args.overwrite) print(f"downloaded={downloaded} skipped={skipped} failed={len(failed)}") if failed: