added conopy

scripts/bootstrap_canopy.py

@@ -8,13 +8,15 @@ from collections.abc import Iterable
 from pathlib import Path
 from typing import Any
 from urllib.error import HTTPError, URLError
-from urllib.request import urlopen, urlretrieve
+from urllib.request import Request, urlopen
 
-META_CHM_V2_BASE_URL = (
+META_CHM_V2_INDEX_BASE_URL = (
     "https://dataforgood-fb-data.s3.amazonaws.com/"
     "forests/v2/global/dinov3_global_chm_v2_ml3"
 )
-META_CHM_V2_INDEX_URL = f"{META_CHM_V2_BASE_URL}/tiles.geojson"
+META_CHM_V2_BASE_URL = "https://data.source.coop/tge-labs/meta-chm-v2/chm"
+META_CHM_V2_INDEX_URL = f"{META_CHM_V2_INDEX_BASE_URL}/tiles.geojson"
+USER_AGENT = "RadioApi canopy bootstrap/0.1"
 
 
 def parse_bbox(raw: str) -> tuple[float, float, float, float]:
@@ -83,11 +85,28 @@ def _asset_url(feature: dict[str, Any], base_url: str = META_CHM_V2_BASE_URL) ->
     return None
 
 
+def validate_urls(urls: list[str], sample_size: int = 2) -> None:
+    for url in urls[:sample_size]:
+        request = Request(url, headers={"Range": "bytes=0-0", "User-Agent": USER_AGENT})
+        try:
+            with urlopen(request) as response:
+                if response.status >= 400:
+                    raise SystemExit(f"Canopy URL is not reachable: {url} ({response.status})")
+        except HTTPError as exc:
+            raise SystemExit(
+                f"Canopy URL is not reachable: {url} ({exc.code}). "
+                "Check --index-url/--urls-file or the dataset URL template."
+            ) from exc
+        except URLError as exc:
+            raise SystemExit(f"Canopy URL validation failed: {url}: {exc}") from exc
+
+
 def load_index(index_url: str) -> dict[str, Any]:
     path = Path(index_url)
     if path.exists():
         return json.loads(path.read_text(encoding="utf-8"))
-    with urlopen(index_url) as response:
+    request = Request(index_url, headers={"User-Agent": USER_AGENT})
+    with urlopen(request) as response:
         return json.loads(response.read().decode("utf-8"))
 
 
@@ -138,7 +157,9 @@ def download_urls(urls: list[str], output_dir: Path, overwrite: bool) -> tuple[i
 
         print(f"download {url}")
         try:
-            urlretrieve(url, destination)
+            request = Request(url, headers={"User-Agent": USER_AGENT})
+            with urlopen(request) as response, destination.open("wb") as output:
+                output.write(response.read())
         except (HTTPError, URLError) as exc:
             failed.append(f"{url}: {exc}")
             if destination.exists():
@@ -156,6 +177,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--source", default="meta-chm-v2", choices=["meta-chm-v2"])
     parser.add_argument("--index-url", default=META_CHM_V2_INDEX_URL)
     parser.add_argument("--urls-file", type=Path)
+    parser.add_argument("--skip-url-check", action="store_true")
     parser.add_argument("--overwrite", action="store_true")
     return parser.parse_args()
 
@@ -174,6 +196,9 @@ def main() -> None:
     if not urls:
         raise SystemExit("No canopy tiles intersect requested AOI")
 
+    if not args.skip_url_check:
+        validate_urls(urls)
+
     downloaded, skipped, failed = download_urls(urls, args.output_dir, args.overwrite)
     print(f"downloaded={downloaded} skipped={skipped} failed={len(failed)}")
     if failed: