added conopy script proxy

README.md

@@ -103,6 +103,9 @@ Use `socks5h://` (DNS via proxy), same as `curl --proxy socks5h://`. Plain `sock
 resolves hostnames locally and may hang if local DNS is blocked.
 
 The same URL can be passed via `CANOPY_PROXY` env var instead of `--proxy`.
+Large canopy tiles are downloaded to `.part` files first. If the connection times
+out, rerun the same command; existing `.part` files are resumed with HTTP Range
+requests. Use `--retries` and `--timeout-s` to tune unstable links.
 
 The `/api/v1/landcover/path` endpoint samples all `.tif`/`.tiff` files under
 `LANDCOVER_PATH` recursively. Canopy data is optional; when it is missing,

api/tests/test_viewshed_and_canopy.py

@@ -77,6 +77,16 @@ def test_canopy_parse_socks5_proxy_url() -> None:
     assert proxy.endpoint == "socks5h://194.33.35.46:38599"
 
 
+def test_canopy_response_total_size_from_content_range() -> None:
+    response = type(
+        "Response",
+        (),
+        {"headers": {"Content-Range": "bytes 1048576-2097151/257131787"}},
+    )()
+
+    assert bootstrap_canopy._response_total_size(response, 1_048_576) == 257_131_787
+
+
 def test_canopy_urls_file_ignores_comments(tmp_path: Path) -> None:
     urls_file = tmp_path / "urls.txt"
     urls_file.write_text(

scripts/bootstrap_canopy.py

@@ -209,14 +209,51 @@ def _format_mb(value: int) -> str:
     return f"{value / (1024 * 1024):.1f} MB"
 
 
-def _download_one(url: str, destination: Path, timeout_s: int, chunk_size: int) -> None:
-    partial = destination.with_suffix(f"{destination.suffix}.part")
-    request = Request(url, headers={"User-Agent": USER_AGENT})
-    with urlopen(request, timeout=timeout_s) as response, partial.open("wb") as output:
+def _response_total_size(response: Any, offset: int) -> int | None:
+    content_range = response.headers.get("Content-Range")
+    if content_range and "/" in content_range:
+        total = content_range.rsplit("/", 1)[-1]
+        if total.isdigit():
+            return int(total)
+
     total_header = response.headers.get("Content-Length")
-        total = int(total_header) if total_header and total_header.isdigit() else None
-        read = 0
+    if total_header and total_header.isdigit():
+        return offset + int(total_header) if offset else int(total_header)
+    return None
+
+
+def _download_one(
+    url: str,
+    destination: Path,
+    timeout_s: int,
+    chunk_size: int,
+    retries: int,
+) -> None:
+    partial = destination.with_suffix(f"{destination.suffix}.part")
+    attempt = 0
+
+    while True:
+        attempt += 1
+        offset = partial.stat().st_size if partial.exists() else 0
+        headers = {"User-Agent": USER_AGENT}
+        if offset:
+            headers["Range"] = f"bytes={offset}-"
+            print(f"  resume from {_format_mb(offset)}")
+
+        request = Request(url, headers=headers)
+        try:
+            with urlopen(request, timeout=timeout_s) as response:
+                append = offset > 0 and response.status == 206
+                if offset > 0 and not append:
+                    print("  server ignored Range header, restarting this tile")
+                    offset = 0
+
+                total = _response_total_size(response, offset)
+                read = offset
                 last_report = time.monotonic()
+                mode = "ab" if append else "wb"
+
+                with partial.open(mode) as output:
                     while True:
                         chunk = response.read(chunk_size)
                         if not chunk:
@@ -227,11 +264,22 @@ def _download_one(url: str, destination: Path, timeout_s: int, chunk_size: int)
                         if now - last_report >= 5:
                             if total:
                                 pct = (read / total) * 100
-                    print(f"  {_format_mb(read)} / {_format_mb(total)} ({pct:.1f}%)")
+                                print(
+                                    f"  {_format_mb(read)} / {_format_mb(total)} "
+                                    f"({pct:.1f}%)"
+                                )
                             else:
                                 print(f"  {_format_mb(read)}")
                             last_report = now
+        except (TimeoutError, URLError, OSError) as exc:
+            if attempt > retries:
+                raise
+            print(f"  retry {attempt}/{retries} after {type(exc).__name__}: {exc}")
+            time.sleep(min(30, attempt * 5))
+            continue
+
         partial.replace(destination)
+        return
 
 
 def download_urls(
@@ -241,6 +289,7 @@ def download_urls(
     *,
     timeout_s: int = 60,
     chunk_size: int = 1024 * 1024,
+    retries: int = 5,
 ) -> tuple[int, int, list[str]]:
     downloaded = 0
     skipped = 0
@@ -255,16 +304,14 @@ def download_urls(
             continue
 
         partial = destination.with_suffix(f"{destination.suffix}.part")
-        if partial.exists() and not overwrite:
+        if partial.exists() and overwrite:
             partial.unlink()
 
         print(f"download {url}")
         try:
-            _download_one(url, destination, timeout_s, chunk_size)
-        except (HTTPError, URLError) as exc:
+            _download_one(url, destination, timeout_s, chunk_size, retries)
+        except (HTTPError, URLError, TimeoutError, OSError) as exc:
             failed.append(f"{url}: {exc}")
-            if partial.exists():
-                partial.unlink()
             continue
         downloaded += 1
 
@@ -280,6 +327,12 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--urls-file", type=Path)
     parser.add_argument("--skip-url-check", action="store_true")
     parser.add_argument("--timeout-s", default=60, type=int)
+    parser.add_argument(
+        "--retries",
+        default=5,
+        type=int,
+        help="Retry count per tile after network timeouts; partial files are resumed",
+    )
     parser.add_argument(
         "--proxy",
         default=os.environ.get("CANOPY_PROXY"),
@@ -317,6 +370,7 @@ def main() -> None:
         args.output_dir,
         args.overwrite,
         timeout_s=args.timeout_s,
+        retries=args.retries,
     )
     print(f"downloaded={downloaded} skipped={skipped} failed={len(failed)}")
     if failed: