added conopy script proxy

scripts/bootstrap_canopy.py

@@ -209,29 +209,77 @@ def _format_mb(value: int) -> str:
     return f"{value / (1024 * 1024):.1f} MB"
 
 
-def _download_one(url: str, destination: Path, timeout_s: int, chunk_size: int) -> None:
+def _response_total_size(response: Any, offset: int) -> int | None:
+    content_range = response.headers.get("Content-Range")
+    if content_range and "/" in content_range:
+        total = content_range.rsplit("/", 1)[-1]
+        if total.isdigit():
+            return int(total)
+
+    total_header = response.headers.get("Content-Length")
+    if total_header and total_header.isdigit():
+        return offset + int(total_header) if offset else int(total_header)
+    return None
+
+
+def _download_one(
+    url: str,
+    destination: Path,
+    timeout_s: int,
+    chunk_size: int,
+    retries: int,
+) -> None:
     partial = destination.with_suffix(f"{destination.suffix}.part")
-    request = Request(url, headers={"User-Agent": USER_AGENT})
-    with urlopen(request, timeout=timeout_s) as response, partial.open("wb") as output:
-        total_header = response.headers.get("Content-Length")
-        total = int(total_header) if total_header and total_header.isdigit() else None
-        read = 0
-        last_report = time.monotonic()
-        while True:
-            chunk = response.read(chunk_size)
-            if not chunk:
-                break
-            output.write(chunk)
-            read += len(chunk)
-            now = time.monotonic()
-            if now - last_report >= 5:
-                if total:
-                    pct = (read / total) * 100
-                    print(f"  {_format_mb(read)} / {_format_mb(total)} ({pct:.1f}%)")
-                else:
-                    print(f"  {_format_mb(read)}")
-                last_report = now
-    partial.replace(destination)
+    attempt = 0
+
+    while True:
+        attempt += 1
+        offset = partial.stat().st_size if partial.exists() else 0
+        headers = {"User-Agent": USER_AGENT}
+        if offset:
+            headers["Range"] = f"bytes={offset}-"
+            print(f"  resume from {_format_mb(offset)}")
+
+        request = Request(url, headers=headers)
+        try:
+            with urlopen(request, timeout=timeout_s) as response:
+                append = offset > 0 and response.status == 206
+                if offset > 0 and not append:
+                    print("  server ignored Range header, restarting this tile")
+                    offset = 0
+
+                total = _response_total_size(response, offset)
+                read = offset
+                last_report = time.monotonic()
+                mode = "ab" if append else "wb"
+
+                with partial.open(mode) as output:
+                    while True:
+                        chunk = response.read(chunk_size)
+                        if not chunk:
+                            break
+                        output.write(chunk)
+                        read += len(chunk)
+                        now = time.monotonic()
+                        if now - last_report >= 5:
+                            if total:
+                                pct = (read / total) * 100
+                                print(
+                                    f"  {_format_mb(read)} / {_format_mb(total)} "
+                                    f"({pct:.1f}%)"
+                                )
+                            else:
+                                print(f"  {_format_mb(read)}")
+                            last_report = now
+        except (TimeoutError, URLError, OSError) as exc:
+            if attempt > retries:
+                raise
+            print(f"  retry {attempt}/{retries} after {type(exc).__name__}: {exc}")
+            time.sleep(min(30, attempt * 5))
+            continue
+
+        partial.replace(destination)
+        return
 
 
 def download_urls(
@@ -241,6 +289,7 @@ def download_urls(
     *,
     timeout_s: int = 60,
     chunk_size: int = 1024 * 1024,
+    retries: int = 5,
 ) -> tuple[int, int, list[str]]:
     downloaded = 0
     skipped = 0
@@ -255,16 +304,14 @@ def download_urls(
             continue
 
         partial = destination.with_suffix(f"{destination.suffix}.part")
-        if partial.exists() and not overwrite:
+        if partial.exists() and overwrite:
             partial.unlink()
 
         print(f"download {url}")
         try:
-            _download_one(url, destination, timeout_s, chunk_size)
-        except (HTTPError, URLError) as exc:
+            _download_one(url, destination, timeout_s, chunk_size, retries)
+        except (HTTPError, URLError, TimeoutError, OSError) as exc:
             failed.append(f"{url}: {exc}")
-            if partial.exists():
-                partial.unlink()
             continue
         downloaded += 1
 
@@ -280,6 +327,12 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--urls-file", type=Path)
     parser.add_argument("--skip-url-check", action="store_true")
     parser.add_argument("--timeout-s", default=60, type=int)
+    parser.add_argument(
+        "--retries",
+        default=5,
+        type=int,
+        help="Retry count per tile after network timeouts; partial files are resumed",
+    )
     parser.add_argument(
         "--proxy",
         default=os.environ.get("CANOPY_PROXY"),
@@ -317,6 +370,7 @@ def main() -> None:
         args.output_dir,
         args.overwrite,
         timeout_s=args.timeout_s,
+        retries=args.retries,
     )
     print(f"downloaded={downloaded} skipped={skipped} failed={len(failed)}")
     if failed: