From 925c648528d5a4b88c89ceeb47ce211c2a2f6eb3 Mon Sep 17 00:00:00 2001 From: grigo Date: Wed, 24 Jun 2026 10:08:12 +0300 Subject: [PATCH] added conopy script proxy --- README.md | 3 + api/tests/test_viewshed_and_canopy.py | 10 +++ scripts/bootstrap_canopy.py | 108 +++++++++++++++++++------- 3 files changed, 94 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index c3fe91e..0a095a4 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,9 @@ Use `socks5h://` (DNS via proxy), same as `curl --proxy socks5h://`. Plain `sock resolves hostnames locally and may hang if local DNS is blocked. The same URL can be passed via `CANOPY_PROXY` env var instead of `--proxy`. +Large canopy tiles are downloaded to `.part` files first. If the connection times +out, rerun the same command; existing `.part` files are resumed with HTTP Range +requests. Use `--retries` and `--timeout-s` to tune unstable links. The `/api/v1/landcover/path` endpoint samples all `.tif`/`.tiff` files under `LANDCOVER_PATH` recursively. Canopy data is optional; when it is missing, diff --git a/api/tests/test_viewshed_and_canopy.py b/api/tests/test_viewshed_and_canopy.py index 87d353e..f5c6482 100644 --- a/api/tests/test_viewshed_and_canopy.py +++ b/api/tests/test_viewshed_and_canopy.py @@ -77,6 +77,16 @@ def test_canopy_parse_socks5_proxy_url() -> None: assert proxy.endpoint == "socks5h://194.33.35.46:38599" +def test_canopy_response_total_size_from_content_range() -> None: + response = type( + "Response", + (), + {"headers": {"Content-Range": "bytes 1048576-2097151/257131787"}}, + )() + + assert bootstrap_canopy._response_total_size(response, 1_048_576) == 257_131_787 + + def test_canopy_urls_file_ignores_comments(tmp_path: Path) -> None: urls_file = tmp_path / "urls.txt" urls_file.write_text( diff --git a/scripts/bootstrap_canopy.py b/scripts/bootstrap_canopy.py index 50ab70c..6864825 100644 --- a/scripts/bootstrap_canopy.py +++ b/scripts/bootstrap_canopy.py @@ -209,29 +209,77 @@ def _format_mb(value: int) -> str: return f"{value / (1024 * 1024):.1f} MB" -def _download_one(url: str, destination: Path, timeout_s: int, chunk_size: int) -> None: +def _response_total_size(response: Any, offset: int) -> int | None: + content_range = response.headers.get("Content-Range") + if content_range and "/" in content_range: + total = content_range.rsplit("/", 1)[-1] + if total.isdigit(): + return int(total) + + total_header = response.headers.get("Content-Length") + if total_header and total_header.isdigit(): + return offset + int(total_header) if offset else int(total_header) + return None + + +def _download_one( + url: str, + destination: Path, + timeout_s: int, + chunk_size: int, + retries: int, +) -> None: partial = destination.with_suffix(f"{destination.suffix}.part") - request = Request(url, headers={"User-Agent": USER_AGENT}) - with urlopen(request, timeout=timeout_s) as response, partial.open("wb") as output: - total_header = response.headers.get("Content-Length") - total = int(total_header) if total_header and total_header.isdigit() else None - read = 0 - last_report = time.monotonic() - while True: - chunk = response.read(chunk_size) - if not chunk: - break - output.write(chunk) - read += len(chunk) - now = time.monotonic() - if now - last_report >= 5: - if total: - pct = (read / total) * 100 - print(f" {_format_mb(read)} / {_format_mb(total)} ({pct:.1f}%)") - else: - print(f" {_format_mb(read)}") - last_report = now - partial.replace(destination) + attempt = 0 + + while True: + attempt += 1 + offset = partial.stat().st_size if partial.exists() else 0 + headers = {"User-Agent": USER_AGENT} + if offset: + headers["Range"] = f"bytes={offset}-" + print(f" resume from {_format_mb(offset)}") + + request = Request(url, headers=headers) + try: + with urlopen(request, timeout=timeout_s) as response: + append = offset > 0 and response.status == 206 + if offset > 0 and not append: + print(" server ignored Range header, restarting this tile") + offset = 0 + + total = _response_total_size(response, offset) + read = offset + last_report = time.monotonic() + mode = "ab" if append else "wb" + + with partial.open(mode) as output: + while True: + chunk = response.read(chunk_size) + if not chunk: + break + output.write(chunk) + read += len(chunk) + now = time.monotonic() + if now - last_report >= 5: + if total: + pct = (read / total) * 100 + print( + f" {_format_mb(read)} / {_format_mb(total)} " + f"({pct:.1f}%)" + ) + else: + print(f" {_format_mb(read)}") + last_report = now + except (TimeoutError, URLError, OSError) as exc: + if attempt > retries: + raise + print(f" retry {attempt}/{retries} after {type(exc).__name__}: {exc}") + time.sleep(min(30, attempt * 5)) + continue + + partial.replace(destination) + return def download_urls( @@ -241,6 +289,7 @@ def download_urls( *, timeout_s: int = 60, chunk_size: int = 1024 * 1024, + retries: int = 5, ) -> tuple[int, int, list[str]]: downloaded = 0 skipped = 0 @@ -255,16 +304,14 @@ def download_urls( continue partial = destination.with_suffix(f"{destination.suffix}.part") - if partial.exists() and not overwrite: + if partial.exists() and overwrite: partial.unlink() print(f"download {url}") try: - _download_one(url, destination, timeout_s, chunk_size) - except (HTTPError, URLError) as exc: + _download_one(url, destination, timeout_s, chunk_size, retries) + except (HTTPError, URLError, TimeoutError, OSError) as exc: failed.append(f"{url}: {exc}") - if partial.exists(): - partial.unlink() continue downloaded += 1 @@ -280,6 +327,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--urls-file", type=Path) parser.add_argument("--skip-url-check", action="store_true") parser.add_argument("--timeout-s", default=60, type=int) + parser.add_argument( + "--retries", + default=5, + type=int, + help="Retry count per tile after network timeouts; partial files are resumed", + ) parser.add_argument( "--proxy", default=os.environ.get("CANOPY_PROXY"), @@ -317,6 +370,7 @@ def main() -> None: args.output_dir, args.overwrite, timeout_s=args.timeout_s, + retries=args.retries, ) print(f"downloaded={downloaded} skipped={skipped} failed={len(failed)}") if failed: