added conopy script proxy

This commit is contained in:
2026-06-24 09:37:09 +03:00
parent 6355f8afd1
commit ac33e8fb28
3 changed files with 86 additions and 0 deletions
+11
View File
@@ -90,6 +90,17 @@ Or download manually into `data/landcover` and `data/canopy`. If the canopy tile
index is unavailable, pass a newline-separated list of COG URLs with
`scripts/bootstrap_canopy.py --urls-file urls.txt --output-dir data/canopy`.
If direct download is blocked, use a SOCKS5 proxy (`pip install PySocks`):
```bash
python scripts/bootstrap_canopy.py \
--bbox 27.3,58.4,35.8,61.4 \
--output-dir data/canopy \
--proxy socks5://user:pass@host:port
```
The same URL can be passed via `CANOPY_PROXY` env var instead of `--proxy`.
The `/api/v1/landcover/path` endpoint samples all `.tif`/`.tiff` files under
`LANDCOVER_PATH` recursively. Canopy data is optional; when it is missing,
`canopy_height_m` is returned as `null`.
+11
View File
@@ -65,6 +65,17 @@ def test_canopy_selects_intersecting_tiles_from_geojson_index() -> None:
assert urls[0].endswith("/tile_a.tif")
def test_canopy_parse_socks5_proxy_url() -> None:
proxy = bootstrap_canopy.parse_socks5_proxy(
"socks5://proxy_user:secret@194.33.35.46:38599"
)
assert proxy.host == "194.33.35.46"
assert proxy.port == 38599
assert proxy.username == "proxy_user"
assert proxy.password == "secret"
assert proxy.endpoint == "socks5://194.33.35.46:38599"
def test_canopy_urls_file_ignores_comments(tmp_path: Path) -> None:
urls_file = tmp_path / "urls.txt"
urls_file.write_text(
+64
View File
@@ -4,11 +4,15 @@ from __future__ import annotations
import argparse
import json
import os
import socket
import time
from collections.abc import Iterable
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.error import HTTPError, URLError
from urllib.parse import unquote, urlparse
from urllib.request import Request, urlopen
META_CHM_V2_INDEX_BASE_URL = (
@@ -20,6 +24,56 @@ META_CHM_V2_INDEX_URL = f"{META_CHM_V2_INDEX_BASE_URL}/tiles.geojson"
USER_AGENT = "RadioApi canopy bootstrap/0.1"
@dataclass(frozen=True)
class Socks5Proxy:
scheme: str
host: str
port: int
username: str | None
password: str | None
@property
def endpoint(self) -> str:
return f"{self.scheme}://{self.host}:{self.port}"
def parse_socks5_proxy(proxy_url: str) -> Socks5Proxy:
parsed = urlparse(proxy_url)
if parsed.scheme not in {"socks5", "socks5h"}:
raise SystemExit(
f"Unsupported proxy scheme: {parsed.scheme!r}. Use socks5:// or socks5h://"
)
if not parsed.hostname:
raise SystemExit("Proxy URL must include hostname, e.g. socks5://user:pass@host:port")
return Socks5Proxy(
scheme=parsed.scheme,
host=parsed.hostname,
port=parsed.port or 1080,
username=unquote(parsed.username) if parsed.username else None,
password=unquote(parsed.password) if parsed.password else None,
)
def configure_socks5_proxy(proxy_url: str) -> Socks5Proxy:
try:
import socks
except ImportError as exc:
raise SystemExit(
"SOCKS5 proxy requires PySocks. Install with: pip install PySocks"
) from exc
proxy = parse_socks5_proxy(proxy_url)
socks.set_default_proxy(
socks.SOCKS5,
proxy.host,
proxy.port,
username=proxy.username,
password=proxy.password,
)
socket.socket = socks.socksocket
return proxy
def parse_bbox(raw: str) -> tuple[float, float, float, float]:
parts = [float(part.strip()) for part in raw.split(",")]
if len(parts) != 4:
@@ -218,6 +272,12 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--urls-file", type=Path)
parser.add_argument("--skip-url-check", action="store_true")
parser.add_argument("--timeout-s", default=60, type=int)
parser.add_argument(
"--proxy",
default=os.environ.get("CANOPY_PROXY"),
help="SOCKS5 proxy URL, e.g. socks5://user:pass@host:port. "
"Default: CANOPY_PROXY env var",
)
parser.add_argument("--overwrite", action="store_true")
return parser.parse_args()
@@ -227,6 +287,10 @@ def main() -> None:
if args.urls_file is None and args.bbox is None:
raise SystemExit("--bbox is required unless --urls-file is provided")
if args.proxy:
proxy = configure_socks5_proxy(args.proxy)
print(f"using proxy {proxy.endpoint}")
if args.urls_file is not None:
urls = urls_from_file(args.urls_file)
else: