added conopy script proxy
This commit is contained in:
@@ -90,6 +90,17 @@ Or download manually into `data/landcover` and `data/canopy`. If the canopy tile
|
|||||||
index is unavailable, pass a newline-separated list of COG URLs with
|
index is unavailable, pass a newline-separated list of COG URLs with
|
||||||
`scripts/bootstrap_canopy.py --urls-file urls.txt --output-dir data/canopy`.
|
`scripts/bootstrap_canopy.py --urls-file urls.txt --output-dir data/canopy`.
|
||||||
|
|
||||||
|
If direct download is blocked, use a SOCKS5 proxy (`pip install PySocks`):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python scripts/bootstrap_canopy.py \
|
||||||
|
--bbox 27.3,58.4,35.8,61.4 \
|
||||||
|
--output-dir data/canopy \
|
||||||
|
--proxy socks5://user:pass@host:port
|
||||||
|
```
|
||||||
|
|
||||||
|
The same URL can be passed via `CANOPY_PROXY` env var instead of `--proxy`.
|
||||||
|
|
||||||
The `/api/v1/landcover/path` endpoint samples all `.tif`/`.tiff` files under
|
The `/api/v1/landcover/path` endpoint samples all `.tif`/`.tiff` files under
|
||||||
`LANDCOVER_PATH` recursively. Canopy data is optional; when it is missing,
|
`LANDCOVER_PATH` recursively. Canopy data is optional; when it is missing,
|
||||||
`canopy_height_m` is returned as `null`.
|
`canopy_height_m` is returned as `null`.
|
||||||
|
|||||||
@@ -65,6 +65,17 @@ def test_canopy_selects_intersecting_tiles_from_geojson_index() -> None:
|
|||||||
assert urls[0].endswith("/tile_a.tif")
|
assert urls[0].endswith("/tile_a.tif")
|
||||||
|
|
||||||
|
|
||||||
|
def test_canopy_parse_socks5_proxy_url() -> None:
|
||||||
|
proxy = bootstrap_canopy.parse_socks5_proxy(
|
||||||
|
"socks5://proxy_user:secret@194.33.35.46:38599"
|
||||||
|
)
|
||||||
|
assert proxy.host == "194.33.35.46"
|
||||||
|
assert proxy.port == 38599
|
||||||
|
assert proxy.username == "proxy_user"
|
||||||
|
assert proxy.password == "secret"
|
||||||
|
assert proxy.endpoint == "socks5://194.33.35.46:38599"
|
||||||
|
|
||||||
|
|
||||||
def test_canopy_urls_file_ignores_comments(tmp_path: Path) -> None:
|
def test_canopy_urls_file_ignores_comments(tmp_path: Path) -> None:
|
||||||
urls_file = tmp_path / "urls.txt"
|
urls_file = tmp_path / "urls.txt"
|
||||||
urls_file.write_text(
|
urls_file.write_text(
|
||||||
|
|||||||
@@ -4,11 +4,15 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
import time
|
import time
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from urllib.error import HTTPError, URLError
|
from urllib.error import HTTPError, URLError
|
||||||
|
from urllib.parse import unquote, urlparse
|
||||||
from urllib.request import Request, urlopen
|
from urllib.request import Request, urlopen
|
||||||
|
|
||||||
META_CHM_V2_INDEX_BASE_URL = (
|
META_CHM_V2_INDEX_BASE_URL = (
|
||||||
@@ -20,6 +24,56 @@ META_CHM_V2_INDEX_URL = f"{META_CHM_V2_INDEX_BASE_URL}/tiles.geojson"
|
|||||||
USER_AGENT = "RadioApi canopy bootstrap/0.1"
|
USER_AGENT = "RadioApi canopy bootstrap/0.1"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Socks5Proxy:
|
||||||
|
scheme: str
|
||||||
|
host: str
|
||||||
|
port: int
|
||||||
|
username: str | None
|
||||||
|
password: str | None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def endpoint(self) -> str:
|
||||||
|
return f"{self.scheme}://{self.host}:{self.port}"
|
||||||
|
|
||||||
|
|
||||||
|
def parse_socks5_proxy(proxy_url: str) -> Socks5Proxy:
|
||||||
|
parsed = urlparse(proxy_url)
|
||||||
|
if parsed.scheme not in {"socks5", "socks5h"}:
|
||||||
|
raise SystemExit(
|
||||||
|
f"Unsupported proxy scheme: {parsed.scheme!r}. Use socks5:// or socks5h://"
|
||||||
|
)
|
||||||
|
if not parsed.hostname:
|
||||||
|
raise SystemExit("Proxy URL must include hostname, e.g. socks5://user:pass@host:port")
|
||||||
|
return Socks5Proxy(
|
||||||
|
scheme=parsed.scheme,
|
||||||
|
host=parsed.hostname,
|
||||||
|
port=parsed.port or 1080,
|
||||||
|
username=unquote(parsed.username) if parsed.username else None,
|
||||||
|
password=unquote(parsed.password) if parsed.password else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def configure_socks5_proxy(proxy_url: str) -> Socks5Proxy:
|
||||||
|
try:
|
||||||
|
import socks
|
||||||
|
except ImportError as exc:
|
||||||
|
raise SystemExit(
|
||||||
|
"SOCKS5 proxy requires PySocks. Install with: pip install PySocks"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
proxy = parse_socks5_proxy(proxy_url)
|
||||||
|
socks.set_default_proxy(
|
||||||
|
socks.SOCKS5,
|
||||||
|
proxy.host,
|
||||||
|
proxy.port,
|
||||||
|
username=proxy.username,
|
||||||
|
password=proxy.password,
|
||||||
|
)
|
||||||
|
socket.socket = socks.socksocket
|
||||||
|
return proxy
|
||||||
|
|
||||||
|
|
||||||
def parse_bbox(raw: str) -> tuple[float, float, float, float]:
|
def parse_bbox(raw: str) -> tuple[float, float, float, float]:
|
||||||
parts = [float(part.strip()) for part in raw.split(",")]
|
parts = [float(part.strip()) for part in raw.split(",")]
|
||||||
if len(parts) != 4:
|
if len(parts) != 4:
|
||||||
@@ -218,6 +272,12 @@ def parse_args() -> argparse.Namespace:
|
|||||||
parser.add_argument("--urls-file", type=Path)
|
parser.add_argument("--urls-file", type=Path)
|
||||||
parser.add_argument("--skip-url-check", action="store_true")
|
parser.add_argument("--skip-url-check", action="store_true")
|
||||||
parser.add_argument("--timeout-s", default=60, type=int)
|
parser.add_argument("--timeout-s", default=60, type=int)
|
||||||
|
parser.add_argument(
|
||||||
|
"--proxy",
|
||||||
|
default=os.environ.get("CANOPY_PROXY"),
|
||||||
|
help="SOCKS5 proxy URL, e.g. socks5://user:pass@host:port. "
|
||||||
|
"Default: CANOPY_PROXY env var",
|
||||||
|
)
|
||||||
parser.add_argument("--overwrite", action="store_true")
|
parser.add_argument("--overwrite", action="store_true")
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
@@ -227,6 +287,10 @@ def main() -> None:
|
|||||||
if args.urls_file is None and args.bbox is None:
|
if args.urls_file is None and args.bbox is None:
|
||||||
raise SystemExit("--bbox is required unless --urls-file is provided")
|
raise SystemExit("--bbox is required unless --urls-file is provided")
|
||||||
|
|
||||||
|
if args.proxy:
|
||||||
|
proxy = configure_socks5_proxy(args.proxy)
|
||||||
|
print(f"using proxy {proxy.endpoint}")
|
||||||
|
|
||||||
if args.urls_file is not None:
|
if args.urls_file is not None:
|
||||||
urls = urls_from_file(args.urls_file)
|
urls = urls_from_file(args.urls_file)
|
||||||
else:
|
else:
|
||||||
|
|||||||
Reference in New Issue
Block a user