Add -A/--user-agent to dvx import-url --git

Persists `user_agent` in `.dvc` dep so `dvx update` reuses it
for HEAD/GET requests. Needed for sites with bot protection
(e.g. Cloudflare).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: ryan-williams

specs/done/configurable-user-agent.md

@@ -0,0 +1,100 @@
+# Configurable User-Agent for HTTP imports
+
+## Context
+
+`dvx import-url --git` sends `User-Agent: dvx/0.1` which gets 403'd by sites with bot protection (e.g. `njsp.njoag.gov` uses Cloudflare). A browser-like User-Agent works fine via `curl -H "User-Agent: Mozilla/5.0 ..."`.
+
+## Proposed behavior
+
+### 1. CLI flag: `--user-agent` / `-A`
+
+```bash
+dvx import-url --git -A "Mozilla/5.0" \
+  https://njsp.njoag.gov/.../2024-UCR.xlsx \
+  -o crime/2024-UCR.xlsx
+```
+
+### 2. Stored in `.dvc` file
+
+The User-Agent is needed for subsequent `dvx update` calls too, so persist it in the `.dvc` deps:
+
+```yaml
+deps:
+- path: https://njsp.njoag.gov/.../2024-UCR.xlsx
+  checksum: '"etag"'
+  size: 204114
+  mtime: '2026-02-24T00:00:00+00:00'
+  user_agent: 'Mozilla/5.0 (compatible; dvx/0.1)'
+outs:
+- md5: e2154bc8...
+  path: 2024-UCR.xlsx
+meta:
+  git_tracked: true
+```
+
+`dvx update` reads `user_agent` from the dep and uses it for HEAD/GET requests.
+
+### 3. Global config fallback
+
+```bash
+dvx config http.user_agent "Mozilla/5.0 (compatible; dvx/0.1)"
+```
+
+Stored in `.dvc/config` (or `.dvc/config.local`). Per-dep `user_agent` in `.dvc` overrides the global config.
+
+### 4. Default
+
+Keep `dvx/0.1` as the default (honest about what we are). Only override when needed.
+
+## Implementation
+
+### `src/dvx/git_import.py`
+
+```python
+def _get_headers(user_agent: str | None = None) -> dict:
+    ua = user_agent or dvc_config_get("http.user_agent", "dvx/0.1")
+    return {"User-Agent": ua}
+
+def git_import_url(url, out, no_download=False, user_agent=None):
+    headers = _get_headers(user_agent)
+    req = Request(url, headers=headers)
+    # ... download ...
+    # Store user_agent in dep if non-default
+    dep_info = {"path": url, ...}
+    if user_agent:
+        dep_info["user_agent"] = user_agent
+
+def update_git_import(dvc_path, no_download=False):
+    # Read user_agent from existing dep
+    dep = load_dvc(dvc_path)["deps"][0]
+    user_agent = dep.get("user_agent")
+    headers = _get_headers(user_agent)
+    # ... HEAD/GET with headers ...
+```
+
+### `src/dvx/cli/external.py`
+
+Add `-A`/`--user-agent` option to `import-url` and `update`.
+
+### DVC core (`dvc_data`)
+
+For non-git-tracked HTTP imports (regular `import-url`), DVC uses fsspec's `HTTPFileSystem`. User-Agent can be passed via `client_kwargs`:
+
+```python
+fs = HTTPFileSystem(client_kwargs={"headers": {"User-Agent": ua}})
+```
+
+This is already configurable via `--fs-config` but that's verbose. The `http.user_agent` config key would be a convenience.
+
+## Scope
+
+Minimal: just `git_import.py` + CLI flag + `.dvc` persistence. The global config and fsspec integration can follow.
+
+## Implementation status
+
+Items 1, 2, 4 done:
+- `-A`/`--user-agent` flag on `dvx import-url` (passed through to `git_import_url()`)
+- `user_agent` persisted in `.dvc` dep; `update_git_import()` reads it back for subsequent requests
+- Default remains `dvx/0.1`
+
+Item 3 (global config fallback) deferred — not needed for the immediate use case.

src/dvx/cli/external.py

@@ -27,25 +27,27 @@ def import_cmd(url, path, out, rev):
 
 @click.command("import-url")
 @click.argument("url")
+@click.option("-A", "--user-agent", help="Custom User-Agent header (persisted for updates).")
 @click.option("-F", "--fs-config", multiple=True, help="Filesystem config (key=value).")
 @click.option("-G", "--git", is_flag=True, help="Track in Git (not DVC cache). For small files.")
 @click.option("-N", "--no-download", is_flag=True, help="Track metadata only (no download).")
 @click.option("-o", "--out", help="Output path.")
 @click.option("-V", "--version-aware", is_flag=True, help="Track S3 version IDs.")
-def import_url(url, fs_config, git, no_download, out, version_aware):
+def import_url(url, user_agent, fs_config, git, no_download, out, version_aware):
     """Import a file from a URL.
 
     Use --git to commit the file to Git (instead of DVC cache) with URL
     provenance. Good for small files (< 1MB) you want in the repo.
 
     Use --no-download to track metadata (ETag, size) without downloading.
     Use --fs-config allow_anonymous_login=true for public buckets.
+    Use --user-agent to set a custom User-Agent (needed for some sites).
     """
     if git:
         from dvx.git_import import git_import_url
 
         try:
-            dvc_path = git_import_url(url=url, out=out, no_download=no_download)
+            dvc_path = git_import_url(url=url, out=out, no_download=no_download, user_agent=user_agent)
             action = "Tracked" if no_download else "Imported"
             click.echo(f"{action} {url} (git-tracked)")
             click.echo(f"  {dvc_path}")

src/dvx/git_import.py

@@ -13,6 +13,8 @@
 
 import yaml
 
+DEFAULT_USER_AGENT = "dvx/0.1"
+
 
 def _default_out(url: str) -> str:
     """Derive output filename from URL path."""
@@ -23,12 +25,17 @@ def _default_out(url: str) -> str:
     return name
 
 
-def _download(url: str, out: Path) -> tuple[str, int, dict[str, str]]:
+def _download(
+    url: str,
+    out: Path,
+    user_agent: str | None = None,
+) -> tuple[str, int, dict[str, str]]:
     """Download URL to `out`, returning (md5, size, http_headers).
 
     Headers dict includes ETag, Last-Modified, Content-Length if present.
     """
-    req = Request(url, headers={"User-Agent": "dvx/0.1"})
+    ua = user_agent or DEFAULT_USER_AGENT
+    req = Request(url, headers={"User-Agent": ua})
     with urlopen(req) as resp:  # noqa: S310
         data = resp.read()
         headers = {
@@ -44,9 +51,10 @@ def _download(url: str, out: Path) -> tuple[str, int, dict[str, str]]:
     return md5, len(data), headers
 
 
-def _head_metadata(url: str) -> dict[str, str]:
+def _head_metadata(url: str, user_agent: str | None = None) -> dict[str, str]:
     """HEAD request to get ETag/Last-Modified without downloading."""
-    req = Request(url, method="HEAD", headers={"User-Agent": "dvx/0.1"})
+    ua = user_agent or DEFAULT_USER_AGENT
+    req = Request(url, method="HEAD", headers={"User-Agent": ua})
     with urlopen(req) as resp:  # noqa: S310
         return {
             k: resp.headers[k]
@@ -69,6 +77,7 @@ def _build_dvc_data(
     size: int,
     headers: dict[str, str],
     out_name: str,
+    user_agent: str | None = None,
 ) -> dict:
     """Build the .dvc YAML structure for a git-tracked import."""
     dep: dict = {"path": url}
@@ -78,6 +87,8 @@ def _build_dvc_data(
         dep["size"] = size
     if "Last-Modified" in headers:
         dep["mtime"] = _parse_last_modified(headers["Last-Modified"])
+    if user_agent:
+        dep["user_agent"] = user_agent
 
     out_entry = {
         "md5": md5,
@@ -101,6 +112,7 @@ def git_import_url(
     url: str,
     out: str | None = None,
     no_download: bool = False,
+    user_agent: str | None = None,
 ) -> Path:
     """Import a URL as a git-tracked file with DVX provenance.
 
@@ -111,22 +123,21 @@ def git_import_url(
         url: HTTP(S) URL to import.
         out: Output path (default: derived from URL filename).
         no_download: If True, only create .dvc with metadata (HEAD request).
+        user_agent: Custom User-Agent header (persisted in .dvc for updates).
 
     Returns:
         Path to the created .dvc file.
     """
     out_path = Path(out or _default_out(url))
 
     if no_download:
-        headers = _head_metadata(url)
+        headers = _head_metadata(url, user_agent=user_agent)
         size = int(headers.get("Content-Length", 0))
-        # No file to hash; leave md5 empty
-        dvc_data = _build_dvc_data(url, "", size, headers, out_path.name)
-        # Remove empty md5 from outs
+        dvc_data = _build_dvc_data(url, "", size, headers, out_path.name, user_agent=user_agent)
         del dvc_data["outs"][0]["md5"]
     else:
-        md5, size, headers = _download(url, out_path)
-        dvc_data = _build_dvc_data(url, md5, size, headers, out_path.name)
+        md5, size, headers = _download(url, out_path, user_agent=user_agent)
+        dvc_data = _build_dvc_data(url, md5, size, headers, out_path.name, user_agent=user_agent)
 
     dvc_path = Path(str(out_path) + ".dvc")
     dvc_path.parent.mkdir(parents=True, exist_ok=True)
@@ -178,29 +189,30 @@ def update_git_import(dvc_path: Path, no_download: bool = False) -> bool:
     if not meta.get("git_tracked"):
         return False
 
-    url = data["deps"][0]["path"]
-    old_checksum = data["deps"][0].get("checksum")
+    dep = data["deps"][0]
+    url = dep["path"]
+    old_checksum = dep.get("checksum")
+    user_agent = dep.get("user_agent")
     out_name = data["outs"][0]["path"]
     out_path = dvc_path.parent / out_name
 
     if no_download:
-        headers = _head_metadata(url)
+        headers = _head_metadata(url, user_agent=user_agent)
         new_checksum = headers.get("ETag")
         if new_checksum and new_checksum == old_checksum:
             return False
         size = int(headers.get("Content-Length", 0))
-        new_data = _build_dvc_data(url, "", size, headers, out_name)
+        new_data = _build_dvc_data(url, "", size, headers, out_name, user_agent=user_agent)
         if "md5" in data["outs"][0]:
             new_data["outs"][0]["md5"] = data["outs"][0]["md5"]
         else:
             del new_data["outs"][0]["md5"]
     else:
-        md5, size, headers = _download(url, out_path)
+        md5, size, headers = _download(url, out_path, user_agent=user_agent)
         new_checksum = headers.get("ETag")
         if new_checksum and new_checksum == old_checksum:
-            # ETag unchanged but we already downloaded; keep the new file
             pass
-        new_data = _build_dvc_data(url, md5, size, headers, out_name)
+        new_data = _build_dvc_data(url, md5, size, headers, out_name, user_agent=user_agent)
 
     with open(dvc_path, "w") as f:
         yaml.dump(new_data, f, sort_keys=False, default_flow_style=False)