Skip to content

Commit 1891699

Browse files
ryan-williamsclaude
andcommitted
Add tests and docs for recursive run, stage output, error output
Tests: - `test_run_discovers_dvc_files_recursively`: verifies `dvx run` finds .dvc files in subdirs and excludes `.dvc/` directory - `test_failed_stage_exit_code_and_log`: verifies error output includes exit code and creates log file - `test_summary_file_output`: verifies `$DVX_SUMMARY_FILE` works - `test_env_vars_are_set`: verifies both env vars are set README: per-stage commits, error output, `--commit` flag, recursive discovery. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 0677f62 commit 1891699

3 files changed

Lines changed: 148 additions & 1 deletion

File tree

README.md

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ pip install dvx[all]
191191
### Running Pipelines
192192

193193
```bash
194-
# Run all .dvc computations (parallel by default)
194+
# Run all .dvc computations (recursive discovery, parallel)
195195
dvx run
196196

197197
# Run specific target
@@ -205,6 +205,33 @@ dvx run --dry-run
205205

206206
# Force re-run (ignore freshness)
207207
dvx run --force
208+
209+
# Auto-commit after each stage
210+
dvx run --commit
211+
```
212+
213+
#### Per-Stage Commits
214+
215+
Stages can trigger commits by writing to `$DVX_COMMIT_MSG_FILE` (set by DVX before each cmd):
216+
217+
```bash
218+
# In your stage script:
219+
echo "Refresh data: 5 new records" > "$DVX_COMMIT_MSG_FILE"
220+
```
221+
222+
With `dvx run --commit`, stages that don't write a commit message get a default one (e.g. "Run refresh"). DVX also sets `$DVX_SUMMARY_FILE` for short status output.
223+
224+
#### Error Output
225+
226+
When a stage fails, DVX shows the exit code, last 20 lines of stderr, and saves the full log:
227+
228+
```
229+
✗ refresh: failed (exit code 1)
230+
231+
stderr (last 20 lines):
232+
ConnectionError: Failed to fetch https://...
233+
234+
Full output: tmp/dvx-run-refresh.log
208235
```
209236

210237
### Tracking Data
@@ -309,6 +336,8 @@ with Repo() as repo:
309336
- Fetch schedules - Periodic re-fetch with daily/hourly/weekly/cron staleness
310337
- Directory dependencies - Git tree SHA tracking for `git_deps`
311338
- `dvx import-url --git` - Git-tracked imports with URL provenance
339+
- Per-stage commits - `$DVX_COMMIT_MSG_FILE` env var + `--commit` flag
340+
- Detailed error output - Exit code, stderr tail, log file on failure
312341
- `dvx diff` preprocessing - Pipe through commands before diffing (with `{}` placeholder)
313342
- `dvx cache path/md5` - Cache introspection
314343
- `dvx cat` - View cached files directly

tests/test_cli.py

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -421,3 +421,37 @@ def test_status_dep_changed(runner, temp_dvc_repo):
421421
assert output_line.startswith("✗")
422422
assert "dep changed" in output_line
423423
assert "input.txt" in output_line
424+
425+
426+
def test_run_discovers_dvc_files_recursively(runner, tmp_path):
427+
"""Test that `dvx run` with no targets finds .dvc files in subdirectories."""
428+
os.chdir(tmp_path)
429+
430+
# Create .dvc files in nested subdirectories
431+
sub1 = tmp_path / "sub1"
432+
sub1.mkdir()
433+
sub2 = tmp_path / "sub1" / "sub2"
434+
sub2.mkdir()
435+
436+
for d, name in [(tmp_path, "top.txt"), (sub1, "mid.txt"), (sub2, "deep.txt")]:
437+
dvc_content = {
438+
"outs": [{"md5": "", "size": 0, "path": name}],
439+
"meta": {"computation": {"cmd": f"echo {name} > {name}"}},
440+
}
441+
dvc_file = d / f"{name}.dvc"
442+
with open(dvc_file, "w") as f:
443+
yaml.dump(dvc_content, f)
444+
445+
# Also create a .dvc/config dir to make sure .dvc/ directory files are excluded
446+
dvc_dir = tmp_path / ".dvc"
447+
dvc_dir.mkdir()
448+
spurious = dvc_dir / "something.dvc"
449+
spurious.write_text("should be ignored")
450+
451+
result = runner.invoke(cli, ["run", "--dry-run"])
452+
assert result.exit_code == 0
453+
454+
# All three .dvc files should be discovered
455+
assert "top.txt" in result.output
456+
assert "mid.txt" in result.output
457+
assert "deep.txt" in result.output

tests/test_executor.py

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -296,3 +296,87 @@ def test_run_with_git_deps_in_dvc_file(tmp_workdir):
296296

297297
assert len(results) == 1
298298
assert results[0].success
299+
300+
301+
def test_failed_stage_exit_code_and_log(tmp_workdir):
302+
"""Test that a failing stage records exit code in reason and writes a log file."""
303+
artifact = Artifact(
304+
path=str(tmp_workdir / "fail_output.txt"),
305+
computation=Computation(cmd="echo fail_msg >&2 && exit 42", deps=[]),
306+
)
307+
308+
output = StringIO()
309+
config = ExecutionConfig()
310+
executor = ParallelExecutor([artifact], config, output)
311+
results = executor.execute()
312+
313+
assert len(results) == 1
314+
result = results[0]
315+
assert not result.success
316+
assert "fail_msg" in result.reason
317+
318+
# Log file should exist in tmp/
319+
log_path = tmp_workdir / "tmp" / "dvx-run-fail_output.log"
320+
assert log_path.exists()
321+
log_content = log_path.read_text()
322+
assert "fail_msg" in log_content
323+
324+
325+
def test_summary_file_output(tmp_workdir):
326+
"""Test that a stage writing to $DVX_SUMMARY_FILE has its summary shown."""
327+
output_path = tmp_workdir / "summary_test.txt"
328+
329+
cmd = f'echo result > {output_path} && echo "Stage completed successfully" > "$DVX_SUMMARY_FILE"'
330+
331+
artifact = Artifact(
332+
path=str(output_path),
333+
computation=Computation(cmd=cmd, deps=[]),
334+
)
335+
336+
output = StringIO()
337+
config = ExecutionConfig()
338+
executor = ParallelExecutor([artifact], config, output)
339+
results = executor.execute()
340+
341+
assert len(results) == 1
342+
assert results[0].success
343+
344+
log_output = output.getvalue()
345+
assert "Stage completed successfully" in log_output
346+
347+
348+
def test_env_vars_are_set(tmp_workdir):
349+
"""Test that $DVX_COMMIT_MSG_FILE and $DVX_SUMMARY_FILE are set to non-empty paths."""
350+
output_path = tmp_workdir / "env_test.txt"
351+
env_dump = tmp_workdir / "env_dump.txt"
352+
353+
cmd = (
354+
f'echo "COMMIT=$DVX_COMMIT_MSG_FILE" > {env_dump} && '
355+
f'echo "SUMMARY=$DVX_SUMMARY_FILE" >> {env_dump} && '
356+
f'echo ok > {output_path}'
357+
)
358+
359+
artifact = Artifact(
360+
path=str(output_path),
361+
computation=Computation(cmd=cmd, deps=[]),
362+
)
363+
364+
output = StringIO()
365+
config = ExecutionConfig()
366+
executor = ParallelExecutor([artifact], config, output)
367+
results = executor.execute()
368+
369+
assert len(results) == 1
370+
assert results[0].success
371+
372+
env_content = env_dump.read_text()
373+
lines = env_content.strip().split("\n")
374+
commit_line = [l for l in lines if l.startswith("COMMIT=")][0]
375+
summary_line = [l for l in lines if l.startswith("SUMMARY=")][0]
376+
377+
commit_val = commit_line.split("=", 1)[1]
378+
summary_val = summary_line.split("=", 1)[1]
379+
380+
assert commit_val != "", "DVX_COMMIT_MSG_FILE should be non-empty"
381+
assert summary_val != "", "DVX_SUMMARY_FILE should be non-empty"
382+
assert commit_val != summary_val, "Commit and summary files should be different paths"

0 commit comments

Comments
 (0)