feat(experimental): support merge reports for non-sharded multi-environment runs (take 2)

.github/workflows/ci.yml

@@ -116,6 +116,8 @@ jobs:
 
       - name: Test
         run: pnpm run test:ci
+        env:
+          VITEST_CI_BLOB_LABEL: ${{ matrix.os }}-node-${{ matrix.node_version }}
 
       - name: Test Examples
         run: pnpm run test:examples
@@ -130,6 +132,17 @@ jobs:
           path: test/ui/test-results/
           retention-days: 30
 
+      - uses: actions/upload-artifact@v7
+        if: ${{ !cancelled() }}
+        with:
+          name: vitest-blob-${{ matrix.os }}-node-${{ matrix.node_version }}
+          path: |
+            README.md
+            test/core/.vitest-reports
+            test/cli/.vitest-reports
+          retention-days: 1
+          include-hidden-files: true
+
   test-cached:
     needs: changed
     name: 'Cache&Test: node-${{ matrix.node_version }}, ${{ matrix.os }}'
@@ -251,3 +264,48 @@ jobs:
           name: playwright-report-rolldown
           path: rolldown/test/ui/test-results/
           retention-days: 30
+
+  merge-reports:
+    needs: test
+    if: ${{ !cancelled() }}
+    runs-on: ubuntu-latest
+    name: Merge Reports
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: ./.github/actions/setup-and-cache
+
+      - name: Install
+        run: pnpm i
+
+      - name: Build
+        run: pnpm run build
+
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: vitest-blob-*
+          merge-multiple: true
+
+      - name: Merge reports
+        continue-on-error: true
+        run: pnpm --filter=./test/core --filter=./test/cli --no-bail --sequential test --merge-reports --reporter=html
+
+      - name: Merge reports html
+        id: merge-html
+        run: |
+          mkdir -p html-all
+          cp -rf test/core/html html-all/core
+          cp -rf test/cli/html html-all/cli
+          echo "short_sha=${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT
+
+      - uses: actions/upload-artifact@v7
+        id: upload-report
+        with:
+          name: vitest-ci-report-${{ steps.merge-html.outputs.short_sha }}
+          path: html-all
+          retention-days: 7
+
+      - name: Viewer link in summary
+        run: |
+          echo "[View HTML report](https://viewer.vitest.dev/?url=${{ steps.upload-report.outputs.artifact-url }})" >> $GITHUB_STEP_SUMMARY

docs/guide/cli.md

@@ -225,7 +225,7 @@ You cannot use this option with `--watch` enabled (enabled in dev by default).
 :::
 
 ::: tip
-If `--reporter=blob` is used without an output file, the default path will include the current shard config to avoid collisions with other Vitest processes.
+If `--reporter=blob` is used without an output file, the default path will include the current shard config and blob label from `VITEST_BLOB_LABEL` or the blob reporter `label` option to avoid collisions with other Vitest processes.
 :::
 
 ### merge-reports

docs/guide/improving-performance.md

@@ -132,6 +132,12 @@ Collect the results stored in `.vitest-reports` directory from each machine and
 vitest run --merge-reports
 ```
 
+When running the same shards across multiple environments, set the `VITEST_BLOB_LABEL` environment variable so merged reports can display them separately:
+
+```sh
+VITEST_BLOB_LABEL=linux vitest run --reporter=blob --shard=1/3
+```
+
 ::: details GitHub Actions example
 This setup is also used at https://github.com/vitest-tests/test-sharding.
 
@@ -144,9 +150,10 @@ on:
       - main
 jobs:
   tests:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        os: [ubuntu-latest, macos-latest]
         shardIndex: [1, 2, 3, 4]
         shardTotal: [4]
     steps:
@@ -163,12 +170,14 @@ jobs:
 
       - name: Run tests
         run: pnpm run test --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        env:
+          VITEST_BLOB_LABEL: ${{ matrix.os }}
 
       - name: Upload blob report to GitHub Actions Artifacts
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: blob-report-${{ matrix.shardIndex }}
+          name: blob-report-${{ matrix.os }}-${{ matrix.shardIndex }}
           path: .vitest-reports/*
           include-hidden-files: true
           retention-days: 1
@@ -177,7 +186,7 @@ jobs:
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: blob-attachments-${{ matrix.shardIndex }}
+          name: blob-attachments-${{ matrix.os }}-${{ matrix.shardIndex }}
           path: .vitest-attachments/**
           include-hidden-files: true
           retention-days: 1

docs/guide/reporters.md

@@ -687,13 +687,32 @@ By default, stores all results in `.vitest-reports` folder, but can be overridde
 npx vitest --reporter=blob --outputFile=reports/blob-1.json
 ```
 
-We recommend using this reporter if you are running Vitest on different machines with the [`--shard`](/guide/cli#shard) flag.
-All blob reports can be merged into any report by using `--merge-reports` command at the end of your CI pipeline:
+We recommend using this reporter if you are running Vitest on different machines with the [`--shard`](/guide/cli#shard) flag or across multiple environments (e.g., linux/macos/windows). All blob reports can be merged into any report by using `--merge-reports` command at the end of your CI pipeline:
 
 ```bash
 npx vitest --merge-reports=reports --reporter=json --reporter=default
 ```
 
+When running the same tests across multiple environments, use the `VITEST_BLOB_LABEL` environment variable to distinguish each environment's blob. Vitest reads labels at merge time and displays results separately:
+
+```bash
+VITEST_BLOB_LABEL=linux vitest run --reporter=blob
+```
+
+You can also provide the label via the blob reporter option:
+
+```ts [vitest.config.ts]
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({
+  test: {
+    reporters: [
+      ['blob', { label: 'linux' }],
+    ],
+  },
+})
+```
+
 Blob reporter output doesn't include file-based [attachments](/api/advanced/artifacts.html#testattachment).
 Make sure to merge [`attachmentsDir`](/config/attachmentsdir) separately alongside blob reports on CI when using this feature.
 

packages/runner/src/collect.ts

@@ -47,7 +47,14 @@ export async function collectTests(
 
         const fileTags: string[] = typeof spec === 'string' ? [] : (spec.fileTags || [])
 
-        const file = createFileTask(filepath, config.root, config.name, runner.pool, runner.viteEnvironment)
+        const file = createFileTask(
+          filepath,
+          config.root,
+          config.name,
+          runner.pool,
+          runner.viteEnvironment,
+          { __vitest_label__: config.mergeReportsLabel },
+        )
         file.tags = fileTags
         file.shuffle = config.sequence.shuffle
 

packages/runner/src/types/runner.ts

@@ -45,6 +45,7 @@ export interface VitestRunnerConfig {
   tags: TestTagDefinition[]
   tagsFilter: string[] | undefined
   strictTags: boolean
+  mergeReportsLabel: string | undefined
 }
 
 /**

packages/runner/src/utils/collect.ts

@@ -193,23 +193,29 @@ export function calculateSuiteHash(parent: Suite): void {
   })
 }
 
+interface HashMeta {
+  typecheck?: boolean
+  __vitest_label__?: string
+}
+
 export function createFileTask(
   filepath: string,
   root: string,
   projectName: string | undefined,
   pool?: string,
   viteEnvironment?: string,
+  meta?: HashMeta,
 ): File {
   const path = relative(root, filepath)
   const file: File = {
-    id: generateFileHash(path, projectName),
+    id: generateFileHash(path, projectName, meta),
     name: path,
     fullName: path,
     type: 'suite',
     mode: 'queued',
     filepath,
     tasks: [],
-    meta: Object.create(null),
+    meta: Object.assign(Object.create(null), meta),
     projectName,
     file: undefined!,
     pool,
@@ -228,8 +234,15 @@ export function createFileTask(
 export function generateFileHash(
   file: string,
   projectName: string | undefined,
+  meta?: HashMeta,
 ): string {
-  return /* @__PURE__ */ generateHash(`${file}${projectName || ''}`)
+  const seed = [
+    file,
+    projectName || '',
+    meta?.typecheck ? '__typecheck__' : '',
+    meta?.__vitest_label__ || '',
+  ].join('\0')
+  return generateHash(seed)
 }
 
 export function findTestFileStackTrace(testFilePath: string, error: string): ParsedStack | undefined {

packages/ui/client/components/FileDetails.vue

@@ -60,6 +60,8 @@ const isTypecheck = computed(() => {
   return !!current.value?.meta?.typecheck
 })
 
+const label = computed(() => current.value?.meta?.__vitest_label__)
+
 function open() {
   const filePath = current.value?.filepath
   if (filePath) {
@@ -206,6 +208,7 @@ const tags = computed(() => {
       <div p="2" h-10 flex="~ gap-2" items-center bg-header border="b base">
         <StatusIcon :state="current.result?.state" :mode="current.mode" :failed-snapshot="failedSnapshot" />
         <div v-if="isTypecheck" v-tooltip.bottom="'This is a typecheck test. It won\'t report results of the runtime tests'" class="i-logos:typescript-icon" flex-shrink-0 />
+        <span v-if="label" class="rounded-sm px-1 text-xs font-light bg-cyan-500/20 text-cyan-700 dark:text-cyan-300" flex-shrink-0>{{ label }}</span>
         <span
           v-if="current?.file.projectName"
           class="rounded-full py-0.5 px-2 text-xs font-light"

packages/ui/client/components/explorer/Explorer.vue

@@ -331,6 +331,7 @@ const {
                 :indent="item.indent"
                 :name="item.name"
                 :typecheck="item.typecheck === true"
+                :label="item.label"
                 :project-name="item.projectName ?? ''"
                 :project-name-color="item.projectNameColor ?? ''"
                 :state="item.state"

packages/ui/client/components/explorer/ExplorerItem.vue

@@ -25,6 +25,7 @@ const {
   opened,
   expandable,
   typecheck,
+  label,
   type,
   disableTaskLocation,
   onItemClick,
@@ -35,6 +36,7 @@ const {
   name: string
   indent: number
   typecheck?: boolean
+  label?: string
   duration?: number
   slow?: boolean
   state?: TaskState
@@ -216,6 +218,7 @@ const tagsBgGradient = computed(() => {
     <StatusIcon :state="state" :mode="task.mode" :failed-snapshot="failedSnapshot" w-4 />
     <div flex items-baseline gap-2 overflow-hidden>
       <div v-if="type === 'file' && typecheck" v-tooltip.bottom="'This is a typecheck test. It won\'t report results of the runtime tests'" class="i-logos:typescript-icon" flex-shrink-0 />
+      <span v-if="type === 'file' && label" class="rounded-sm px-1 text-xs font-light bg-cyan-500/20 text-cyan-700 dark:text-cyan-300" flex-shrink-0>{{ label }}</span>
       <span text-sm truncate font-light>
         <span v-if="type === 'file' && projectName" class="rounded-full py-0.5 px-2 mr-1 text-xs" :style="{ backgroundColor: projectNameColor, color: projectNameTextColor }">
           {{ projectName }}

packages/ui/client/components/views/ViewTestReport.vue

@@ -25,6 +25,7 @@ const failed = computed(() => {
 const kWellKnownMeta = new Set([
   'benchmark',
   'typecheck',
+  'label',
 ])
 const meta = computed(() => {
   return Object.entries(props.test.meta).filter(([name]) => {

packages/ui/client/composables/explorer/types.ts

@@ -63,6 +63,7 @@ export interface FileTreeNode extends ParentTreeNode {
   type: 'file'
   filepath: string
   typecheck: boolean | undefined
+  label?: string
   projectName?: string
   projectNameColor: string
   collectDuration?: number

packages/ui/client/composables/explorer/utils.ts

@@ -84,6 +84,7 @@ export function createOrUpdateFileNode(
 
   if (fileNode) {
     fileNode.typecheck = !!file.meta && 'typecheck' in file.meta
+    fileNode.label = file.meta?.__vitest_label__
     fileNode.state = file.result?.state
     fileNode.mode = file.mode
     fileNode.duration = typeof file.result?.duration === 'number' ? Math.round(file.result.duration) : undefined
@@ -106,6 +107,7 @@ export function createOrUpdateFileNode(
       children: new Set(),
       tasks: [],
       typecheck: !!file.meta && 'typecheck' in file.meta,
+      label: file.meta?.__vitest_label__,
       indent: 0,
       duration: typeof file.result?.duration === 'number' ? Math.round(file.result.duration) : undefined,
       slow: false,

packages/utils/src/helpers.ts

@@ -429,3 +429,8 @@ export function deepMerge<T extends object = object>(
 export function unique<T>(array: T[]): T[] {
   return Array.from(new Set(array))
 }
+
+export function sanitizeFilePath(s: string): string {
+  // eslint-disable-next-line no-control-regex
+  return s.replace(/[\x00-\x2C\x2E\x2F\x3A-\x40\x5B-\x60\x7B-\x7F]+/g, '-')
+}