Merge branch 'main' into op-MindGYM

Author: HYLcool

.github/workflows/publish-pypi.yml

@@ -30,9 +30,9 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools
+        pip install build setuptools
     - name: Build package
-      run: python setup.py sdist bdist_wheel
+      run: python -m build
     - name: Publish package
       uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
       with:

.github/workflows/unit-test-partial.yml

@@ -6,9 +6,6 @@ name: unittest-partial
 on:
   workflow_dispatch:
   pull_request:
-  push:
-    branches:
-      - main
 
 permissions:
   contents: read
@@ -36,6 +33,12 @@ jobs:
       run: |
         docker compose exec ray-head bash -c '/root/.local/bin/uv pip install --system -e .\[all\]'
 
+    - name: Print Pip Dependency Tree
+      working-directory: dj-${{ github.run_id }}/.github/workflows/docker
+      run: |
+        docker compose exec ray-head bash -c '/root/.local/bin/uv pip install --system pipdeptree'
+        docker compose exec ray-head bash -c 'pipdeptree'
+
     - name: Clean dataset cache
       working-directory: dj-${{ github.run_id }}/.github/workflows/docker
       run: |

.github/workflows/unit-test.yml

@@ -37,6 +37,12 @@ jobs:
         docker compose exec ray-head bash -c '/root/.local/bin/uv pip install --system -e .\[all\]'
         docker compose exec ray-worker bash -c '/root/.local/bin/uv pip install --system -e .\[all\]'
 
+    - name: Print Pip Dependency Tree
+      working-directory: dj-${{ github.run_id }}/.github/workflows/docker
+      run: |
+        docker compose exec ray-head bash -c '/root/.local/bin/uv pip install --system pipdeptree'
+        docker compose exec ray-head bash -c 'pipdeptree'
+
     - name: Clean dataset cache
       working-directory: dj-${{ github.run_id }}/.github/workflows/docker
       run: |

.gitignore

@@ -15,6 +15,7 @@ dist
 wandb/
 __pycache__
 .vscode/
+.ipynb_checkpoints/
 
 # label studio related
 label_studio_data/
@@ -30,5 +31,3 @@ tests/ops/data/*dup*
 tests/tools/tmp_*/
 tests/ops/deduplicator/chinese_dedup/
 tests/ops/deduplicator/english_dedup/
-
-.ipynb_checkpoints/

.pre-commit-config.yaml

@@ -1,22 +1,12 @@
 repos:
-  - repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
-    hooks:
-      - id: flake8
-        additional_dependencies:
-          - flake8-pyproject
-          - pycodestyle<2.12.0
   - repo: https://github.com/PyCQA/isort.git
     rev: 5.12.0
     hooks:
       - id: isort
-  - repo: https://github.com/pre-commit/mirrors-yapf
-    rev: v0.32.0
+  - repo: https://github.com/psf/black
+    rev: 25.1.0
     hooks:
-      - id: yapf
-        exclude: data_juicer/ops/common/special_characters.py
-        additional_dependencies:
-          - toml
+      - id: black
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:
@@ -28,8 +18,6 @@ repos:
         exclude: thirdparty/
       - id: requirements-txt-fixer
         exclude: thirdparty/
-      - id: double-quote-string-fixer
-        exclude: ^(thirdparty/|data_juicer/ops/common/special_characters.py)
       - id: check-merge-conflict
         exclude: thirdparty/
       - id: fix-encoding-pragma
@@ -38,6 +26,13 @@ repos:
       - id: mixed-line-ending
         exclude: thirdparty/
         args: [ "--fix=lf" ]
+  - repo: https://github.com/PyCQA/flake8
+    rev: 6.1.0
+    hooks:
+      - id: flake8
+        additional_dependencies:
+          - flake8-pyproject
+          - flake8-black
   - repo: local
     hooks:
       - id: build-op-doc