style: format code with pre-commit

Author: Bat-Reality

data_juicer/ops/mapper/generate_challenging_qa_mapper.py

@@ -6,10 +6,10 @@
 from data_juicer.utils.lazy_loader import LazyLoader
 from data_juicer.utils.model_utils import get_model, prepare_model
 
-torch = LazyLoader('torch', 'torch')
-vllm = LazyLoader('vllm', 'vllm')
+torch = LazyLoader("torch", "torch")
+vllm = LazyLoader("vllm", "vllm")
 
-OP_NAME = 'generate_challenging_qa_mapper'
+OP_NAME = "generate_challenging_qa_mapper"
 
 
 def retry_on_error(func, max_retries=5, delay=1):
@@ -28,7 +28,7 @@ def wrapper(*args, **kwargs):
                 return func(*args, **kwargs)
             except Exception as e:
                 retries += 1
-                print(f'Error: {e}, retry {retries}/{max_retries}...')
+                print(f"Error: {e}, retry {retries}/{max_retries}...")
                 if retries >= max_retries:
                     raise
                 time.sleep(delay)
@@ -47,14 +47,16 @@ class GenerateChallengingQAMapper(Mapper):
     It helps AI models autonomously create high-quality, reasoning-focused ​​QA pairs​​ without human input.
     """
 
-    _accelerator = 'cuda'
+    _accelerator = "cuda"
 
-    def __init__(self,
-                 hf_model: str = 'Qwen/Qwen2.5-VL-7B-Instruct',
-                 category: str = 'Mathematical Reasoning',
-                 model_name: str = 'Qwen',
-                 *args,
-                 **kwargs):
+    def __init__(
+        self,
+        hf_model: str = "Qwen/Qwen2.5-VL-7B-Instruct",
+        category: str = "Mathematical Reasoning",
+        model_name: str = "Qwen",
+        *args,
+        **kwargs,
+    ):
         """
         Initialization method.
 
@@ -99,8 +101,7 @@ def __init__(self,
         """
         super().__init__(*args, **kwargs)
         self.hf_model = hf_model
-        self.model_key = prepare_model(model_type='huggingface',
-                                       pretrained_model_name_or_path=hf_model)
+        self.model_key = prepare_model(model_type="huggingface", pretrained_model_name_or_path=hf_model)
         self.category = category
         self.model_name = model_name
         self.system_prompt = system_prompt
@@ -111,18 +112,14 @@ def __init__(self,
 
         # tensor_parallel_size = torch.cuda.device_count()
         model_params = {}
-        model_params['tensor_parallel_size'] = 4
-        self.model_key = prepare_model(model_type='vllm',
-                                       pretrained_model_name_or_path=hf_model,
-                                       **model_params)
-        self.sampling_params = vllm.SamplingParams(temperature=0.9,
-                                                   top_p=0.95,
-                                                   top_k=40,
-                                                   repetition_penalty=1.1,
-                                                   max_tokens=2048)
+        model_params["tensor_parallel_size"] = 4
+        self.model_key = prepare_model(model_type="vllm", pretrained_model_name_or_path=hf_model, **model_params)
+        self.sampling_params = vllm.SamplingParams(
+            temperature=0.9, top_p=0.95, top_k=40, repetition_penalty=1.1, max_tokens=2048
+        )
 
     def extract_json(self, text):
-        pattern = r'```json\s*(\{.*?\})\s*```'
+        pattern = r"```json\s*(\{.*?\})\s*```"
 
         match = re.search(pattern, text, re.DOTALL)
 
@@ -132,71 +129,55 @@ def extract_json(self, text):
                 json_data = json.loads(json_str)
                 return json_data
             except json.JSONDecodeError as e:
-                print(f'JSON parse error: {e}')
+                print(f"JSON parse error: {e}")
                 return None
         else:
-            print('None of valid JSON data')
+            print("None of valid JSON data")
             return None
 
     @retry_on_error
     def process_single(self, sample=None, rank=None):
 
         if self.category is None:
-            print(
-                'This OP requires processing multiple fields, and you need to specify valid `category`'
-            )
+            print("This OP requires processing multiple fields, and you need to specify valid `category`")
 
         model, _ = get_model(self.model_key, rank, self.use_cuda())
 
-        messages = [{
-            'role':
-            'system',
-            'content':
-            self.system_prompt.replace('Qwen', self.model_name)
-        }, {
-            'role':
-            'user',
-            'content':
-            self.user_prompt_background.format(category=self.category).replace(
-                'Qwen', self.model_name)
-        }]
+        messages = [
+            {"role": "system", "content": self.system_prompt.replace("Qwen", self.model_name)},
+            {
+                "role": "user",
+                "content": self.user_prompt_background.format(category=self.category).replace("Qwen", self.model_name),
+            },
+        ]
         background = model.chat(messages, self.sampling_params)
 
-        messages.extend([{
-            'role': 'system',
-            'content': background[0].outputs[0].text
-        }, {
-            'role':
-            'user',
-            'content':
-            self.user_prompt_subquestion.replace('Qwen', self.model_name)
-        }])
+        messages.extend(
+            [
+                {"role": "system", "content": background[0].outputs[0].text},
+                {"role": "user", "content": self.user_prompt_subquestion.replace("Qwen", self.model_name)},
+            ]
+        )
         sub_questions = model.chat(messages, self.sampling_params)
 
-        messages.extend([{
-            'role': 'system',
-            'content': sub_questions[0].outputs[0].text
-        }, {
-            'role':
-            'user',
-            'content':
-            self.user_prompt_multihop.replace('Qwen', self.model_name)
-        }])
+        messages.extend(
+            [
+                {"role": "system", "content": sub_questions[0].outputs[0].text},
+                {"role": "user", "content": self.user_prompt_multihop.replace("Qwen", self.model_name)},
+            ]
+        )
         multihop = model.chat(messages, self.sampling_params)
 
-        messages.extend([{
-            'role': 'system',
-            'content': multihop[0].outputs[0].text
-        }, {
-            'role':
-            'user',
-            'content':
-            self.extract_prompt_qa.replace('Qwen', self.model_name)
-        }])
+        messages.extend(
+            [
+                {"role": "system", "content": multihop[0].outputs[0].text},
+                {"role": "user", "content": self.extract_prompt_qa.replace("Qwen", self.model_name)},
+            ]
+        )
         qa = model.chat(messages, self.sampling_params)
 
         qa = self.extract_json(qa[0].outputs[0].text)
-        qa['thinking'] = multihop[0].outputs[0].text
+        qa["thinking"] = multihop[0].outputs[0].text
 
         sample.clear()
         sample.update(qa)

docs/Operators.md

@@ -46,7 +46,7 @@ Data-Juicer 中的算子分为以下 7 种类型。
 | [filter](#filter) | 54 | Filters out low-quality samples. 过滤低质量样本。 |
 | [formatter](#formatter) | 8 | Discovers, loads, and canonicalizes source data. 发现、加载、规范化原始数据。 |
 | [grouper](#grouper) | 3 | Group samples to batched samples. 将样本分组，每一组组成一个批量样本。 |
-| [mapper](#mapper) | 87 | Edits and transforms samples. 对数据样本进行编辑和转换。 |
+| [mapper](#mapper) | 88 | Edits and transforms samples. 对数据样本进行编辑和转换。 |
 | [selector](#selector) | 5 | Selects top samples based on ranking. 基于排序选取高质量样本。 |
 
 All the specific operators are listed below, each featured with several capability tags. 
@@ -205,6 +205,7 @@ All the specific operators are listed below, each featured with several capabili
 | extract_support_text_mapper | 🔤Text 💻CPU 🔗API 🟢Stable | Extracts a supporting sub-text from the original text based on a given summary. 根据给定的摘要从原始文本中提取支持子文本。 | [info](operators/mapper/extract_support_text_mapper.md) | - |
 | extract_tables_from_html_mapper | 🔤Text 💻CPU 🟡Beta | Extracts tables from HTML content and stores them in a specified field. 从HTML内容中提取表并将其存储在指定字段中。 | [info](operators/mapper/extract_tables_from_html_mapper.md) | - |
 | fix_unicode_mapper | 🔤Text 💻CPU 🟢Stable | Fixes unicode errors in text samples. 修复文本示例中的unicode错误。 | [info](operators/mapper/fix_unicode_mapper.md) | - |
+| generate_challenging_qa_mapper | 🚀GPU 🧩HF 🌊vLLM 🟡Beta | Mapper to generate self-challenging question and answer pairs. 映射器生成自我挑战的问题和答案对。 | - | - |
 | generate_qa_from_examples_mapper | 🚀GPU 🌊vLLM 🧩HF 🟢Stable | Generates question and answer pairs from examples using a Hugging Face model. 使用拥抱面部模型从示例生成问题和答案对。 | [info](operators/mapper/generate_qa_from_examples_mapper.md) | - |
 | generate_qa_from_text_mapper | 🔤Text 🚀GPU 🌊vLLM 🧩HF 🟢Stable | Generates question and answer pairs from text using a specified model. 使用指定的模型从文本生成问题和答案对。 | [info](operators/mapper/generate_qa_from_text_mapper.md) | - |
 | image_blur_mapper | 🏞Image 💻CPU 🟢Stable | Blurs images in the dataset with a specified probability and blur type. 使用指定的概率和模糊类型对数据集中的图像进行模糊处理。 | [info](operators/mapper/image_blur_mapper.md) | - |