mathjax渲染器方案逻辑修改

llm_web_kit/extractor/html/recognizer/cc_math/common.py

@@ -557,6 +557,8 @@ def process(match_text):
                     return match
                 math_text = self.extract_asciimath(match.strip('`').replace('\\','')) if asciimath_wrap else match
                 wrapped_text = func(math_text) if func else math_text
+                # html保留原始的，而不是传入修改过的wrapped_text
+                original_wrapped = wrapped_text
                 wrapped_text = self.wrap_math_md(wrapped_text)
                 if not wrapped_text:
                     return match
@@ -566,7 +568,7 @@ def process(match_text):
                     tail='',
                     type=math_type,
                     by=math_render,
-                    html=wrapped_text
+                    html=original_wrapped
                 )
             except Exception:
                 return match

llm_web_kit/extractor/html/recognizer/cc_math/render/mathjax.py

@@ -269,7 +269,9 @@ def _find_math_in_element(self, element: HtmlElement, inline_pattern: re.Pattern
         # 跳过ccmath标签
         from llm_web_kit.extractor.html.recognizer.recognizer import \
             BaseHTMLElementRecognizer
-        if BaseHTMLElementRecognizer.is_cc_html(element):
+
+        # 新增方法is_cc_tag_node，该方法值判断当前节点是否是cc标签，而不判断子节点
+        if BaseHTMLElementRecognizer.is_cc_tag_node(element):
             return
 
         # 检查是否应该忽略该元素

llm_web_kit/extractor/html/recognizer/ccmath.py

@@ -139,8 +139,6 @@
                 if ZHIHU.DOMAIN in self.cm.url and node.tag == 'span' and node.get('class') == ZHIHU.MATH:
                     tag_script.process_zhihu_custom_tag(self.cm, math_render_type, node)
 
-                # if 'mathinsight.org' in self.cm.url and node.tag == 'span' and node.get('class') == '':
-
                 # tag = span， class 为 math-containerm， 或者 mathjax 或者 wp-katex-eq
                 if node.tag == 'span' and node.get('class') and (
                         'math-container' in node.get('class') or
@@ -183,14 +181,17 @@
                     if node.tag == 'p' and len(node.getchildren()) == 0:
                         tag_common_modify.modify_tree(self.cm, math_render_type, original_html, node, parent)
 
-                    # TODO: 待优化，渲染器通用方案兜底
-                    try:
-                        if math_render_type == MathRenderType.MATHJAX:
-                            math_render.find_math(node)
-                    except Exception as e:
-                        raise HtmlMathMathjaxRenderRecognizerException(f'处理MathjaxRender数学公式失败: {e}')
+            # 修改：传入tree节点，mathjax方案作为process2，不参与上面process1节点的遍历
+            if math_render_type:
+                try:
+                    if math_render_type == MathRenderType.MATHJAX:
+                        # 对 tree 节点应用 find_math
+                        math_render.find_math(tree)
+                except Exception as e:
+                    raise HtmlMathMathjaxRenderRecognizerException(f'处理MathjaxRender数学公式失败: {e}')
+
             # 保存处理后的html
-            # with open('math_courses_processed.html', 'w', encoding='utf-8') as f:
+            # with open('test20250702_result.html', 'w', encoding='utf-8') as f:
             #     f.write(self._element_to_html(tree))
         except Exception as e:
             raise HtmlMathRecognizerException(f'处理数学公式失败: {e}')
@@ -224,7 +225,10 @@
     #     '</head> '
     #     '<p>这是p的text<span class="mathjax_display">$$a^2 + b^2 = c^2$$</span>这是span的tail<b>这是b的text</b>这是b的tail</p>'
     # )
-    # with open(r'C:\Users\10412\.ssh\llm-webkit-mirror\tests\llm_web_kit\extractor\assets\extractor_chain_input\good_data\html\testmathjax.html', 'r', encoding='utf-8') as f:
+    # with open(r'C:\Users\10412\.ssh\llm-webkit-mirror\tests\st\test20250702.html', 'r', encoding='utf-8') as f:
+    # # with open(
+    # #         r'C:\Users\10412\.ssh\llm-webkit-mirror\bench\data\origin\math_mathjax_asciimath_1.html',
+    # #         'r', encoding='utf-8') as f:
     #     raw_html = f.read()
     # from llm_web_kit.libs.html_utils import html_to_element
     # root = html_to_element(raw_html)

llm_web_kit/extractor/html/recognizer/recognizer.py

@@ -263,3 +263,18 @@ def is_cc_html(el: HtmlElement, tag_name: str | list = None) -> bool:
         # 构建XPath表达式，检查子元素是否包含目标标签
         xpath_expr = ' or '.join([f'self::{tag}' for tag in tags])
         return bool(el.xpath(f'.//*[{xpath_expr}]'))
+
+    @staticmethod
+    def is_cc_tag_node(el: HtmlElement) -> bool:
+        """判断html片段是否是cc标签.
+
+        在is_cc_html上做修改，只判断该节点是否为cc标签，而不检查其子节点是否包含cc标签，用在mathjax渲染器方法中.
+
+        Args:
+            el: str|HtmlElement: html片段或HtmlElement对象
+        """
+        default_tag_names = [
+            CCTag.CC_CODE, CCTag.CC_MATH_INLINE, CCTag.CC_MATH_INTERLINE, CCTag.CC_IMAGE, CCTag.CC_VIDEO,
+            CCTag.CC_AUDIO, CCTag.CC_TABLE, CCTag.CC_LIST, CCTag.CC_TEXT, CCTag.CC_TITLE
+        ]
+        return hasattr(el, 'tag') and isinstance(el.tag, str) and el.tag in default_tag_names

requirements/runtime.txt

@@ -11,7 +11,7 @@ jupyter==1.1.1
 langdetect_zh==1.0.4
 lightgbm==4.5.0
 loguru==0.7.2
-lxml>=5.3.0
+lxml==5.3.0
 nbconvert==7.16.6
 nltk==3.8.1
 notebook==7.4.2

tests/llm_web_kit/extractor/html/recognizer/assets/ccmath/math_physicsforums_2_1.html

@@ -14,7 +14,7 @@
 \langle \psi | \phi \rangle \quad \text{versus} \quad \braket{\psi}{\phi}
 [\hat{A}, \hat{B}] \quad \text{versus} \quad \comm{\hat{A}}{\hat{B}}
 \{\hat{A}, \hat{B}\} \quad \text{versus} \quad \acomm{\hat{A}}{\hat{B}}
-\begin{pmatrix}a & b \\c & d\end{pmatrix}
+\begin{pmatrix}a & b \\c & d\end{pmatrix} \quad \text{versus} \quad \mqty(a & b \\ c & d)
 |\psi\rangle \quad \text{versus} \quad \ket{\psi}
 \langle\psi| \quad \text{versus} \quad \bra{\psi}
 \hat{H}|\psi\rangle \quad \text{versus} \quad \op{H}{\psi}

tests/llm_web_kit/extractor/html/recognizer/test_math.py

@@ -54,10 +54,10 @@
         ],
         'raw_html': '<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-MML-AM_CHTML"> </script><p>$x = 5$,$$x=6$$</p>',
         'expected': [
-            ('<p><ccmath-inline type="latex" by="mathjax" html="x = 5">x = 5</ccmath-inline>,</p>',
-             '<p><ccmath-inline type="latex" by="mathjax" html="x = 5">x = 5</ccmath-inline>,</p>'),
-             ('<p><ccmath-interline type="latex" by="mathjax" html="x=6">x=6</ccmath-interline></p>',
-              '<p><ccmath-interline type="latex" by="mathjax" html="x=6">x=6</ccmath-interline></p>')
+            ('<p><ccmath-inline type="latex" by="mathjax" html="$x = 5$">x = 5</ccmath-inline>,</p>',
+             '<p><ccmath-inline type="latex" by="mathjax" html="$x = 5$">x = 5</ccmath-inline>,</p>'),
+             ('<p><ccmath-interline type="latex" by="mathjax" html="$$x=6$$">x=6</ccmath-interline></p>',
+              '<p><ccmath-interline type="latex" by="mathjax" html="$$x=6$$">x=6</ccmath-interline></p>')
         ]
     },
     {
@@ -67,9 +67,9 @@
         ],
         'raw_html': '<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-MML-AM_CHTML"> </script> <p>$x = 5$,$$x=6$$,$x=4$</p>',
         'expected': [
-            ('<p><ccmath-inline type="latex" by="mathjax" html="x = 5">x = 5</ccmath-inline>,</p>', '<p><ccmath-inline type="latex" by="mathjax" html="x = 5">x = 5</ccmath-inline>,</p>'),
+            ('<p><ccmath-inline type="latex" by="mathjax" html="$x = 5$">x = 5</ccmath-inline>,</p>', '<p><ccmath-inline type="latex" by="mathjax" html="$x = 5$">x = 5</ccmath-inline>,</p>'),
             ('<p><ccmath-interline type="latex" by="mathjax" html="$$x=6$$">x=6</ccmath-interline></p>', '<p><ccmath-interline type="latex" by="mathjax" html="$$x=6$$">x=6</ccmath-interline></p>'),
-            ('<p>,<ccmath-inline type="latex" by="mathjax" html="x=4">x=4</ccmath-inline></p>', '<p>,<ccmath-inline type="latex" by="mathjax" html="x=4">x=4</ccmath-inline></p>')
+            ('<p>,<ccmath-inline type="latex" by="mathjax" html="$x=4$">x=4</ccmath-inline></p>', '<p>,<ccmath-inline type="latex" by="mathjax" html="$x=4$">x=4</ccmath-inline></p>')
         ]
     },
     {
@@ -81,12 +81,12 @@
         'expected': [
             ('<p>By substituting </p>',
              '<p>By substituting </p>'),
-            ('<p><ccmath-interline type="latex" by="mathjax" html="x">x</ccmath-interline></p>',
-             '<p><ccmath-interline type="latex" by="mathjax" html="x">x</ccmath-interline></p>'),
+            ('<p><ccmath-interline type="latex" by="mathjax" html="$$x$$">x</ccmath-interline></p>',
+             '<p><ccmath-interline type="latex" by="mathjax" html="$$x$$">x</ccmath-interline></p>'),
             ('<p> with </p>',
              '<p> with </p>'),
-            ('<p><ccmath-interline type="latex" by="mathjax" html="t - \\dfrac{b}{3a}">t - \\dfrac{b}{3a}</ccmath-interline></p>',
-             '<p><ccmath-interline type="latex" by="mathjax" html="t - \\dfrac{b}{3a}">t - \\dfrac{b}{3a}</ccmath-interline></p>'),
+            ('<p><ccmath-interline type="latex" by="mathjax" html="$$t - \\dfrac{b}{3a}$$">t - \\dfrac{b}{3a}</ccmath-interline></p>',
+             '<p><ccmath-interline type="latex" by="mathjax" html="$$t - \\dfrac{b}{3a}$$">t - \\dfrac{b}{3a}</ccmath-interline></p>'),
             ('<p>, the general</p>',
              '<p>, the general</p>')
         ]
@@ -407,6 +407,19 @@
         'expected_formula': r'\begin{aligned}  & p(r|s,a) \\  & \sum_{r\in\mathcal{R}(s,a)}p(r|s,a)=1\text{ for any }(s,a). \end{aligned}'
     }
 ]
+
+TEST_IS_CC_TAG_NODE = [
+    {
+        'input': r'<div><p>行内公式1：$A+B=C$</p><p>行内公式2：$a+b=c$。</p><p>行间公式1：$$E+F=G$$。<\p><ccmath-interline type="latex" by="mathjax" html="$$t - \\dfrac{b}{3a}$$">t - \\dfrac{b}{3a}</ccmath-interline><p>行内公式3：$2A+2B=2C$。</p><p>行间公式2：$$2E+2F=2G$$。</p></div>',
+        'cc_tag': '1',
+        'not_cc_tag': '6',
+    },
+    {
+        'input': r'<body><p><ccmath-interline type="latex" by="mathjax" html="$$x$$">x</ccmath-interline><p><ccmath-inline type="latex" by="mathjax" html="$x$">x</ccmath-inline></p><p>行内公式1：$A+B=C$</p><span class="math-container"><p>行间公式2$$D+E=F$$</p></span></body>',
+        'cc_tag': '2',
+        'not_cc_tag': '6',
+    }
+]
 base_dir = Path(__file__).parent
 
 
@@ -577,6 +590,35 @@ def test_zhihu_ztext_math(self):
                 formula = element.xpath(f'//{expected_tag}/text()')[0]
                 self.assertIn(test_case['expected_formula'], formula)
 
+    def test_is_cc_tag_node(self):
+        from llm_web_kit.extractor.html.recognizer.recognizer import \
+            BaseHTMLElementRecognizer
+        for test_case in TEST_IS_CC_TAG_NODE:
+            with self.subTest(input=test_case['input']):
+                root = html_to_element(test_case['input'])
+                cc_tag_count = 0
+                not_cc_tag_count = 0
+
+                def check_nodes(element):
+                    nonlocal cc_tag_count, not_cc_tag_count
+                    if element is None:
+                        return
+                    if BaseHTMLElementRecognizer.is_cc_tag_node(element):
+                        cc_tag_count += 1
+                        return
+                    else:
+                        not_cc_tag_count += 1
+                    for child in element:
+                        check_nodes(child)
+                # mathjax方案是传入根节点递归调用，与其保持一致
+                check_nodes(root)
+                expected_cc_tag = int(test_case['cc_tag'])
+                expected_not_cc_tag = int(test_case['not_cc_tag'])
+                self.assertEqual(cc_tag_count, expected_cc_tag,
+                                 f'Expected {expected_cc_tag} cc tags, but found {cc_tag_count}')
+                self.assertEqual(not_cc_tag_count, expected_not_cc_tag,
+                                 f'Expected {expected_not_cc_tag} non-cc tags, but found {not_cc_tag_count}')
+
 
 if __name__ == '__main__':
     r = TestMathRecognizer()

tests/llm_web_kit/extractor/test_extractor_chain.py

@@ -193,7 +193,7 @@ def test_html_pipeline_suit_2(self):
         self.assertEqual(result['track_id'], 'stackoverflow_math')
 
         html_content_list = result.get_content_list()[0]
-        assert len(html_content_list) == 24
+        assert len(html_content_list) == 22
 
     def test_mathlab_html_to_md(self):
         """测试第二个数据：这个数据会丢失一些文本信息."""

tests/st/test_st.py

@@ -78,6 +78,8 @@ def test_st_bench(self):
             output_path=output_path,
         )
 
+        failed_cases = []
+
         with open(self.pipeline_data_path, 'r', encoding='utf-8') as f:
             for line in f:
                 data_json = json.loads(line.strip())
@@ -105,7 +107,14 @@ def test_st_bench(self):
         detail.finish()
         self.assertIsNotNone(summary)
         self.assertIsNotNone(detail)
-        self.assertEqual(summary.error_summary['count'], 0, msg=f'测试数据抽取有失败, 抽取失败的数据详情: {detail.to_dict()}')
+        # 修改，显示所有断言失败的数据，不仅是显示一条
+        if failed_cases or summary.error_summary['count'] != 0:
+            msg = ''
+            if failed_cases:
+                msg += '有断言失败:\n' + '\n'.join(failed_cases) + '\n'
+            if summary.error_summary['count'] != 0:
+                msg += f'测试数据抽取有失败, 抽取失败的数据详情: {detail.to_dict()}'
+            self.fail(msg)
 
 
 if __name__ == '__main__':