ccprocessor · yogacc33 · Jun 26, 2025 · Jun 26, 2025
diff --git a/llm_web_kit/input/pre_data_json.py b/llm_web_kit/input/pre_data_json.py
@@ -36,6 +36,8 @@ class PreDataJsonKey:
     DYNAMIC_ID_ENABLE = 'dynamic_id_enable'
     # 动态classid开关
     DYNAMIC_CLASSID_ENABLE = 'dynamic_classid_enable'
+    # 正文噪音开关
+    MORE_NOISE_ENABLE = 'more_noise_enable'
     # 推广原网页
     HTML_SOURCE = 'html_source'
     # 推广网页提取正文成功标签, bool类型

diff --git a/llm_web_kit/main_html_parser/parser/layout_batch_parser.py b/llm_web_kit/main_html_parser/parser/layout_batch_parser.py
@@ -27,6 +27,7 @@
         self.template_data = template_data
         self.dynamic_id_enable = False
         self.dynamic_classid_enable = False
+        self.more_noise_enable = False
 
     def parse_tuple_key(self, key_str):
         if key_str.startswith('(') and key_str.endswith(')'):
@@ -41,7 +42,8 @@
         html_source = pre_data[PreDataJsonKey.HTML_SOURCE]
         template_dict_html = pre_data.get(PreDataJsonKey.TYPICAL_DICT_HTML, '<html></html>')
         self.dynamic_id_enable = pre_data.get(PreDataJsonKey.DYNAMIC_ID_ENABLE, False)
-        self.dynamic_classid_enable = pre_data.get('dynamic_classid_enable', False)
+        self.dynamic_classid_enable = pre_data.get(PreDataJsonKey.DYNAMIC_CLASSID_ENABLE, False)
+        self.more_noise_enable = pre_data.get(PreDataJsonKey.MORE_NOISE_ENABLE, False)
         template_data_str = pre_data[PreDataJsonKey.HTML_ELEMENT_DICT]
         template_data = dict()
         if isinstance(template_data_str, str):
@@ -97,6 +99,10 @@
         if not tup:
             return None
         tag, class_id, idd = tup
+        if class_id:
+            class_id = re.sub(r' +', ' ', class_id)
+        if idd:
+            idd = re.sub(r' +', ' ', idd)
         # 如果有id，则无需判断class，因为有的网页和模版id相同，但是class不同
         if tag in ['body', 'html']:
             return (tag, None, None)
@@ -110,14 +116,16 @@
         # 匹配 "post-数字" 或 "postid-数字"（不区分大小写），并替换数字部分为空
         pattern = r'(post|postid)-(\d+)'
         # 使用 \1 保留前面的 "post" 或 "postid"，但替换数字部分
-        return re.sub(pattern, lambda m: f'{m.group(1)}-', text, flags=re.IGNORECASE)
+        return re.sub(pattern, lambda m: f'{m.group(1)}-', text, flags=re.IGNORECASE).strip()
 
     def find_blocks_drop(self, element, depth, element_dict, parent_keyy, parent_label, template_doc):
         # 判断这个tag是否有id
         if isinstance(element, etree._Comment):
             return
         length = len(self.get_tokens(element.text_content().strip()))
         length_tail = 0
+        text = element.xpath('string()').strip()
+        is_natural_language = self.__is_natural_language(text)
         if element.tail:
             length_tail = len(element.tail.strip())
         idd = element.get('id')
@@ -224,10 +232,13 @@
         # 判断当前节点是否是红色节点
         if keyy in layer_nodes_dict:
             if 'red' not in layer_nodes_dict[keyy]:
-                parent = element.getparent()
-                if parent is not None:
-                    parent.remove(element)
-                return
+                if self.more_noise_enable and tag in ['p', 'ul'] and not idd and not class_tag and is_natural_language:
+                    label = 'red'
+                else:
+                    parent = element.getparent()
+                    if parent is not None:
+                        parent.remove(element)
+                    return
             else:
                 label = 'red'
         elif length > 0 or length_tail > 0:
@@ -352,6 +363,20 @@
                 if feature1 is None or feature2 is None:
                     continue
                 template_sim = similarity(feature1, feature2, layer_n=3)
-                if template_sim > 0.9:
+                if template_sim > 0.85:
                     return ele_label, self.normalize_key(ele_keyy[0:3])
         return None, None
+
+    def __is_natural_language(self, text, min_words=3):
+        """判断文本是否像自然语言.
+
+        :param text: 输入文本
+        :param min_words: 至少需要几个有效词才认为是自然语言
+        :return: bool
+        """
+        # 移除标点符号和多余空格
+        cleaned_text = re.sub(r'[^\w\s]', '', text.strip())
+        words = cleaned_text.split()
+        if len(words) <= min_words:
+            return False
+        return True
diff --git a/tests/llm_web_kit/input/test_pre_data_json.py b/tests/llm_web_kit/input/test_pre_data_json.py
@@ -392,6 +392,7 @@ def test_pre_data_json_key_constants(self):
         assert hasattr(PreDataJsonKey, 'TYPICAL_DICT_HTML')
         assert hasattr(PreDataJsonKey, 'DYNAMIC_ID_ENABLE')
         assert hasattr(PreDataJsonKey, 'DYNAMIC_CLASSID_ENABLE')
+        assert hasattr(PreDataJsonKey, 'MORE_NOISE_ENABLE')
 
         # Check actual values
         assert PreDataJsonKey.DOMAIN_NAME == 'domain_name'

diff --git a/tests/llm_web_kit/main_html_parser/parser/test_layout_parser.py b/tests/llm_web_kit/main_html_parser/parser/test_layout_parser.py
@@ -3,6 +3,8 @@
 import unittest
 from pathlib import Path
 
+from lxml import etree, html
+
 from llm_web_kit.input.pre_data_json import PreDataJson, PreDataJsonKey
 from llm_web_kit.main_html_parser.parser.layout_batch_parser import \
     LayoutBatchParser
@@ -235,3 +237,46 @@ def test_dynamic_classid(self):
         parts = parser.parse(pre_data)
         main_html_body = parts[PreDataJsonKey.MAIN_HTML_BODY]
         assert 'Permalink link a questo articolo' not in main_html_body and 'Con la stesura di un' in main_html_body
+
+    def test_more_noise_enable(self):
+        """测试推广方法的动态id识别功能."""
+        # 构造测试html
+        html_source = base_dir.joinpath('assets/input_layout_batch_parser/www.wdi.it.html').read_text(
+            encoding='utf-8')
+        template_source = re.sub(
+            'clearfix post post-37041 type-post status-publish format-standard hentry category-economia ' +
+            'category-societa tag-camera tag-commercio tag-cosenza tag-diritto tag-economia-2 tag-imprese tag-libro ' +
+            'tag-usi item-wrap',
+            'post-id', html_source)
+        template_source = re.sub('post-37041', 'test-37041', template_source)
+        expand_source = re.sub('test-37041', 'test-25031', template_source)
+        tree = etree.HTML(expand_source)
+        content_main = tree.xpath('//*[@id="content-main"]')[0]  # 定位 id="content-main" 的元素
+
+        # 创建新元素 <p>test</p> 并添加到 content-main 下
+        new_p = etree.SubElement(content_main, 'p')
+        new_p.text = 'test more noise'
+        expand_source = html.tostring(tree, encoding='utf-8').decode()
+        # 简化网页
+        simplified_html, typical_raw_tag_html, _ = simplify_html(template_source)
+        # 模型结果格式改写
+        llm_path = base_dir.joinpath(TEST_CASES[0]['input'][2][0])
+        llm_response = json.loads(llm_path.read_text(encoding='utf-8'))
+        for key in llm_response.keys():
+            llm_response[key] = 1 if llm_response[key] == 'Yes' else 0
+        pre_data = {'typical_raw_tag_html': typical_raw_tag_html, 'typical_raw_html': template_source,
+                    'llm_response': llm_response}
+        pre_data = PreDataJson(pre_data)
+        # 映射
+        parser = MapItemToHtmlTagsParser({})
+        pre_data = parser.parse(pre_data)
+        element_dict = pre_data.get(PreDataJsonKey.HTML_ELEMENT_DICT, {})
+        # 推广
+        pre_data[PreDataJsonKey.HTML_SOURCE] = expand_source
+        pre_data[PreDataJsonKey.DYNAMIC_ID_ENABLE] = True
+        pre_data[PreDataJsonKey.DYNAMIC_CLASSID_ENABLE] = True
+        pre_data[PreDataJsonKey.MORE_NOISE_ENABLE] = True
+        parser = LayoutBatchParser(element_dict)
+        parts = parser.parse(pre_data)
+        main_html_body = parts[PreDataJsonKey.MAIN_HTML_BODY]
+        assert 'test more noise' in main_html_body