fix: layout cluster dynamic properties & unit test

llm_web_kit/html_layout/html_layout_cosin.py

@@ -12,6 +12,11 @@
 
 TAGS_TO_IGNORE = ['script', 'style', 'meta', 'link', 'br', 'noscript']  # , 'b', 'i', 'strong'
 TAGS_IGNORE_ATTR = ['a', 'i', 'b', 'li', 'tr', 'td', 'img', 'p', 'body']
+RE_MD5 = re.compile(r'^[0-9a-f]{32}$')
+RE_SHA1 = re.compile(r'^[0-9a-f]{40}$')
+RE_UUID = re.compile(r'^[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}$')
+RE_TIMESTAMP = re.compile(r'^\d{10,13}$')  # 时间戳属性值
+RE_NUM = re.compile(r'\d+')  # 自定义动态属性值
 
 
 def html_to_element(html_data: str) -> HtmlElement:
@@ -97,10 +102,9 @@
                 if tag in TAGS_IGNORE_ATTR:
                     parent_tag_attr.add(f'<{tag}>')
                 else:
-                    attrs_str = ' '.join(
-                        [f'{k}="{__standardizing_dynamic_attributes(v)}"' for k, v in child.attrib.items() if
-                         k in ['class', 'id']])
-                    parent_tag_attr.add(f'<{tag} {attrs_str}>' if attrs_str else f'<{tag}>')
+                    attrs_str = __parse_attributes(child)
+                    if attrs_str is not None:
+                        parent_tag_attr.add(f'<{tag} {attrs_str}>' if attrs_str else f'<{tag}>')
 
             el_tag_attr.append(parent_tag_attr)
 
@@ -117,14 +121,34 @@
     return dict(tag_attr) if tag_attr.get('tags') else None
 
 
+def __parse_attributes(element: HtmlElement):
+    """解析标签属性值."""
+    class_s = None
+    id_s = None
+    class_attr = element.get('class')
+    if class_attr:
+        class_d = class_attr.split()
+        if len(class_d) > 1:
+            class_s = ' '.join([i for i in class_d if not RE_NUM.search(i)])
+        else:
+            class_s = __standardizing_dynamic_attributes(class_d[0])
+
+    id_attr = element.get('id')
+    if id_attr:
+        id_d = id_attr.split()
+        if len(id_d) > 1:
+            id_s = ' '.join([i for i in id_d if not RE_NUM.search(i)])
+        else:
+            id_s = __standardizing_dynamic_attributes(id_d[0])
+
+    if not class_s and not id_s:
+        return None
+    else:
+        return ' '.join([f'{k}= "{v}"' for k, v in {'class': class_s, 'id': id_s}.items() if v])
+
+
 def __standardizing_dynamic_attributes(attr_value):
     """将动态属性值标准化为统一表示."""
-    # 预编译正则表达式
-    RE_MD5 = re.compile(r'^[0-9a-f]{32}$')
-    RE_SHA1 = re.compile(r'^[0-9a-f]{40}$')
-    RE_UUID = re.compile(r'^[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}$')
-    RE_TIMESTAMP = re.compile(r'^\d{10,13}$')  # 时间戳属性值
-    RE_CUSTOM = re.compile(r'^([A-Za-z-_ ]{2,})\d+$')  # 自定义动态属性值
     if RE_MD5.fullmatch(attr_value):
         return '[MD5]'
     if RE_SHA1.fullmatch(attr_value):
@@ -133,9 +157,8 @@
         return '[UUID]'
     if RE_TIMESTAMP.fullmatch(attr_value):
         return '[TIMESTAMP]'
-    match = RE_CUSTOM.fullmatch(attr_value)
-    if match:
-        return match.group(1)
+    if RE_NUM.search(attr_value):
+        return re.sub(r'\d+', '', attr_value)
 
     return attr_value  # 不是可识别的哈希则返回原值
 

tests/llm_web_kit/html_layout/test_html_layout_cosin.py

@@ -13,7 +13,7 @@
     {'input': 'assets/feature4.html', 'expected': 2},
 ]
 TEST_SIMIL_HTMLS = [
-    {'input1': 'assets/feature.html', 'input2': 'assets/cosin.html', 'expected': 0.22013982},
+    {'input1': 'assets/feature.html', 'input2': 'assets/cosin.html', 'expected': 0.2831639},
     {'input1': 'assets/feature1.html', 'input2': 'assets/feature2.html', 'layer_n': 12, 'expected': 0.9463744},
     {'input1': 'assets/feature1.html', 'input2': 'assets/data_structure.html', 'layer_n': 12, 'expected': 0},
 ]
@@ -30,9 +30,9 @@
 ]
 TEST_CLUSTER_HTMLS = [
     {'input': ['assets/feature1.html', 'assets/cosin.html'], 'expected': [-1]},
-    {'input': ['assets/feature1.html', 'assets/feature2.html'], 'expected': [-1]},
+    {'input': ['assets/feature1.html', 'assets/feature2.html'], 'expected': [0]},
     {'input': ['assets/100.html', 'assets/101.html', 'assets/102.html', 'assets/103.html', 'assets/104.html',
-               'assets/105.html', 'assets/106.html'], 'expected': [0, 1, -1]}
+               'assets/105.html', 'assets/106.html'], 'expected': [0]}
 
 ]
 
@@ -62,7 +62,7 @@ def test_get_feature(self):
                 layer_n, total_n = sum_tags(get_feature(raw_html, is_ignore_tag=False)['tags'])
                 ignore_layer_n, ignore_total_n = sum_tags(get_feature(raw_html)['tags'])
                 self.assertEqual(len(layer_n) == len(ignore_layer_n), True)
-                self.assertEqual(total_n > ignore_total_n, True)
+                self.assertEqual(total_n >= ignore_total_n, True)
             else:
                 self.assertEqual(features, test_case['expected'])