diff --git a/docs/llm_web_kit/model/html_simplify_classify.md b/docs/llm_web_kit/model/html_simplify_classify.md index f4f77e4f..bacba487 100644 --- a/docs/llm_web_kit/model/html_simplify_classify.md +++ b/docs/llm_web_kit/model/html_simplify_classify.md @@ -14,23 +14,7 @@ ## HTML simplify -该部分用于简化HTML从而提高网页布局分类的效果。在 `llm_web_kit/model/html_lib/html_lib` 目录下的 `simplify.py` 文件中的 `general_simplify_html_str` 函数实现了对html字符串的简化操作。 - -默认路径为`~/.llm-web-kit.jsonc`中需要使用如下配置,可以自动下载模型: - -```json -{ - "resources": { - "common":{ - "cache_path": "~/.llm_web_kit_cache" - }, - "html_cls-25m2": { - "download_path": "s3://web-parse-huawei/shared_resource/html_layout_cls/html_cls_25m2.zip", - "md5": "e15ea22a9aa65aa8c7c3a0e3c2e0c98a" - }, - } -} -``` +该部分用于简化HTML从而提高网页布局分类的效果。在 `llm_web_kit/model/html_lib/simplify.py` 文件中的 `general_simplify_html_str` 函数实现了对html字符串的简化操作。该部分的功能实现已经迁移到 `html_alg_lib` 库中,目前直接调用 `html_alg_lib.simplify` 中的 `process_to_cls_alg_html` 函数。 使用方法如下: @@ -47,7 +31,7 @@ print(simp_html) ## HTML 分类 -将简化后的html分类article, forum, other三个类别。在`llm_web_kit/model/html_layout_cls.py`中,使用`HTMLLayoutClassifier`类完成自动下载checkpoint和推理过程。 +将简化后的html分类Article, Forum_or_Article_with_commentsection, Content Listing, Other四个类别。在`llm_web_kit/model/html_layout_cls.py`中,使用`HTMLLayoutClassifier`类完成自动下载checkpoint和推理过程。 使用方法如下: @@ -59,3 +43,19 @@ html_str_input = ['layout1', 'layout2'] layout_type = model.predict(html_str_input) print(layout_type) ``` + +默认路径为`~/.llm-web-kit.jsonc`中需要使用如下配置,可以自动下载模型: + +```json +{ + "resources": { + "common":{ + "cache_path": "~/.llm_web_kit_cache" + }, + "html_cls-25m2": { + "download_path": "s3://web-parse-huawei/shared_resource/html_layout_cls/html_cls_25m4.zip", + "md5": "31b4889b4d9c8a1a6da7a5c58270e611" + }, + } +} +``` diff --git a/llm_web_kit/model/html_classify/model.py b/llm_web_kit/model/html_classify/model.py index cdb1d637..6454b766 100644 --- a/llm_web_kit/model/html_classify/model.py +++ b/llm_web_kit/model/html_classify/model.py @@ -1,3 +1,5 @@ +import os + import torch from llm_web_kit.model.resource_utils import import_transformer @@ -6,13 +8,12 @@ class Markuplm(): def __init__(self, path, device): self.path = path - self.model_path = self.path + '/markuplm-base' - self.checkpoint_path = self.path + '/markuplm_202501222031_epoch_2.pt' + self.model_path = os.path.join(self.path, 'markuplm-base') self.device = device - self.num_labels = 3 + self.num_labels = 4 self.max_tokens = 512 - self.label2id = {0:'article', 1:'forum', 2:'other'} + self.label2id = {0: 'Article', 1: 'Forum_or_Article_with_commentsection', 2: 'Content Listing', 3: 'Other'} self.model = self.load_model() self.tokenizer = self.load_tokenizer() @@ -25,8 +26,6 @@ def load_tokenizer(self): def load_model(self): transformers = import_transformer() model = transformers.MarkupLMForSequenceClassification.from_pretrained(self.model_path, num_labels=self.num_labels) - # load checkpoint - model.load_state_dict(torch.load(self.checkpoint_path, map_location=self.device)) model.to(self.device) model.eval() diff --git a/llm_web_kit/model/html_layout_cls.py b/llm_web_kit/model/html_layout_cls.py index e4c86694..981a9e9a 100644 --- a/llm_web_kit/model/html_layout_cls.py +++ b/llm_web_kit/model/html_layout_cls.py @@ -15,8 +15,8 @@ def __init__(self, model_path: str = None, device: str = 'cuda'): self.model = Markuplm(model_path, device) def auto_download(self) -> str: - """Default download the html_cls_25m2.zip model.""" - resource_name = 'html_cls-25m2' + """Default download the html_cls_25m4.zip model.""" + resource_name = 'html_cls-25m4' resource_config = load_config()['resources'] print(resource_config) model_config: dict = resource_config[resource_name] diff --git a/llm_web_kit/model/html_lib/base_func.py b/llm_web_kit/model/html_lib/base_func.py deleted file mode 100644 index bcb7e7e4..00000000 --- a/llm_web_kit/model/html_lib/base_func.py +++ /dev/null @@ -1,97 +0,0 @@ -import re - -from lxml import html - - -def extract_tag_text(element: html.HtmlElement) -> str: - """Extract text from an html element. - - Args: - element (html.HtmlElement): The element to extract text from - - Returns: - str: The text extracted from the element - """ - # 如果元素是img标签,返回其alt属性 - if element.tag == 'img': - return element.get('alt', '') - - # 如果元素没有子节点,返回其文本 - if len(element) == 0: - return element.text if element.text else '' - - # 如果元素有子节点,递归提取每个子节点的文本,并根据元素类型决定是否添加换行符 - newline_tags = ['p', 'br', 'div', 'li', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'tr'] - texts = [] - if element.text: # 添加未被标签包裹的文本 - texts.append(element.text) - for child in element: - child_text = extract_tag_text(child) - if child.tag in newline_tags: - child_text += '\n' - texts.append(child_text) - if child.tail: # 添加标签后的文本 - texts.append(child.tail) - # 将只有空格的行变成空行 - texts = [text if text.strip() else '\n' for text in texts] - # 使用正则表达式将连续的换行符替换为一个 - return re.sub('\n+', '\n', ''.join(texts)) - - -def get_title(root: html.HtmlElement) -> str: - """Get the title of the html page. - - Args: - root (html.HtmlElement): The root of the html page - - Returns: - str: The title of the html page - """ - title = root.xpath('//title/text()') - - return title[0] if len(title) > 0 else None - - -def remove_blank_text(root: html.HtmlElement): - """Remove blank text in html element. Skip
 tag and  tag and
-    there children.
-
-    Args:
-        root (html.HtmlElement): The root of the html page
-
-    Returns:
-        html.HtmlElement: The root of the html page after removing blank text
-    """
-
-    skip_elements = set()
-    for element in root.iter():
-        if element.tag in ['pre', 'code']:
-            skip_elements.add(element)
-            skip_elements.update(element.iterdescendants())
-
-    for element in root.iter():
-        if element in skip_elements:
-            continue
-        if element.text and not element.text.strip():
-            element.text = None
-        if element.tail and not element.tail.strip():
-            element.tail = None
-    return root
-
-
-def document_fromstring(html_str: str):
-    """Parse html string to lxml.html.HtmlElement default remove blank text,
-    comments, processing instructions.
-
-    Args:
-        html_str (str): The html string to parse
-
-    Returns:
-        html.HtmlElement: The root of the html page
-    """
-    parser = html.HTMLParser(
-        remove_blank_text=True, remove_comments=True, remove_pis=True
-    )
-    root = html.document_fromstring(html_str, parser=parser)
-    root = remove_blank_text(root)
-    return root
diff --git a/llm_web_kit/model/html_lib/merge_tags.py b/llm_web_kit/model/html_lib/merge_tags.py
deleted file mode 100644
index 564785c3..00000000
--- a/llm_web_kit/model/html_lib/merge_tags.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from lxml import html
-
-from llm_web_kit.model.html_lib.base_func import extract_tag_text
-
-
-def merge_list(root: html.HtmlElement) -> html.HtmlElement:
-    # merge ul ol and dl
-    list_tag_names = ['ul', 'ol', 'dl']
-    for tag in list_tag_names:
-        for list_elem in root.findall(f'.//{tag}'):
-            list_inner_text = extract_tag_text(list_elem)
-            # clean up the list_elem
-            for child in list_elem.getchildren():
-                if child.getparent() is not None:
-                    child.drop_tree()
-            list_elem.text = list_inner_text
-    return root
diff --git a/llm_web_kit/model/html_lib/modify_tags.py b/llm_web_kit/model/html_lib/modify_tags.py
deleted file mode 100644
index 726772f3..00000000
--- a/llm_web_kit/model/html_lib/modify_tags.py
+++ /dev/null
@@ -1,46 +0,0 @@
-from lxml import html
-
-
-def wrap_bare_text(root: html.HtmlElement) -> html.HtmlElement:
-    for elem in root.iter():
-        # continue if the element has no child
-        if len(elem) == 0:
-            continue
-        else:
-            # check if the element has text and tail
-            if elem.text and elem.text.strip():
-                # wrap the text in a span tag
-                span = html.Element('span')
-                span.text = elem.text
-                elem.text = ''
-                elem.insert(0, span)
-            for child in elem:
-                if child.tail and child.tail.strip():
-                    # wrap the tail in a span tag
-                    span = html.Element('span')
-                    span.text = child.tail
-                    child.tail = ''
-                    child.addnext(span)
-    return root
-
-
-def unwrap_single_child_tag(root: html.HtmlElement) -> html.HtmlElement:
-
-    def merge_class(A_elem: html.HtmlElement, B_elem: html.HtmlElement) -> str:
-        A_str = A_elem.get('class', '').strip()
-        B_str = B_elem.get('class', '').strip()
-        if A_str and B_str:
-            return f'{A_str}|{B_str}'
-        if A_str:
-            return A_str
-        if B_str:
-            return B_str
-        return ''
-
-    for tag in root.iter():
-        children = list(tag.getchildren())
-        if len(children) == 1 and tag.getparent() is not None:
-            child = children[0]
-            child.set('class', merge_class(tag, child))
-            tag.drop_tag()
-    return root
diff --git a/llm_web_kit/model/html_lib/remove_tags.py b/llm_web_kit/model/html_lib/remove_tags.py
deleted file mode 100644
index b978ff4f..00000000
--- a/llm_web_kit/model/html_lib/remove_tags.py
+++ /dev/null
@@ -1,133 +0,0 @@
-from lxml import html
-
-
-def build_tags_to_remove_map() -> dict[str, list[str]]:
-    return {
-        'document_metadata': ['base', 'head', 'link', 'meta', 'style', 'title'],
-        'content_sectioning': ['aside', 'footer', 'header', 'nav'],
-        'text_content': ['menu', 'hr'],
-        'demarcating_edits': ['del'],
-        # "image_and_multimedia": ["audio", "map", "track", "video", "svg"],
-        'image_and_multimedia': ['audio', 'map', 'track', 'svg'],
-        'interactive': ['dialog'],
-        'embed': [
-            'embed',
-            'fencedframe',
-            'iframe',
-            'object',
-            'picture',
-            'portal',
-            'source',
-        ],
-        'script': ['canvas', 'script'],
-        'form': [
-            'button',
-            'datalist',
-            'fieldset',
-            'form',
-            'input',
-            'label',
-            'legend',
-            'meter',
-            'optgroup',
-            'option',
-            'output',
-            'progress',
-            'select',
-            'textarea',
-        ],
-    }
-
-
-def all_tag_list_to_remove() -> list[str]:
-    all_tag_list = []
-    for tag_list in build_tags_to_remove_map().values():
-        all_tag_list.extend(tag_list)
-    return all_tag_list
-
-
-def remove_tags(root: html.HtmlElement, tag_type_list: list[str]) -> html.HtmlElement:
-    """Remove all tags of a certain type from the html page.
-
-    Args:
-        root (etree.ElementTree): The root of the html page
-        tag_type (str): The type of tag to remove
-
-    Returns:
-        etree.ElementTree: The root of the html page with the tags removed
-    """
-    tags_to_remove = []
-    for tag in tag_type_list:
-        for element in root.findall(f'.//{tag}'):
-            tags_to_remove.append(element)
-    for element in tags_to_remove:
-        if element.getparent() is not None:
-            element.drop_tree()
-    return root
-
-
-def remove_all_tags(root: html.HtmlElement) -> html.HtmlElement:
-    """Remove all tags from the html page.
-
-    Args:
-        root (etree.ElementTree): The root of the html page
-
-    Returns:
-        etree.ElementTree: The root of the html page with the tags removed
-    """
-    all_tag_list = all_tag_list_to_remove()
-    return remove_tags(root, all_tag_list)
-
-
-def is_diplay_none(element: html.HtmlElement) -> bool:
-    style = element.get('style', '').replace(' ', '').lower()
-    if 'display:none' in style:
-        return True
-
-
-def remove_invisible_tags(root: html.HtmlElement) -> html.HtmlElement:
-    tags_to_remove = []
-    for element in root.iter():
-        if is_diplay_none(element):
-            tags_to_remove.append(element)
-    for element in tags_to_remove:
-        if element.getparent() is not None:
-            element.drop_tree()
-    return root
-
-
-def is_blank_tag(root: html.HtmlElement) -> bool:
-    """Check if a tag is nonblank None blank tags if this is an image or if it
-    has text or if it has children with tail text.
-
-    Args:
-        tag (html.HtmlElement): The tag to check
-
-    Returns:
-        bool: True if the tag is nonblank, False otherwise
-    """
-    if root.tag == 'img':
-        return False
-    if root.text and root.text.strip():
-        return False
-    # check children's tail
-    childern_tail = [child.tail for child in root.iterchildren()]
-    if any([tail and tail.strip() for tail in childern_tail]):
-        return False
-    for child in root.iterchildren():
-        if not is_blank_tag(child):
-            return False
-    return True
-
-
-def remove_blank_tags_recursive(
-    root: html.HtmlElement,
-) -> tuple[bool, html.HtmlElement]:
-    tags_to_remove = []
-    for tag in root.iter():
-        if is_blank_tag(tag) and tag.getparent() is not None:
-            tags_to_remove.append(tag)
-    for tag in tags_to_remove:
-        if tag.getparent() is not None:
-            tag.drop_tree()
-    return root
diff --git a/llm_web_kit/model/html_lib/simplify.py b/llm_web_kit/model/html_lib/simplify.py
index 998d7554..e6e83a18 100644
--- a/llm_web_kit/model/html_lib/simplify.py
+++ b/llm_web_kit/model/html_lib/simplify.py
@@ -1,53 +1,6 @@
-from lxml import html
 
-from llm_web_kit.model.html_lib.base_func import document_fromstring, get_title
-from llm_web_kit.model.html_lib.merge_tags import merge_list
-from llm_web_kit.model.html_lib.modify_tags import (unwrap_single_child_tag,
-                                                    wrap_bare_text)
-from llm_web_kit.model.html_lib.remove_tags import (
-    remove_all_tags, remove_blank_tags_recursive, remove_invisible_tags)
-from llm_web_kit.model.html_lib.unwrap_tags import unwrap_all_tags
-
-
-def general_simplify(root: html.HtmlElement) -> tuple[str, html.HtmlElement]:
-    # get the title of the html page
-    title = get_title(root)
-
-    # remove all invisible tags
-    root = remove_invisible_tags(root)
-
-    # unwrap all tags we consider can directly unwrap
-    # unwrap means remove the tag and keep the children
-    root = unwrap_all_tags(root)
-
-    # wrap all bare text in span tags
-    # the bare text means the tag.text and tag.child.tail is not blank and len(tag) > 0
-    # 
-    #    tag.text 
-    #    
-    #        text 
-    #    
-    #    child.tail 
-    # 
-    root = wrap_bare_text(root)
-
-    # remove all tags (and it's children) we consider can directly remove
-    root = remove_all_tags(root)
-
-    # extract text from all children of list tags and set the text as the text of the list tags
-    root = merge_list(root)
-
-    # remove all tags
-    root = remove_blank_tags_recursive(root)
-
-    # unwrap all tags have only one child
-    root = unwrap_single_child_tag(root)
-
-    return title, root
+from html_alg_lib.simplify import process_to_cls_alg_html
 
 
 def general_simplify_html_str(html_str: str) -> str:
-    root = document_fromstring(html_str)
-    _, simplified_root = general_simplify(root)
-    simplified_html_str = html.tostring(simplified_root)
-    return simplified_html_str.decode('utf-8')
+    return process_to_cls_alg_html(html_str)
diff --git a/llm_web_kit/model/html_lib/unwrap_tags.py b/llm_web_kit/model/html_lib/unwrap_tags.py
deleted file mode 100644
index c839fc37..00000000
--- a/llm_web_kit/model/html_lib/unwrap_tags.py
+++ /dev/null
@@ -1,53 +0,0 @@
-from lxml import html
-
-
-def build_tags_to_unwrap_map():
-    return {
-        'inline_text_semantics': [
-            'a',
-            'abbr',
-            'b',
-            'bdi',
-            'bdo',
-            'br',
-            'cite',
-            'code',
-            'data',
-            'dfn',
-            'em',
-            'i',
-            'kbd',
-            'mark',
-            'q',
-            'rp',
-            'rt',
-            'ruby',
-            's',
-            'samp',
-            'small',
-            'span',
-            'strong',
-            'sub',
-            'sup',
-            'time',
-            'u',
-            'var',
-            'wbr',
-        ],
-        'demarcating_edits:': ['ins'],
-    }
-
-
-def get_all_tags_to_unwrap():
-    all_tags = []
-    for tag_list in build_tags_to_unwrap_map().values():
-        all_tags.extend(tag_list)
-    return all_tags
-
-
-def unwrap_all_tags(root: html.HtmlElement) -> html.HtmlElement:
-    all_tags = get_all_tags_to_unwrap()
-    for tag in all_tags:
-        for element in root.findall(f'.//{tag}'):
-            element.drop_tag()
-    return root
diff --git a/requirements/runtime.txt b/requirements/runtime.txt
index 8a1bbcea..8ac04789 100644
--- a/requirements/runtime.txt
+++ b/requirements/runtime.txt
@@ -5,6 +5,7 @@ click==8.1.8
 commentjson==0.9.0
 fasttext-wheel==0.9.2
 filelock==3.16.1
+html-alg-lib==2.0.2
 jieba-fast==0.53
 langdetect_zh==1.0.4
 lightgbm==4.5.0
diff --git a/tests/llm_web_kit/model/html_classify/test_model.py b/tests/llm_web_kit/model/html_classify/test_model.py
index 7e9f9b1a..da12fde6 100644
--- a/tests/llm_web_kit/model/html_classify/test_model.py
+++ b/tests/llm_web_kit/model/html_classify/test_model.py
@@ -26,7 +26,6 @@ def test_init(self, mock_load, mock_model_from_pretrained, mock_tokenizer_from_p
         self.assertEqual(markuplm.device, device)
         mock_model_from_pretrained.assert_called_once_with(markuplm.model_path, num_labels=markuplm.num_labels)
         mock_tokenizer_from_pretrained.assert_called_once_with(markuplm.model_path)
-        mock_load.assert_called_once_with(markuplm.checkpoint_path, map_location=device)
 
     @patch('transformers.MarkupLMProcessor.from_pretrained')
     @patch('transformers.MarkupLMForSequenceClassification.from_pretrained')
diff --git a/tests/llm_web_kit/model/html_lib/test_base_func.py b/tests/llm_web_kit/model/html_lib/test_base_func.py
deleted file mode 100644
index acfd2c1c..00000000
--- a/tests/llm_web_kit/model/html_lib/test_base_func.py
+++ /dev/null
@@ -1,139 +0,0 @@
-import unittest
-
-from lxml import html
-
-from llm_web_kit.model.html_lib.base_func import (document_fromstring,
-                                                  extract_tag_text, get_title,
-                                                  remove_blank_text)
-
-
-class TestExtractTagText(unittest.TestCase):
-    def test_img_tag(self):
-        element = html.fromstring('test alt')
-        self.assertEqual(extract_tag_text(element), 'test alt')
-
-    def test_img_tag_no_alt(self):
-        element = html.fromstring('')
-        self.assertEqual(extract_tag_text(element), '')
-
-    def test_plain_text(self):
-        element = html.fromstring('

Hello World

') - self.assertEqual(extract_tag_text(element), 'Hello World') - - def test_nested_elements(self): - element = html.fromstring('
Text inside span tail
') - self.assertEqual(extract_tag_text(element), 'Text inside span tail') - - def test_newline_tags(self): - element = html.fromstring('

Line1


Line2

') - result = extract_tag_text(element) - # Expect "Line1\nLine2\n" after newline processing - self.assertEqual(result.replace('\n', '|'), 'Line1|Line2|') - - def test_mixed_content(self): - element = html.fromstring( - """ -
- Before - bold text - - italic - tail - - after -
- """ - ) - # Some problem here. But now the web classification model is trained with this behavior. - # TODO Fix this bug and retrain the model - # expected = "Before bold text italic tail after" - expected = 'Before bold text italic tail after' - self.assertEqual(extract_tag_text(element).replace('\n', ' ').strip(), expected) - - def test_whitespace_handling(self): - # Some problem here. But now the web classification model is trained with this behavior. - # TODO Fix this bug and retrain the model - # expected = "\n" - element = html.fromstring('
\t\n
') - self.assertEqual(extract_tag_text(element), ' \t\n ') - - -class TestGetTitle(unittest.TestCase): - def test_has_title(self): - root = html.fromstring('Test Title') - self.assertEqual(get_title(root), 'Test Title') - - def test_no_title(self): - root = html.fromstring('') - self.assertIsNone(get_title(root)) - - def test_multiple_titles(self): - root = html.fromstring( - 'FirstSecond' - ) - self.assertEqual(get_title(root), 'First') - - -class TestRemoveBlankText(unittest.TestCase): - def test_remove_whitespace_nodes(self): - root = html.fromstring( - """ -
-

-
  
- - -
- """ - ) - processed = remove_blank_text(root) - - # Check normal elements - p = processed.find('.//p') - self.assertIsNone(p.text) - - span = processed.find('.//span') - self.assertIsNone(span.text) - - # Check preserved elements - pre = processed.find('.//pre') - self.assertEqual(pre.text.strip(), '', 'Pre text should preserve whitespace') - self.assertEqual(pre.text, ' ') - - code = processed.find('.//code') - self.assertEqual(code.text, ' ') - - def test_keep_trailing_text(self): - root = html.fromstring('
Hello World
') - processed = remove_blank_text(root) - b = processed.find('.//b') - self.assertIsNone(b.text) - self.assertEqual(b.tail, 'World') - - -class TestDocumentFromString(unittest.TestCase): - def test_basic_parsing(self): - html_str = """ - - - Test - -

Content

- - - """ - root = document_fromstring(html_str) - self.assertEqual(root.tag, 'html') - self.assertIsNotNone(root.find('.//body/p')) - - def test_whitespace_removal(self): - html_str = '

' - root = document_fromstring(html_str) - div = root.find('.//div') - self.assertIsNone(div.text) - p = div.find('.//p') - self.assertIsNone(p.text) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/llm_web_kit/model/html_lib/test_merge_tags.py b/tests/llm_web_kit/model/html_lib/test_merge_tags.py deleted file mode 100644 index 9bb73f1c..00000000 --- a/tests/llm_web_kit/model/html_lib/test_merge_tags.py +++ /dev/null @@ -1,130 +0,0 @@ -import unittest - -from llm_web_kit.model.html_lib.base_func import document_fromstring -from llm_web_kit.model.html_lib.merge_tags import merge_list - - -class TestMergeList(unittest.TestCase): - def assertMergedList( - self, input_html, expected_text, expected_children_count=0, tag='ul' - ): - element = document_fromstring(input_html) - processed = merge_list(element) - - # 查找目标列表元素 - target = processed.find(f'.//{tag}') - self.assertIsNotNone(target, f'{tag}元素不存在') - - # 验证文本内容 - actual_text = target.text.strip() if target.text else '' - self.assertEqual(actual_text, expected_text, '合并文本不一致') - - # 验证子元素数量 - self.assertEqual(len(target), expected_children_count, '子元素未正确删除') - - return target - - def test_merge_ul_with_li(self): - html_str = """ - - """ - # Some problem here. But now the web classification model is trained with this behavior. - # TODO Fix this bug and retrain the model - # expected = "Item 1 Item 2" - expected = 'Item 1\nItem 2' - self.assertMergedList(html_str, expected) - - def test_merge_ol_with_nested_elements(self): - html_str = """ -
    -
  1. Text bold

  2. -
  3. Another
  4. -
- """ - # Some problem here. But now the web classification model is trained with this behavior. - # TODO Fix this bug and retrain the model - # expected = "Text bold Another" - expected = 'Text bold\nAnother' - self.assertMergedList(html_str, expected, tag='ol') - - def test_merge_dl_with_mixed_content(self): - html_str = """ -
Direct text -
Term
-
Definition
-
- """ - # Some problem here. But now the web classification model is trained with this behavior. - # TODO Fix this bug and retrain the model - # expected = "Direct text Term Definition" - expected = 'Direct text\n TermDefinition' - self.assertMergedList(html_str, expected, tag='dl') - - def test_preserve_attributes(self): - html_str = """ - - """ - target = self.assertMergedList(html_str, 'Item') - self.assertEqual(target.get('class'), 'list', 'class属性未保留') - self.assertEqual(target.get('id'), 'main-list', 'id属性未保留') - - def test_empty_list(self): - html_str = '' - self.assertMergedList(html_str, '', tag='ul') - - def test_deeply_nested_list(self): - html_str = """ - - """ - # 外层ul应该被处理,内层ul会被合并 - element = document_fromstring(html_str) - processed = merge_list(element) - - outer_ul = processed.find('.//ul') - - # Some problem here. But now the web classification model is trained with this behavior. - # TODO Fix this bug and retrain the model - # expected = "Level 1 Level 2" - expected = 'Level 1\n Level 2' - self.assertEqual(outer_ul.text.strip(), expected, '嵌套列表未正确合并') - self.assertEqual(len(outer_ul), 0, '子元素未删除') - - # 验证内层ul是否已被删除 - inner_ul = outer_ul.find('.//ul') - self.assertIsNone(inner_ul, '嵌套列表未被处理') - - def test_multiple_list_types(self): - html_str = """ -
-
  • Unordered
-
  1. Ordered
-
Term
-
- """ - element = document_fromstring(html_str) - processed = merge_list(element) - - for tag in ['ul', 'ol', 'dl']: - list_elem = processed.find(f'.//{tag}') - self.assertIsNotNone(list_elem, f'{tag}元素不存在') - self.assertTrue( - list_elem.text.strip() in ['Unordered', 'Ordered', 'Term'], - '文本内容错误', - ) - self.assertEqual(len(list_elem), 0, '子元素未删除') - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/llm_web_kit/model/html_lib/test_modify_tags.py b/tests/llm_web_kit/model/html_lib/test_modify_tags.py deleted file mode 100644 index 9f8c0591..00000000 --- a/tests/llm_web_kit/model/html_lib/test_modify_tags.py +++ /dev/null @@ -1,132 +0,0 @@ -import unittest - -from lxml import html - -from llm_web_kit.model.html_lib.modify_tags import (unwrap_single_child_tag, - wrap_bare_text) - - -class TestWrapBareText(unittest.TestCase): - - def test_text_with_child_element(self): - # 测试父元素有文本和子元素时,文本被包裹 - root = html.fromstring('
TextChild
') - wrapped = wrap_bare_text(root) - self.assertEqual(wrapped.text, '') # 原文本应被清空 - self.assertEqual(len(wrapped), 2) # 新span + 原span - self.assertEqual(wrapped[0].tag, 'span') - self.assertEqual(wrapped[0].text, 'Text') - self.assertEqual(wrapped[1].text, 'Child') - - def test_tail_after_child_element(self): - # 测试子元素尾部文本被包裹 - root = html.fromstring('
ChildTail
') - wrapped = wrap_bare_text(root) - span = wrapped.find('.//span') - self.assertEqual(span.tail, '') # 原尾部文本应被清空 - next_span = span.getnext() - self.assertEqual(next_span.tag, 'span') - self.assertEqual(next_span.text, 'Tail') - - def test_multiple_children_with_text(self): - # 测试多个子元素和文本混合的情况 - root = html.fromstring('
StartAMiddleBEnd
') - wrapped = wrap_bare_text(root) - # 预期结构:3个文本span + 原span + 原b - self.assertEqual(len(wrapped), 5) - self.assertEqual(wrapped[0].text, 'Start') - self.assertEqual(wrapped[1].tag, 'span') # 原span - self.assertEqual(wrapped[2].text, 'Middle') - self.assertEqual(wrapped[3].tag, 'b') # 原b标签 - - def test_nested_elements_text_wrapping(self): - # 测试嵌套元素中的文本处理 - root = html.fromstring('
D1S1

P1

S2
D2
') - wrapped = wrap_bare_text(root) - # 检查div层 - self.assertEqual(wrapped[0].text, 'D1') - self.assertEqual(wrapped[1].tag, 'span') - self.assertEqual(wrapped[2].text, 'D2') - # 检查span层 - span = wrapped[1] - self.assertEqual(span[0].text, 'S1') - self.assertEqual(span[1].tag, 'p') - self.assertEqual(span[2].text, 'S2') - - def test_no_wrapping_when_no_children(self): - # 测试没有子元素时不进行包裹 - root = html.fromstring('

Simple text

') - original_text = root.text - wrapped = wrap_bare_text(root) - self.assertEqual(len(wrapped), 0) # 没有子元素,不处理 - self.assertEqual(wrapped.text, original_text) - - -class TestUnwrapSingleChildTag(unittest.TestCase): - - def test_single_child_unwrapping(self): - # 测试单层解包和类合并 - root = html.fromstring( - "

" - ) - unwrapped = unwrap_single_child_tag(root) - p = unwrapped.find('.//p') - self.assertEqual(p.get('class'), 'parent|child') - # 检查div是否被移除 - self.assertIsNone(unwrapped.find('.//div')) - - def test_class_merging_edge_cases(self): - # 测试类合并的边界情况 - cases = [ - ('

', ''), # 都没有class - ('

', ''), # 空class - ("

", 'child'), - ("

", 'parent'), - ] - for html_str, expected in cases: - root = html.fromstring(f'{html_str}') - unwrapped = unwrap_single_child_tag(root) - p = unwrapped.find('.//p') - self.assertEqual(p.get('class'), expected) - - def test_multilevel_unwrapping(self): - # 测试多层嵌套解包 - root = html.fromstring( - """ - - -
-
-

-
-
- - - """ - ) - unwrapped = unwrap_single_child_tag(root) - p = unwrapped.find('.//p') - self.assertEqual(p.get('class'), 'l1|l2|l3') - # 检查祖先元素是否都被移除 - self.assertIsNone(unwrapped.find('.//div')) - self.assertIsNone(unwrapped.find('.//section')) - - def test_no_unwrap_with_multiple_children(self): - # 测试多个子元素时不解包 - root = html.fromstring('

A

B

') - original_structure = html.tostring(root) - unwrapped = unwrap_single_child_tag(root) - # 结构应保持不变 - self.assertEqual(html.tostring(unwrapped), original_structure) - - def test_root_element_not_unwrapped(self): - # 测试根元素不解包 - root = html.fromstring('

Content

') - original_tag = root.tag - unwrapped = unwrap_single_child_tag(root) - self.assertEqual(unwrapped.tag, original_tag) - self.assertEqual(len(unwrapped), 1) # 仍然只有一个p子元素 - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/llm_web_kit/model/html_lib/test_remove_tags.py b/tests/llm_web_kit/model/html_lib/test_remove_tags.py deleted file mode 100644 index 68d9a975..00000000 --- a/tests/llm_web_kit/model/html_lib/test_remove_tags.py +++ /dev/null @@ -1,190 +0,0 @@ -import unittest - -from lxml import html - -from llm_web_kit.model.html_lib.remove_tags import ( - all_tag_list_to_remove, build_tags_to_remove_map, is_blank_tag, - is_diplay_none, remove_all_tags, remove_blank_tags_recursive, - remove_invisible_tags, remove_tags) - - -class TestBuildTagsToRemoveMap(unittest.TestCase): - def test_map_structure(self): - tags_map = build_tags_to_remove_map() - # 验证基础分类键存在 - self.assertIn('document_metadata', tags_map) - self.assertIn('script', tags_map) - # 验证具体标签归属 - self.assertIn('iframe', tags_map['embed']) - self.assertIn('form', tags_map['form']) - self.assertIn('canvas', tags_map['script']) - - -class TestAllTagListToRemove(unittest.TestCase): - def test_tag_aggregation(self): - all_tags = all_tag_list_to_remove() - tags_map = build_tags_to_remove_map() - expected = [] - for v in tags_map.values(): - expected.extend(v) - # 验证总数一致且包含关键标签 - self.assertEqual(len(all_tags), len(expected)) - self.assertIn('iframe', all_tags) - - -class TestRemoveTags(unittest.TestCase): - def setUp(self): - self.sample_html = """ - - - -
- -

保留内容

- - """ - self.root = html.fromstring(self.sample_html) - - def test_remove_single_category(self): - # 测试移除script相关标签 - new_root = remove_tags(self.root, ['script']) - self.assertEqual(len(new_root.findall('.//script')), 0) - self.assertGreater(len(new_root.findall('.//form')), 0) - - def test_remove_multiple_categories(self): - # 同时移除表单和嵌入式内容 - new_root = remove_tags(self.root, ['form', 'embed']) - self.assertEqual(len(new_root.findall('.//form')), 0) - self.assertEqual(len(new_root.findall('.//embed')), 0) - self.assertGreater(len(new_root.findall('.//p')), 0) - - def test_remove_nested_tags(self): - # 测试嵌套标签移除 - html_content = ( - '
' - ) - root = html.fromstring(html_content) - new_root = remove_tags(root, ['form']) - self.assertEqual(len(new_root.findall('.//form')), 0) - self.assertIsNotNone(new_root.find('.//div')) - - -class TestRemoveAllTags(unittest.TestCase): - def test_full_removal(self): - html_content = """ -
- - -
- -

保留内容

-
- """ - root = html.fromstring(html_content) - new_root = remove_all_tags(root) - # 验证黑名单标签已移除 - self.assertEqual(len(new_root.findall('.//script')), 0) - self.assertEqual(len(new_root.findall('.//iframe')), 0) - # 验证非黑名单标签保留 - self.assertGreater(len(new_root.findall('.//p')), 0) - - -class TestIsDisplayNone(unittest.TestCase): - def test_display_detection(self): - # 标准display:none检测 - element = html.fromstring('
') - self.assertTrue(is_diplay_none(element)) - - def test_case_insensitive(self): - # 大小写不敏感检测 - element = html.fromstring('
') - self.assertTrue(is_diplay_none(element)) - - def test_partial_match(self): - # 不匹配部分匹配的情况 - element = html.fromstring('
') - self.assertTrue(is_diplay_none(element)) - - -class TestRemoveInvisibleTags(unittest.TestCase): - def test_invisible_removal(self): - html_content = """ -
-

隐藏内容

- 隐藏span -
可见内容
-
- """ - root = html.fromstring(html_content) - new_root = remove_invisible_tags(root) - # 验证隐藏元素已移除 - self.assertEqual(len(new_root.findall('.//p')), 0) - self.assertIsNotNone(new_root.find('.//div')) - - -class TestIsBlankTag(unittest.TestCase): - def test_img_tag(self): - # 图片标签永远不空白 - element = html.fromstring('') - self.assertFalse(is_blank_tag(element)) - - def test_plain_text(self): - # 包含真实文本 - element = html.fromstring('
真实内容
') - self.assertFalse(is_blank_tag(element)) - - def test_nested_blank(self): - # 多层空白结构 - element = html.fromstring('
') - self.assertTrue(is_blank_tag(element)) - - def test_tail_text(self): - # 包含尾部文本的情况 - element = html.fromstring('
尾部文本
') - self.assertFalse(is_blank_tag(element)) - - -class TestRemoveBlankTagsRecursive(unittest.TestCase): - def test_shallow_removal(self): - # 简单空白标签移除 - html_content = '

' - root = html.fromstring(html_content) - new_root = remove_blank_tags_recursive(root) - self.assertEqual(len(new_root.findall('.//p')), 0) - - def test_deep_removal(self): - # 深层嵌套空白标签 - html_content = """ -
-
-
-

-
-
-
-
- """ - root = html.fromstring(html_content) - new_root = remove_blank_tags_recursive(root) - # 验证所有空白标签已移除 - self.assertEqual(len(new_root.findall('.//section')), 0) - self.assertEqual(len(new_root.findall('.//div')), 0) - - def test_mixed_content(self): - # 混合空白和非空白内容 - html_content = """ -
-

-
真实内容
- -
- """ - root = html.fromstring(html_content) - new_root = remove_blank_tags_recursive(root) - # 验证空白标签移除,保留有效内容 - self.assertEqual(len(new_root.findall('.//p')), 0) - self.assertIsNotNone(new_root.find('.//div')) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/llm_web_kit/model/html_lib/test_simplify.py b/tests/llm_web_kit/model/html_lib/test_simplify.py index 148abbbb..30603c50 100644 --- a/tests/llm_web_kit/model/html_lib/test_simplify.py +++ b/tests/llm_web_kit/model/html_lib/test_simplify.py @@ -1,221 +1,22 @@ -import unittest -from lxml import html +from llm_web_kit.model.html_lib.simplify import general_simplify_html_str -from llm_web_kit.model.html_lib.simplify import (general_simplify, - general_simplify_html_str) +def test_general_simplify_html_str(): + # 测试用例1:简单的HTML字符串 + html_input = '

Title

Paragraph

' + # 假设简化后不变 + expected_output = '

Title

Paragraph

' + assert general_simplify_html_str(html_input) == expected_output -class TestGeneralSimplify(unittest.TestCase): - def test_general_simplify_with_title(self): - # 测试带有标题的HTML页面 - html_str = """ - - - Test Title - - -
-

Main Heading

-

This is a paragraph.

-
-

Hidden content

-
-
    -
  • Item 1
  • -
  • Item 2
  • -
-
- - - """ - root = html.fromstring(html_str) - title, simplified_root = general_simplify(root) - self.assertEqual(title, 'Test Title') - # 找到动态变更的`

`标签和``标签 - p_tags = simplified_root.xpath('//p') - self.assertTrue(all(p.tag == 'p' for p in p_tags)) - # 表结构未被保留 - ul_tag = simplified_root.xpath('//ul') - self.assertEqual(len(ul_tag), 1) - # `None`标签结构未被保留 - # 需要进一步检查具体的删除逻辑,但在此处假设所有不可见内容都被移除 - hidden_divs = simplified_root.xpath("//div[@style='display:none;']") - self.assertEqual(len(hidden_divs), 0) + # 测试用例2:空标签 + html_input = '

Title

Paragraph

' + # 假设简化后删除空标签 + expected_output = '

Title

Paragraph

' + assert general_simplify_html_str(html_input) == expected_output - def test_general_simplify_no_title(self): - # 测试没有标题的HTML页面 - html_str = """ - - -
-

No Title Here

-

This is a paragraph without a title.

-
- - - """ - root = html.fromstring(html_str) - title, simplified_root = general_simplify(root) - self.assertIsNone(title) - - def test_general_simplify_remove_invisible_tags(self): - # 测试移除不可见的HTML标签 - html_str = """ - - -
-

Visible content

-
-

Hidden content

-
-
- - - """ - root = html.fromstring(html_str) - title, simplified_root = general_simplify(root) - hidden_divs = simplified_root.xpath("//div[@style='display:none;']") - self.assertEqual(len(hidden_divs), 0) - - def test_general_simplify_merge_list(self): - # 测试合并列表标签 - html_str = """ - - -
    -
  • Item 1
  • -
  • Item 2
  • -
-
    -
  1. Item A
  2. -
  3. Item B
  4. -
- - - """ - root = html.fromstring(html_str) - title, simplified_root = general_simplify(root) - ul_tags = simplified_root.xpath('//ul') - ol_tags = simplified_root.xpath('//ol') - # 合并后的列表标签应保留 - self.assertEqual(len(ul_tags), 1) - self.assertEqual(len(ol_tags), 1) - # 但是li标签应该被移除 - li_tags = simplified_root.xpath('//li') - self.assertEqual(len(li_tags), 0) - - def test_general_simplify_remove_blank_tags(self): - # 测试移除空白标签 - html_str = """ - - -
-

- -
-

Content

-
-
- - - """ - root = html.fromstring(html_str) - title, simplified_root = general_simplify(root) - # 检查子节点是否被删除 - p_tags = simplified_root.xpath('//p') - self.assertEqual(len(p_tags), 1) - self.assertEqual(p_tags[0].text, 'Content') - - def test_general_simplify_unwrap_tags(self): - # 测试解包单子节点标签 - html_str = """ - - -
-

Content

-
- Text - - - """ - root = html.fromstring(html_str) - title, simplified_root = general_simplify(root) - # 检查`div`标签是否被解包 - div_tags = simplified_root.xpath('//div') - self.assertEqual(len(div_tags), 0) - # 检查`p`标签是否保留 - p_tags = simplified_root.xpath('//p') - self.assertEqual(len(p_tags), 1) - self.assertEqual(p_tags[0].text, 'Content') - - def test_general_simplify_html_str(self): - # 测试`general_simplify_html_str`函数 - html_str = """ - - -
-

Hello World

- -
- - - """ - simplified_html_str = general_simplify_html_str(html_str) - # 确保脚本标记被移除 - self.assertNotIn('