-
Main Heading
-This is a paragraph.
- --
-
- Item 1 -
- Item 2 -
diff --git a/docs/llm_web_kit/model/html_simplify_classify.md b/docs/llm_web_kit/model/html_simplify_classify.md index f4f77e4f..bacba487 100644 --- a/docs/llm_web_kit/model/html_simplify_classify.md +++ b/docs/llm_web_kit/model/html_simplify_classify.md @@ -14,23 +14,7 @@ ## HTML simplify -该部分用于简化HTML从而提高网页布局分类的效果。在 `llm_web_kit/model/html_lib/html_lib` 目录下的 `simplify.py` 文件中的 `general_simplify_html_str` 函数实现了对html字符串的简化操作。 - -默认路径为`~/.llm-web-kit.jsonc`中需要使用如下配置,可以自动下载模型: - -```json -{ - "resources": { - "common":{ - "cache_path": "~/.llm_web_kit_cache" - }, - "html_cls-25m2": { - "download_path": "s3://web-parse-huawei/shared_resource/html_layout_cls/html_cls_25m2.zip", - "md5": "e15ea22a9aa65aa8c7c3a0e3c2e0c98a" - }, - } -} -``` +该部分用于简化HTML从而提高网页布局分类的效果。在 `llm_web_kit/model/html_lib/simplify.py` 文件中的 `general_simplify_html_str` 函数实现了对html字符串的简化操作。该部分的功能实现已经迁移到 `html_alg_lib` 库中,目前直接调用 `html_alg_lib.simplify` 中的 `process_to_cls_alg_html` 函数。 使用方法如下: @@ -47,7 +31,7 @@ print(simp_html) ## HTML 分类 -将简化后的html分类article, forum, other三个类别。在`llm_web_kit/model/html_layout_cls.py`中,使用`HTMLLayoutClassifier`类完成自动下载checkpoint和推理过程。 +将简化后的html分类Article, Forum_or_Article_with_commentsection, Content Listing, Other四个类别。在`llm_web_kit/model/html_layout_cls.py`中,使用`HTMLLayoutClassifier`类完成自动下载checkpoint和推理过程。 使用方法如下: @@ -59,3 +43,19 @@ html_str_input = ['layout1', 'layout2'] layout_type = model.predict(html_str_input) print(layout_type) ``` + +默认路径为`~/.llm-web-kit.jsonc`中需要使用如下配置,可以自动下载模型: + +```json +{ + "resources": { + "common":{ + "cache_path": "~/.llm_web_kit_cache" + }, + "html_cls-25m2": { + "download_path": "s3://web-parse-huawei/shared_resource/html_layout_cls/html_cls_25m4.zip", + "md5": "31b4889b4d9c8a1a6da7a5c58270e611" + }, + } +} +``` diff --git a/llm_web_kit/model/html_classify/model.py b/llm_web_kit/model/html_classify/model.py index cdb1d637..6454b766 100644 --- a/llm_web_kit/model/html_classify/model.py +++ b/llm_web_kit/model/html_classify/model.py @@ -1,3 +1,5 @@ +import os + import torch from llm_web_kit.model.resource_utils import import_transformer @@ -6,13 +8,12 @@ class Markuplm(): def __init__(self, path, device): self.path = path - self.model_path = self.path + '/markuplm-base' - self.checkpoint_path = self.path + '/markuplm_202501222031_epoch_2.pt' + self.model_path = os.path.join(self.path, 'markuplm-base') self.device = device - self.num_labels = 3 + self.num_labels = 4 self.max_tokens = 512 - self.label2id = {0:'article', 1:'forum', 2:'other'} + self.label2id = {0: 'Article', 1: 'Forum_or_Article_with_commentsection', 2: 'Content Listing', 3: 'Other'} self.model = self.load_model() self.tokenizer = self.load_tokenizer() @@ -25,8 +26,6 @@ def load_tokenizer(self): def load_model(self): transformers = import_transformer() model = transformers.MarkupLMForSequenceClassification.from_pretrained(self.model_path, num_labels=self.num_labels) - # load checkpoint - model.load_state_dict(torch.load(self.checkpoint_path, map_location=self.device)) model.to(self.device) model.eval() diff --git a/llm_web_kit/model/html_layout_cls.py b/llm_web_kit/model/html_layout_cls.py index e4c86694..981a9e9a 100644 --- a/llm_web_kit/model/html_layout_cls.py +++ b/llm_web_kit/model/html_layout_cls.py @@ -15,8 +15,8 @@ def __init__(self, model_path: str = None, device: str = 'cuda'): self.model = Markuplm(model_path, device) def auto_download(self) -> str: - """Default download the html_cls_25m2.zip model.""" - resource_name = 'html_cls-25m2' + """Default download the html_cls_25m4.zip model.""" + resource_name = 'html_cls-25m4' resource_config = load_config()['resources'] print(resource_config) model_config: dict = resource_config[resource_name] diff --git a/llm_web_kit/model/html_lib/base_func.py b/llm_web_kit/model/html_lib/base_func.py deleted file mode 100644 index bcb7e7e4..00000000 --- a/llm_web_kit/model/html_lib/base_func.py +++ /dev/null @@ -1,97 +0,0 @@ -import re - -from lxml import html - - -def extract_tag_text(element: html.HtmlElement) -> str: - """Extract text from an html element. - - Args: - element (html.HtmlElement): The element to extract text from - - Returns: - str: The text extracted from the element - """ - # 如果元素是img标签,返回其alt属性 - if element.tag == 'img': - return element.get('alt', '') - - # 如果元素没有子节点,返回其文本 - if len(element) == 0: - return element.text if element.text else '' - - # 如果元素有子节点,递归提取每个子节点的文本,并根据元素类型决定是否添加换行符 - newline_tags = ['p', 'br', 'div', 'li', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'tr'] - texts = [] - if element.text: # 添加未被标签包裹的文本 - texts.append(element.text) - for child in element: - child_text = extract_tag_text(child) - if child.tag in newline_tags: - child_text += '\n' - texts.append(child_text) - if child.tail: # 添加标签后的文本 - texts.append(child.tail) - # 将只有空格的行变成空行 - texts = [text if text.strip() else '\n' for text in texts] - # 使用正则表达式将连续的换行符替换为一个 - return re.sub('\n+', '\n', ''.join(texts)) - - -def get_title(root: html.HtmlElement) -> str: - """Get the title of the html page. - - Args: - root (html.HtmlElement): The root of the html page - - Returns: - str: The title of the html page - """ - title = root.xpath('//title/text()') - - return title[0] if len(title) > 0 else None - - -def remove_blank_text(root: html.HtmlElement): - """Remove blank text in html element. Skip
tag and tag and
- there children.
-
- Args:
- root (html.HtmlElement): The root of the html page
-
- Returns:
- html.HtmlElement: The root of the html page after removing blank text
- """
-
- skip_elements = set()
- for element in root.iter():
- if element.tag in ['pre', 'code']:
- skip_elements.add(element)
- skip_elements.update(element.iterdescendants())
-
- for element in root.iter():
- if element in skip_elements:
- continue
- if element.text and not element.text.strip():
- element.text = None
- if element.tail and not element.tail.strip():
- element.tail = None
- return root
-
-
-def document_fromstring(html_str: str):
- """Parse html string to lxml.html.HtmlElement default remove blank text,
- comments, processing instructions.
-
- Args:
- html_str (str): The html string to parse
-
- Returns:
- html.HtmlElement: The root of the html page
- """
- parser = html.HTMLParser(
- remove_blank_text=True, remove_comments=True, remove_pis=True
- )
- root = html.document_fromstring(html_str, parser=parser)
- root = remove_blank_text(root)
- return root
diff --git a/llm_web_kit/model/html_lib/merge_tags.py b/llm_web_kit/model/html_lib/merge_tags.py
deleted file mode 100644
index 564785c3..00000000
--- a/llm_web_kit/model/html_lib/merge_tags.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from lxml import html
-
-from llm_web_kit.model.html_lib.base_func import extract_tag_text
-
-
-def merge_list(root: html.HtmlElement) -> html.HtmlElement:
- # merge ul ol and dl
- list_tag_names = ['ul', 'ol', 'dl']
- for tag in list_tag_names:
- for list_elem in root.findall(f'.//{tag}'):
- list_inner_text = extract_tag_text(list_elem)
- # clean up the list_elem
- for child in list_elem.getchildren():
- if child.getparent() is not None:
- child.drop_tree()
- list_elem.text = list_inner_text
- return root
diff --git a/llm_web_kit/model/html_lib/modify_tags.py b/llm_web_kit/model/html_lib/modify_tags.py
deleted file mode 100644
index 726772f3..00000000
--- a/llm_web_kit/model/html_lib/modify_tags.py
+++ /dev/null
@@ -1,46 +0,0 @@
-from lxml import html
-
-
-def wrap_bare_text(root: html.HtmlElement) -> html.HtmlElement:
- for elem in root.iter():
- # continue if the element has no child
- if len(elem) == 0:
- continue
- else:
- # check if the element has text and tail
- if elem.text and elem.text.strip():
- # wrap the text in a span tag
- span = html.Element('span')
- span.text = elem.text
- elem.text = ''
- elem.insert(0, span)
- for child in elem:
- if child.tail and child.tail.strip():
- # wrap the tail in a span tag
- span = html.Element('span')
- span.text = child.tail
- child.tail = ''
- child.addnext(span)
- return root
-
-
-def unwrap_single_child_tag(root: html.HtmlElement) -> html.HtmlElement:
-
- def merge_class(A_elem: html.HtmlElement, B_elem: html.HtmlElement) -> str:
- A_str = A_elem.get('class', '').strip()
- B_str = B_elem.get('class', '').strip()
- if A_str and B_str:
- return f'{A_str}|{B_str}'
- if A_str:
- return A_str
- if B_str:
- return B_str
- return ''
-
- for tag in root.iter():
- children = list(tag.getchildren())
- if len(children) == 1 and tag.getparent() is not None:
- child = children[0]
- child.set('class', merge_class(tag, child))
- tag.drop_tag()
- return root
diff --git a/llm_web_kit/model/html_lib/remove_tags.py b/llm_web_kit/model/html_lib/remove_tags.py
deleted file mode 100644
index b978ff4f..00000000
--- a/llm_web_kit/model/html_lib/remove_tags.py
+++ /dev/null
@@ -1,133 +0,0 @@
-from lxml import html
-
-
-def build_tags_to_remove_map() -> dict[str, list[str]]:
- return {
- 'document_metadata': ['base', 'head', 'link', 'meta', 'style', 'title'],
- 'content_sectioning': ['aside', 'footer', 'header', 'nav'],
- 'text_content': ['menu', 'hr'],
- 'demarcating_edits': ['del'],
- # "image_and_multimedia": ["audio", "map", "track", "video", "svg"],
- 'image_and_multimedia': ['audio', 'map', 'track', 'svg'],
- 'interactive': ['dialog'],
- 'embed': [
- 'embed',
- 'fencedframe',
- 'iframe',
- 'object',
- 'picture',
- 'portal',
- 'source',
- ],
- 'script': ['canvas', 'script'],
- 'form': [
- 'button',
- 'datalist',
- 'fieldset',
- 'form',
- 'input',
- 'label',
- 'legend',
- 'meter',
- 'optgroup',
- 'option',
- 'output',
- 'progress',
- 'select',
- 'textarea',
- ],
- }
-
-
-def all_tag_list_to_remove() -> list[str]:
- all_tag_list = []
- for tag_list in build_tags_to_remove_map().values():
- all_tag_list.extend(tag_list)
- return all_tag_list
-
-
-def remove_tags(root: html.HtmlElement, tag_type_list: list[str]) -> html.HtmlElement:
- """Remove all tags of a certain type from the html page.
-
- Args:
- root (etree.ElementTree): The root of the html page
- tag_type (str): The type of tag to remove
-
- Returns:
- etree.ElementTree: The root of the html page with the tags removed
- """
- tags_to_remove = []
- for tag in tag_type_list:
- for element in root.findall(f'.//{tag}'):
- tags_to_remove.append(element)
- for element in tags_to_remove:
- if element.getparent() is not None:
- element.drop_tree()
- return root
-
-
-def remove_all_tags(root: html.HtmlElement) -> html.HtmlElement:
- """Remove all tags from the html page.
-
- Args:
- root (etree.ElementTree): The root of the html page
-
- Returns:
- etree.ElementTree: The root of the html page with the tags removed
- """
- all_tag_list = all_tag_list_to_remove()
- return remove_tags(root, all_tag_list)
-
-
-def is_diplay_none(element: html.HtmlElement) -> bool:
- style = element.get('style', '').replace(' ', '').lower()
- if 'display:none' in style:
- return True
-
-
-def remove_invisible_tags(root: html.HtmlElement) -> html.HtmlElement:
- tags_to_remove = []
- for element in root.iter():
- if is_diplay_none(element):
- tags_to_remove.append(element)
- for element in tags_to_remove:
- if element.getparent() is not None:
- element.drop_tree()
- return root
-
-
-def is_blank_tag(root: html.HtmlElement) -> bool:
- """Check if a tag is nonblank None blank tags if this is an image or if it
- has text or if it has children with tail text.
-
- Args:
- tag (html.HtmlElement): The tag to check
-
- Returns:
- bool: True if the tag is nonblank, False otherwise
- """
- if root.tag == 'img':
- return False
- if root.text and root.text.strip():
- return False
- # check children's tail
- childern_tail = [child.tail for child in root.iterchildren()]
- if any([tail and tail.strip() for tail in childern_tail]):
- return False
- for child in root.iterchildren():
- if not is_blank_tag(child):
- return False
- return True
-
-
-def remove_blank_tags_recursive(
- root: html.HtmlElement,
-) -> tuple[bool, html.HtmlElement]:
- tags_to_remove = []
- for tag in root.iter():
- if is_blank_tag(tag) and tag.getparent() is not None:
- tags_to_remove.append(tag)
- for tag in tags_to_remove:
- if tag.getparent() is not None:
- tag.drop_tree()
- return root
diff --git a/llm_web_kit/model/html_lib/simplify.py b/llm_web_kit/model/html_lib/simplify.py
index 998d7554..e6e83a18 100644
--- a/llm_web_kit/model/html_lib/simplify.py
+++ b/llm_web_kit/model/html_lib/simplify.py
@@ -1,53 +1,6 @@
-from lxml import html
-from llm_web_kit.model.html_lib.base_func import document_fromstring, get_title
-from llm_web_kit.model.html_lib.merge_tags import merge_list
-from llm_web_kit.model.html_lib.modify_tags import (unwrap_single_child_tag,
- wrap_bare_text)
-from llm_web_kit.model.html_lib.remove_tags import (
- remove_all_tags, remove_blank_tags_recursive, remove_invisible_tags)
-from llm_web_kit.model.html_lib.unwrap_tags import unwrap_all_tags
-
-
-def general_simplify(root: html.HtmlElement) -> tuple[str, html.HtmlElement]:
- # get the title of the html page
- title = get_title(root)
-
- # remove all invisible tags
- root = remove_invisible_tags(root)
-
- # unwrap all tags we consider can directly unwrap
- # unwrap means remove the tag and keep the children
- root = unwrap_all_tags(root)
-
- # wrap all bare text in span tags
- # the bare text means the tag.text and tag.child.tail is not blank and len(tag) > 0
- #
- # tag.text
- #
- # text
- #
- # child.tail
- #
- root = wrap_bare_text(root)
-
- # remove all tags (and it's children) we consider can directly remove
- root = remove_all_tags(root)
-
- # extract text from all children of list tags and set the text as the text of the list tags
- root = merge_list(root)
-
- # remove all tags
- root = remove_blank_tags_recursive(root)
-
- # unwrap all tags have only one child
- root = unwrap_single_child_tag(root)
-
- return title, root
+from html_alg_lib.simplify import process_to_cls_alg_html
def general_simplify_html_str(html_str: str) -> str:
- root = document_fromstring(html_str)
- _, simplified_root = general_simplify(root)
- simplified_html_str = html.tostring(simplified_root)
- return simplified_html_str.decode('utf-8')
+ return process_to_cls_alg_html(html_str)
diff --git a/llm_web_kit/model/html_lib/unwrap_tags.py b/llm_web_kit/model/html_lib/unwrap_tags.py
deleted file mode 100644
index c839fc37..00000000
--- a/llm_web_kit/model/html_lib/unwrap_tags.py
+++ /dev/null
@@ -1,53 +0,0 @@
-from lxml import html
-
-
-def build_tags_to_unwrap_map():
- return {
- 'inline_text_semantics': [
- 'a',
- 'abbr',
- 'b',
- 'bdi',
- 'bdo',
- 'br',
- 'cite',
- 'code',
- 'data',
- 'dfn',
- 'em',
- 'i',
- 'kbd',
- 'mark',
- 'q',
- 'rp',
- 'rt',
- 'ruby',
- 's',
- 'samp',
- 'small',
- 'span',
- 'strong',
- 'sub',
- 'sup',
- 'time',
- 'u',
- 'var',
- 'wbr',
- ],
- 'demarcating_edits:': ['ins'],
- }
-
-
-def get_all_tags_to_unwrap():
- all_tags = []
- for tag_list in build_tags_to_unwrap_map().values():
- all_tags.extend(tag_list)
- return all_tags
-
-
-def unwrap_all_tags(root: html.HtmlElement) -> html.HtmlElement:
- all_tags = get_all_tags_to_unwrap()
- for tag in all_tags:
- for element in root.findall(f'.//{tag}'):
- element.drop_tag()
- return root
diff --git a/requirements/runtime.txt b/requirements/runtime.txt
index 8a1bbcea..8ac04789 100644
--- a/requirements/runtime.txt
+++ b/requirements/runtime.txt
@@ -5,6 +5,7 @@ click==8.1.8
commentjson==0.9.0
fasttext-wheel==0.9.2
filelock==3.16.1
+html-alg-lib==2.0.2
jieba-fast==0.53
langdetect_zh==1.0.4
lightgbm==4.5.0
diff --git a/tests/llm_web_kit/model/html_classify/test_model.py b/tests/llm_web_kit/model/html_classify/test_model.py
index 7e9f9b1a..da12fde6 100644
--- a/tests/llm_web_kit/model/html_classify/test_model.py
+++ b/tests/llm_web_kit/model/html_classify/test_model.py
@@ -26,7 +26,6 @@ def test_init(self, mock_load, mock_model_from_pretrained, mock_tokenizer_from_p
self.assertEqual(markuplm.device, device)
mock_model_from_pretrained.assert_called_once_with(markuplm.model_path, num_labels=markuplm.num_labels)
mock_tokenizer_from_pretrained.assert_called_once_with(markuplm.model_path)
- mock_load.assert_called_once_with(markuplm.checkpoint_path, map_location=device)
@patch('transformers.MarkupLMProcessor.from_pretrained')
@patch('transformers.MarkupLMForSequenceClassification.from_pretrained')
diff --git a/tests/llm_web_kit/model/html_lib/test_base_func.py b/tests/llm_web_kit/model/html_lib/test_base_func.py
deleted file mode 100644
index acfd2c1c..00000000
--- a/tests/llm_web_kit/model/html_lib/test_base_func.py
+++ /dev/null
@@ -1,139 +0,0 @@
-import unittest
-
-from lxml import html
-
-from llm_web_kit.model.html_lib.base_func import (document_fromstring,
- extract_tag_text, get_title,
- remove_blank_text)
-
-
-class TestExtractTagText(unittest.TestCase):
- def test_img_tag(self):
- element = html.fromstring('
')
- self.assertEqual(extract_tag_text(element), 'test alt')
-
- def test_img_tag_no_alt(self):
- element = html.fromstring('
')
- self.assertEqual(extract_tag_text(element), '')
-
- def test_plain_text(self):
- element = html.fromstring('Hello World
')
- self.assertEqual(extract_tag_text(element), 'Hello World')
-
- def test_nested_elements(self):
- element = html.fromstring('Text inside span tail')
- self.assertEqual(extract_tag_text(element), 'Text inside span tail')
-
- def test_newline_tags(self):
- element = html.fromstring('Line1
Line2
')
- result = extract_tag_text(element)
- # Expect "Line1\nLine2\n" after newline processing
- self.assertEqual(result.replace('\n', '|'), 'Line1|Line2|')
-
- def test_mixed_content(self):
- element = html.fromstring(
- """
-
- Before
- bold text
-
- italic
- tail
-
- after
-
- """
- )
- # Some problem here. But now the web classification model is trained with this behavior.
- # TODO Fix this bug and retrain the model
- # expected = "Before bold text italic tail after"
- expected = 'Before bold text italic tail after'
- self.assertEqual(extract_tag_text(element).replace('\n', ' ').strip(), expected)
-
- def test_whitespace_handling(self):
- # Some problem here. But now the web classification model is trained with this behavior.
- # TODO Fix this bug and retrain the model
- # expected = "\n"
- element = html.fromstring(' \t\n ')
- self.assertEqual(extract_tag_text(element), ' \t\n ')
-
-
-class TestGetTitle(unittest.TestCase):
- def test_has_title(self):
- root = html.fromstring('Test Title ')
- self.assertEqual(get_title(root), 'Test Title')
-
- def test_no_title(self):
- root = html.fromstring('')
- self.assertIsNone(get_title(root))
-
- def test_multiple_titles(self):
- root = html.fromstring(
- 'First Second '
- )
- self.assertEqual(get_title(root), 'First')
-
-
-class TestRemoveBlankText(unittest.TestCase):
- def test_remove_whitespace_nodes(self):
- root = html.fromstring(
- """
-
-
-
-
-
-
- """
- )
- processed = remove_blank_text(root)
-
- # Check normal elements
- p = processed.find('.//p')
- self.assertIsNone(p.text)
-
- span = processed.find('.//span')
- self.assertIsNone(span.text)
-
- # Check preserved elements
- pre = processed.find('.//pre')
- self.assertEqual(pre.text.strip(), '', 'Pre text should preserve whitespace')
- self.assertEqual(pre.text, ' ')
-
- code = processed.find('.//code')
- self.assertEqual(code.text, ' ')
-
- def test_keep_trailing_text(self):
- root = html.fromstring('Hello World')
- processed = remove_blank_text(root)
- b = processed.find('.//b')
- self.assertIsNone(b.text)
- self.assertEqual(b.tail, 'World')
-
-
-class TestDocumentFromString(unittest.TestCase):
- def test_basic_parsing(self):
- html_str = """
-
-
- Test
-
- Content
-
-
- """
- root = document_fromstring(html_str)
- self.assertEqual(root.tag, 'html')
- self.assertIsNotNone(root.find('.//body/p'))
-
- def test_whitespace_removal(self):
- html_str = '
'
- root = document_fromstring(html_str)
- div = root.find('.//div')
- self.assertIsNone(div.text)
- p = div.find('.//p')
- self.assertIsNone(p.text)
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/tests/llm_web_kit/model/html_lib/test_merge_tags.py b/tests/llm_web_kit/model/html_lib/test_merge_tags.py
deleted file mode 100644
index 9bb73f1c..00000000
--- a/tests/llm_web_kit/model/html_lib/test_merge_tags.py
+++ /dev/null
@@ -1,130 +0,0 @@
-import unittest
-
-from llm_web_kit.model.html_lib.base_func import document_fromstring
-from llm_web_kit.model.html_lib.merge_tags import merge_list
-
-
-class TestMergeList(unittest.TestCase):
- def assertMergedList(
- self, input_html, expected_text, expected_children_count=0, tag='ul'
- ):
- element = document_fromstring(input_html)
- processed = merge_list(element)
-
- # 查找目标列表元素
- target = processed.find(f'.//{tag}')
- self.assertIsNotNone(target, f'{tag}元素不存在')
-
- # 验证文本内容
- actual_text = target.text.strip() if target.text else ''
- self.assertEqual(actual_text, expected_text, '合并文本不一致')
-
- # 验证子元素数量
- self.assertEqual(len(target), expected_children_count, '子元素未正确删除')
-
- return target
-
- def test_merge_ul_with_li(self):
- html_str = """
-
- - Item 1
- - Item 2
-
- """
- # Some problem here. But now the web classification model is trained with this behavior.
- # TODO Fix this bug and retrain the model
- # expected = "Item 1 Item 2"
- expected = 'Item 1\nItem 2'
- self.assertMergedList(html_str, expected)
-
- def test_merge_ol_with_nested_elements(self):
- html_str = """
-
- Text bold
- - Another
-
- """
- # Some problem here. But now the web classification model is trained with this behavior.
- # TODO Fix this bug and retrain the model
- # expected = "Text bold Another"
- expected = 'Text bold\nAnother'
- self.assertMergedList(html_str, expected, tag='ol')
-
- def test_merge_dl_with_mixed_content(self):
- html_str = """
- Direct text
- - Term
- - Definition
-
- """
- # Some problem here. But now the web classification model is trained with this behavior.
- # TODO Fix this bug and retrain the model
- # expected = "Direct text Term Definition"
- expected = 'Direct text\n TermDefinition'
- self.assertMergedList(html_str, expected, tag='dl')
-
- def test_preserve_attributes(self):
- html_str = """
-
- - Item
-
- """
- target = self.assertMergedList(html_str, 'Item')
- self.assertEqual(target.get('class'), 'list', 'class属性未保留')
- self.assertEqual(target.get('id'), 'main-list', 'id属性未保留')
-
- def test_empty_list(self):
- html_str = '
'
- self.assertMergedList(html_str, '', tag='ul')
-
- def test_deeply_nested_list(self):
- html_str = """
-
- -
- Level 1
-
- - Level 2
-
-
-
- """
- # 外层ul应该被处理,内层ul会被合并
- element = document_fromstring(html_str)
- processed = merge_list(element)
-
- outer_ul = processed.find('.//ul')
-
- # Some problem here. But now the web classification model is trained with this behavior.
- # TODO Fix this bug and retrain the model
- # expected = "Level 1 Level 2"
- expected = 'Level 1\n Level 2'
- self.assertEqual(outer_ul.text.strip(), expected, '嵌套列表未正确合并')
- self.assertEqual(len(outer_ul), 0, '子元素未删除')
-
- # 验证内层ul是否已被删除
- inner_ul = outer_ul.find('.//ul')
- self.assertIsNone(inner_ul, '嵌套列表未被处理')
-
- def test_multiple_list_types(self):
- html_str = """
-
- - Unordered
- - Ordered
- - Term
-
- """
- element = document_fromstring(html_str)
- processed = merge_list(element)
-
- for tag in ['ul', 'ol', 'dl']:
- list_elem = processed.find(f'.//{tag}')
- self.assertIsNotNone(list_elem, f'{tag}元素不存在')
- self.assertTrue(
- list_elem.text.strip() in ['Unordered', 'Ordered', 'Term'],
- '文本内容错误',
- )
- self.assertEqual(len(list_elem), 0, '子元素未删除')
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/tests/llm_web_kit/model/html_lib/test_modify_tags.py b/tests/llm_web_kit/model/html_lib/test_modify_tags.py
deleted file mode 100644
index 9f8c0591..00000000
--- a/tests/llm_web_kit/model/html_lib/test_modify_tags.py
+++ /dev/null
@@ -1,132 +0,0 @@
-import unittest
-
-from lxml import html
-
-from llm_web_kit.model.html_lib.modify_tags import (unwrap_single_child_tag,
- wrap_bare_text)
-
-
-class TestWrapBareText(unittest.TestCase):
-
- def test_text_with_child_element(self):
- # 测试父元素有文本和子元素时,文本被包裹
- root = html.fromstring('TextChild')
- wrapped = wrap_bare_text(root)
- self.assertEqual(wrapped.text, '') # 原文本应被清空
- self.assertEqual(len(wrapped), 2) # 新span + 原span
- self.assertEqual(wrapped[0].tag, 'span')
- self.assertEqual(wrapped[0].text, 'Text')
- self.assertEqual(wrapped[1].text, 'Child')
-
- def test_tail_after_child_element(self):
- # 测试子元素尾部文本被包裹
- root = html.fromstring('ChildTail')
- wrapped = wrap_bare_text(root)
- span = wrapped.find('.//span')
- self.assertEqual(span.tail, '') # 原尾部文本应被清空
- next_span = span.getnext()
- self.assertEqual(next_span.tag, 'span')
- self.assertEqual(next_span.text, 'Tail')
-
- def test_multiple_children_with_text(self):
- # 测试多个子元素和文本混合的情况
- root = html.fromstring('StartAMiddleBEnd')
- wrapped = wrap_bare_text(root)
- # 预期结构:3个文本span + 原span + 原b
- self.assertEqual(len(wrapped), 5)
- self.assertEqual(wrapped[0].text, 'Start')
- self.assertEqual(wrapped[1].tag, 'span') # 原span
- self.assertEqual(wrapped[2].text, 'Middle')
- self.assertEqual(wrapped[3].tag, 'b') # 原b标签
-
- def test_nested_elements_text_wrapping(self):
- # 测试嵌套元素中的文本处理
- root = html.fromstring('D1S1P1
S2D2')
- wrapped = wrap_bare_text(root)
- # 检查div层
- self.assertEqual(wrapped[0].text, 'D1')
- self.assertEqual(wrapped[1].tag, 'span')
- self.assertEqual(wrapped[2].text, 'D2')
- # 检查span层
- span = wrapped[1]
- self.assertEqual(span[0].text, 'S1')
- self.assertEqual(span[1].tag, 'p')
- self.assertEqual(span[2].text, 'S2')
-
- def test_no_wrapping_when_no_children(self):
- # 测试没有子元素时不进行包裹
- root = html.fromstring('Simple text
')
- original_text = root.text
- wrapped = wrap_bare_text(root)
- self.assertEqual(len(wrapped), 0) # 没有子元素,不处理
- self.assertEqual(wrapped.text, original_text)
-
-
-class TestUnwrapSingleChildTag(unittest.TestCase):
-
- def test_single_child_unwrapping(self):
- # 测试单层解包和类合并
- root = html.fromstring(
- ""
- )
- unwrapped = unwrap_single_child_tag(root)
- p = unwrapped.find('.//p')
- self.assertEqual(p.get('class'), 'parent|child')
- # 检查div是否被移除
- self.assertIsNone(unwrapped.find('.//div'))
-
- def test_class_merging_edge_cases(self):
- # 测试类合并的边界情况
- cases = [
- ('', ''), # 都没有class
- ('', ''), # 空class
- ("", 'child'),
- ("", 'parent'),
- ]
- for html_str, expected in cases:
- root = html.fromstring(f'{html_str}')
- unwrapped = unwrap_single_child_tag(root)
- p = unwrapped.find('.//p')
- self.assertEqual(p.get('class'), expected)
-
- def test_multilevel_unwrapping(self):
- # 测试多层嵌套解包
- root = html.fromstring(
- """
-
-
-
-
-
-
-
-
-
- """
- )
- unwrapped = unwrap_single_child_tag(root)
- p = unwrapped.find('.//p')
- self.assertEqual(p.get('class'), 'l1|l2|l3')
- # 检查祖先元素是否都被移除
- self.assertIsNone(unwrapped.find('.//div'))
- self.assertIsNone(unwrapped.find('.//section'))
-
- def test_no_unwrap_with_multiple_children(self):
- # 测试多个子元素时不解包
- root = html.fromstring('A
B
')
- original_structure = html.tostring(root)
- unwrapped = unwrap_single_child_tag(root)
- # 结构应保持不变
- self.assertEqual(html.tostring(unwrapped), original_structure)
-
- def test_root_element_not_unwrapped(self):
- # 测试根元素不解包
- root = html.fromstring('Content
')
- original_tag = root.tag
- unwrapped = unwrap_single_child_tag(root)
- self.assertEqual(unwrapped.tag, original_tag)
- self.assertEqual(len(unwrapped), 1) # 仍然只有一个p子元素
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/tests/llm_web_kit/model/html_lib/test_remove_tags.py b/tests/llm_web_kit/model/html_lib/test_remove_tags.py
deleted file mode 100644
index 68d9a975..00000000
--- a/tests/llm_web_kit/model/html_lib/test_remove_tags.py
+++ /dev/null
@@ -1,190 +0,0 @@
-import unittest
-
-from lxml import html
-
-from llm_web_kit.model.html_lib.remove_tags import (
- all_tag_list_to_remove, build_tags_to_remove_map, is_blank_tag,
- is_diplay_none, remove_all_tags, remove_blank_tags_recursive,
- remove_invisible_tags, remove_tags)
-
-
-class TestBuildTagsToRemoveMap(unittest.TestCase):
- def test_map_structure(self):
- tags_map = build_tags_to_remove_map()
- # 验证基础分类键存在
- self.assertIn('document_metadata', tags_map)
- self.assertIn('script', tags_map)
- # 验证具体标签归属
- self.assertIn('iframe', tags_map['embed'])
- self.assertIn('form', tags_map['form'])
- self.assertIn('canvas', tags_map['script'])
-
-
-class TestAllTagListToRemove(unittest.TestCase):
- def test_tag_aggregation(self):
- all_tags = all_tag_list_to_remove()
- tags_map = build_tags_to_remove_map()
- expected = []
- for v in tags_map.values():
- expected.extend(v)
- # 验证总数一致且包含关键标签
- self.assertEqual(len(all_tags), len(expected))
- self.assertIn('iframe', all_tags)
-
-
-class TestRemoveTags(unittest.TestCase):
- def setUp(self):
- self.sample_html = """
-
-
-
-
-
- 保留内容
-
- """
- self.root = html.fromstring(self.sample_html)
-
- def test_remove_single_category(self):
- # 测试移除script相关标签
- new_root = remove_tags(self.root, ['script'])
- self.assertEqual(len(new_root.findall('.//script')), 0)
- self.assertGreater(len(new_root.findall('.//form')), 0)
-
- def test_remove_multiple_categories(self):
- # 同时移除表单和嵌入式内容
- new_root = remove_tags(self.root, ['form', 'embed'])
- self.assertEqual(len(new_root.findall('.//form')), 0)
- self.assertEqual(len(new_root.findall('.//embed')), 0)
- self.assertGreater(len(new_root.findall('.//p')), 0)
-
- def test_remove_nested_tags(self):
- # 测试嵌套标签移除
- html_content = (
- ''
- )
- root = html.fromstring(html_content)
- new_root = remove_tags(root, ['form'])
- self.assertEqual(len(new_root.findall('.//form')), 0)
- self.assertIsNotNone(new_root.find('.//div'))
-
-
-class TestRemoveAllTags(unittest.TestCase):
- def test_full_removal(self):
- html_content = """
-
-
-
-
-
- 保留内容
-
- """
- root = html.fromstring(html_content)
- new_root = remove_all_tags(root)
- # 验证黑名单标签已移除
- self.assertEqual(len(new_root.findall('.//script')), 0)
- self.assertEqual(len(new_root.findall('.//iframe')), 0)
- # 验证非黑名单标签保留
- self.assertGreater(len(new_root.findall('.//p')), 0)
-
-
-class TestIsDisplayNone(unittest.TestCase):
- def test_display_detection(self):
- # 标准display:none检测
- element = html.fromstring('')
- self.assertTrue(is_diplay_none(element))
-
- def test_case_insensitive(self):
- # 大小写不敏感检测
- element = html.fromstring('')
- self.assertTrue(is_diplay_none(element))
-
- def test_partial_match(self):
- # 不匹配部分匹配的情况
- element = html.fromstring('')
- self.assertTrue(is_diplay_none(element))
-
-
-class TestRemoveInvisibleTags(unittest.TestCase):
- def test_invisible_removal(self):
- html_content = """
-
-
-
- 可见内容
-
- """
- root = html.fromstring(html_content)
- new_root = remove_invisible_tags(root)
- # 验证隐藏元素已移除
- self.assertEqual(len(new_root.findall('.//p')), 0)
- self.assertIsNotNone(new_root.find('.//div'))
-
-
-class TestIsBlankTag(unittest.TestCase):
- def test_img_tag(self):
- # 图片标签永远不空白
- element = html.fromstring('
')
- self.assertFalse(is_blank_tag(element))
-
- def test_plain_text(self):
- # 包含真实文本
- element = html.fromstring('真实内容')
- self.assertFalse(is_blank_tag(element))
-
- def test_nested_blank(self):
- # 多层空白结构
- element = html.fromstring('')
- self.assertTrue(is_blank_tag(element))
-
- def test_tail_text(self):
- # 包含尾部文本的情况
- element = html.fromstring('尾部文本')
- self.assertFalse(is_blank_tag(element))
-
-
-class TestRemoveBlankTagsRecursive(unittest.TestCase):
- def test_shallow_removal(self):
- # 简单空白标签移除
- html_content = '
'
- root = html.fromstring(html_content)
- new_root = remove_blank_tags_recursive(root)
- self.assertEqual(len(new_root.findall('.//p')), 0)
-
- def test_deep_removal(self):
- # 深层嵌套空白标签
- html_content = """
-
-
-
-
-
-
-
-
- """
- root = html.fromstring(html_content)
- new_root = remove_blank_tags_recursive(root)
- # 验证所有空白标签已移除
- self.assertEqual(len(new_root.findall('.//section')), 0)
- self.assertEqual(len(new_root.findall('.//div')), 0)
-
- def test_mixed_content(self):
- # 混合空白和非空白内容
- html_content = """
-
-
- 真实内容
-
-
- """
- root = html.fromstring(html_content)
- new_root = remove_blank_tags_recursive(root)
- # 验证空白标签移除,保留有效内容
- self.assertEqual(len(new_root.findall('.//p')), 0)
- self.assertIsNotNone(new_root.find('.//div'))
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/tests/llm_web_kit/model/html_lib/test_simplify.py b/tests/llm_web_kit/model/html_lib/test_simplify.py
index 148abbbb..30603c50 100644
--- a/tests/llm_web_kit/model/html_lib/test_simplify.py
+++ b/tests/llm_web_kit/model/html_lib/test_simplify.py
@@ -1,221 +1,22 @@
-import unittest
-from lxml import html
+from llm_web_kit.model.html_lib.simplify import general_simplify_html_str
-from llm_web_kit.model.html_lib.simplify import (general_simplify,
- general_simplify_html_str)
+def test_general_simplify_html_str():
+ # 测试用例1:简单的HTML字符串
+ html_input = 'Title
Paragraph
'
+ # 假设简化后不变
+ expected_output = 'Title
Paragraph
'
+ assert general_simplify_html_str(html_input) == expected_output
-class TestGeneralSimplify(unittest.TestCase):
- def test_general_simplify_with_title(self):
- # 测试带有标题的HTML页面
- html_str = """
-
-
- Test Title
-
-
-
- Main Heading
- This is a paragraph.
-
-
- - Item 1
- - Item 2
-
-
-
-
- """
- root = html.fromstring(html_str)
- title, simplified_root = general_simplify(root)
- self.assertEqual(title, 'Test Title')
- # 找到动态变更的``标签和``标签
- p_tags = simplified_root.xpath('//p')
- self.assertTrue(all(p.tag == 'p' for p in p_tags))
- # 表结构未被保留
- ul_tag = simplified_root.xpath('//ul')
- self.assertEqual(len(ul_tag), 1)
- # `None`标签结构未被保留
- # 需要进一步检查具体的删除逻辑,但在此处假设所有不可见内容都被移除
- hidden_divs = simplified_root.xpath("//div[@style='display:none;']")
- self.assertEqual(len(hidden_divs), 0)
+ # 测试用例2:空标签
+ html_input = 'Title
Paragraph
'
+ # 假设简化后删除空标签
+ expected_output = 'Title
Paragraph
'
+ assert general_simplify_html_str(html_input) == expected_output
- def test_general_simplify_no_title(self):
- # 测试没有标题的HTML页面
- html_str = """
-
-
-
- No Title Here
- This is a paragraph without a title.
-
-
-
- """
- root = html.fromstring(html_str)
- title, simplified_root = general_simplify(root)
- self.assertIsNone(title)
-
- def test_general_simplify_remove_invisible_tags(self):
- # 测试移除不可见的HTML标签
- html_str = """
-
-
-
- Visible content
-
-
-
-
- """
- root = html.fromstring(html_str)
- title, simplified_root = general_simplify(root)
- hidden_divs = simplified_root.xpath("//div[@style='display:none;']")
- self.assertEqual(len(hidden_divs), 0)
-
- def test_general_simplify_merge_list(self):
- # 测试合并列表标签
- html_str = """
-
-
-
- - Item 1
- - Item 2
-
-
- - Item A
- - Item B
-
-
-
- """
- root = html.fromstring(html_str)
- title, simplified_root = general_simplify(root)
- ul_tags = simplified_root.xpath('//ul')
- ol_tags = simplified_root.xpath('//ol')
- # 合并后的列表标签应保留
- self.assertEqual(len(ul_tags), 1)
- self.assertEqual(len(ol_tags), 1)
- # 但是li标签应该被移除
- li_tags = simplified_root.xpath('//li')
- self.assertEqual(len(li_tags), 0)
-
- def test_general_simplify_remove_blank_tags(self):
- # 测试移除空白标签
- html_str = """
-
-
-
-
-
-
- Content
-
-
-
-
- """
- root = html.fromstring(html_str)
- title, simplified_root = general_simplify(root)
- # 检查子节点是否被删除
- p_tags = simplified_root.xpath('//p')
- self.assertEqual(len(p_tags), 1)
- self.assertEqual(p_tags[0].text, 'Content')
-
- def test_general_simplify_unwrap_tags(self):
- # 测试解包单子节点标签
- html_str = """
-
-
-
- Content
-
- Text
-
-
- """
- root = html.fromstring(html_str)
- title, simplified_root = general_simplify(root)
- # 检查`div`标签是否被解包
- div_tags = simplified_root.xpath('//div')
- self.assertEqual(len(div_tags), 0)
- # 检查`p`标签是否保留
- p_tags = simplified_root.xpath('//p')
- self.assertEqual(len(p_tags), 1)
- self.assertEqual(p_tags[0].text, 'Content')
-
- def test_general_simplify_html_str(self):
- # 测试`general_simplify_html_str`函数
- html_str = """
-
-
-
- Hello World
-
-
-
-
- """
- simplified_html_str = general_simplify_html_str(html_str)
- # 确保脚本标记被移除
- self.assertNotIn('