From 1e27e94540dc0829b83bc02d31201aac351c02f8 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Fri, 11 Apr 2025 19:50:18 +0800 Subject: [PATCH 01/13] =?UTF-8?q?refactor:=20=E9=87=8D=E6=9E=84=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=E6=AE=B5=E8=90=BD=E5=85=83=E7=B4=A0=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm_web_kit/extractor/html/recognizer/text.py | 226 ++-- .../good_data/html/text11.html | 1177 +++++++++++++++++ .../good_data/html/text12.html | 117 ++ .../extractor/html/recognizer/test_para.py | 22 +- .../extractor/html/recognizer/test_text.py | 52 +- .../extractor/test_extractor_chain.py | 4 +- 6 files changed, 1505 insertions(+), 93 deletions(-) create mode 100644 tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/text11.html create mode 100644 tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/text12.html diff --git a/llm_web_kit/extractor/html/recognizer/text.py b/llm_web_kit/extractor/html/recognizer/text.py index 25e18aef..06d934c2 100644 --- a/llm_web_kit/extractor/html/recognizer/text.py +++ b/llm_web_kit/extractor/html/recognizer/text.py @@ -1,4 +1,6 @@ +import copy import json +import re import string from typing import List, Tuple @@ -40,6 +42,17 @@ PARAGRAPH_SEPARATOR = '\n\n' +# 需要保留的html实体，例如：'>' 直接在markdown中无法渲染，需要替换为html实体 +entities_map = {'>': 'gt'} + +# 行内元素 +inline_tags = { + 'a', 'abbr', 'acronym', 'b', 'bdo', 'big', 'br', 'button', 'cite', 'code', + 'dfn', 'em', 'i', 'img', 'input', 'kbd', 'label', 'map', 'object', 'q', + 'samp', 'script', 'select', 'small', 'span', 'strong', 'sub', 'sup', + 'textarea', 'time', 'var', 'u', 's', 'code', 'cccode-inline', 'ccmath-inline' +} + class TextParagraphRecognizer(BaseHTMLElementRecognizer): """解析文本段落元素.""" @@ -105,6 +118,22 @@ def __to_cctext_lst(self, lst: List[Tuple[HtmlElement | str, HtmlElement | str]] new_lst.append((cctext_el, raw_html_element)) return new_lst + def replace_entities(self, text, entities_map): + """使用正则表达式同时替换文本中的多个特定字符为其对应的HTML实体。 + + :param text: 需要处理的文本。 + :param entities_map: 一个字典，键是需要替换的字符，值是对应的HTML实体名 + :return: 替换后的文本。 + """ + # 创建正则表达式模式，匹配所有需要替换的字符 + rx = re.compile('|'.join(re.escape(str(key)) for key in entities_map.keys())) + + def one_xlat(match): + """回调函数，用于将匹配到的字符替换为对应的HTML实体。""" + return f'&{entities_map[match.group(0)]};' + + return rx.sub(one_xlat, text) + def __combine_text(self, text1:str, text2:str, lang='en') -> str: """将两段文本合并，中间加空格. @@ -117,11 +146,11 @@ def __combine_text(self, text1:str, text2:str, lang='en') -> str: text2 = text2.strip(' ') if text2 else '' if lang == 'zh': txt = text1 + text2 - return txt.strip() + return self.replace_entities(txt.strip(), entities_map) else: words_sep = '' if text2[0] in string.punctuation or text2[0] in special_symbols else ' ' txt = text1 + words_sep + text2 - return txt.strip() + return self.replace_entities(txt.strip(), entities_map) def __get_paragraph_text(self, root: HtmlElement) -> List[dict]: """ @@ -178,85 +207,118 @@ def __get_paragraph_text_recusive(el: HtmlElement, text: str) -> str: return para_text def __extract_paragraphs(self, root: HtmlElement): - """解析文本段落元素. - - Args: - root: 根元素 + """解析HtmlElement为段落元素列表. - Returns: - 解析后的文本段落元素 + :param root: 根元素 + :return: 段落元素列表 """ - path: List[HtmlElement] = [] - parser = html.HTMLParser(collect_ids=False, encoding='utf-8', remove_comments=True, remove_pis=True) - - def is_contain_readable_text(text): - return text.strip() if text else text - - def rebuild_path(): - """rebuild path with only tag & attrib.""" - for i in range(len(path)): - elem = path[i] - copied = parser.makeelement(elem.tag, elem.attrib) - if i > 0: - path[i - 1].append(copied) - path[i] = copied - - def copy_helper(elem: HtmlElement): - """deep copy w/o root's tail.""" - copied = parser.makeelement(elem.tag, elem.attrib) - copied.text = elem.text - for sub_elem in elem: - sub_copied = copy_helper(sub_elem) - sub_copied.tail = sub_elem.tail - copied.append(sub_copied) - return copied - - def has_direct_text(elem: HtmlElement): - # hr is not considered - #
return false - #   return false - if is_contain_readable_text(elem.text): - return True - for sub_elem in elem: - if is_contain_readable_text(sub_elem.tail): - return True - return False - - def has_text(elem: HtmlElement): - if has_direct_text(elem): - return True - for sub_elem in elem: - if has_text(sub_elem): - return True - return False - - def helper(elem: HtmlElement): - copied = parser.makeelement(elem.tag, elem.attrib) - copied.text = elem.text - - if path: - path[-1].append(copied) - - path.append(copied) - # elem直接有text，则直接添加返回 - if has_direct_text(elem): - rebuild_path() - path[-1].append(copy_helper(elem)) - yield path[0], path[0] - rebuild_path() - for sub_elem in elem: - if has_direct_text(sub_elem) or (sub_elem.tag == 'p' and has_text(sub_elem)): - rebuild_path() - path[-1].append(copy_helper(sub_elem)) - # yield path[0], element_to_html(path[0]) - yield path[0], path[0] - # detach the yielded tree - rebuild_path() - continue - - yield from helper(sub_elem) - - copied = path.pop() - copied.tail = elem.tail - - return helper(root) + + def is_block_element(node) -> bool: + """如果标签不在内联元素集合中，默认为块级元素。 但是，如果一个内联元素包含块级元素，则该内联元素被视为块级元素。""" + if node.tag in inline_tags: + return any(is_block_element(child) for child in node.iterchildren()) + return isinstance(node, html.HtmlElement) + + def has_block_children(node) -> bool: + return any(is_block_element(child) for child in node.iterchildren()) + + def clone_structure(path: List[html.HtmlElement]) -> Tuple[html.HtmlElement, html.HtmlElement]: + if not path: + raise ValueError('Path cannot be empty') + root = html.Element(path[0].tag, **path[0].attrib) + current = root + for node in path[1:-1]: + new_node = html.Element(node.tag, **node.attrib) + current.append(new_node) + current = new_node + last_node = html.Element(path[-1].tag, **path[-1].attrib) + current.append(last_node) + return root, last_node + + paragraphs = [] + + def process_node(node: html.HtmlElement, path: List[html.HtmlElement]): + current_path = path + [node] + inline_content = [] # 累积内联内容和未包裹文本 + + # 首先处理父节点下的直接文本内容（node.text） + if node.text and node.text.strip(): + inline_content.append(('direct_text', node.text.strip())) + + # 处理所有子节点 + for child in node: + if is_block_element(child): + # 遇到块级元素，先处理之前累积的内容 + if inline_content: + try: + root, last_node = clone_structure(current_path) + merge_inline_content(last_node, inline_content) + paragraphs.append(root) + except ValueError: + pass + inline_content = [] + + # 处理块级元素本身 + if not has_block_children(child): + # 没有子块级元素，作为独立段落 + try: + root, last_node = clone_structure(current_path + [child]) + last_node.text = child.text if child.text else None + for grandchild in child: + last_node.append(copy.deepcopy(grandchild)) + paragraphs.append(root) + except ValueError: + pass + else: + # 有子块级元素，递归处理 + process_node(child, current_path) + + # 处理当前块级元素的tail文本（与块级元素同级） + if child.tail and child.tail.strip(): + inline_content.append(('tail_text', child.tail.strip())) + else: + # 非块级元素 + inline_content.append(('element', child)) + # 处理非块级元素的tail文本 + if child.tail and child.tail.strip(): + inline_content.append(('tail_text', child.tail.strip())) + + # 处理剩余的内联内容（没有更多块级元素的情况） + if inline_content: + try: + root, last_node = clone_structure(current_path) + merge_inline_content(last_node, inline_content) + paragraphs.append(root) + except ValueError: + pass + + def merge_inline_content(parent: html.HtmlElement, content_list: List[Tuple[str, str]]): + """将inline_content列表中的内容合并到给定的parent节点中。 + + :param parent: 目标父节点 + :param content_list: 包含('direct_text'|'tail_text'|'element', content)元组的列表 + """ + last_inserted = None + for item_type, item in content_list: + if item_type in ('direct_text', 'tail_text'): + if last_inserted is None: + if not parent.text: + parent.text = item + else: + parent.text += ' ' + item + else: + if last_inserted.tail is None: + last_inserted.tail = item + else: + last_inserted.tail += ' ' + item + else: # element + parent.append(copy.deepcopy(item)) + last_inserted = item + + process_node(root, []) + + unique_paragraphs = [] + for p in paragraphs: + unique_paragraphs.append((p, p)) + + return unique_paragraphs diff --git a/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/text11.html b/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/text11.html new file mode 100644 index 00000000..b555d9f5 --- /dev/null +++ b/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/text11.html @@ -0,0 +1,1177 @@ + + + + + + + + + + + + + + + + + + + + +Help talk:Theme designer - Wikia Community Central + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + +

Wikia

+
+ + + +
+ +
+ +
+
+
+ +
    + + + +
  • + Log in +
    • +
  • + Sign up
    • + + +
+
+
+
+
+
+ + +
+ + +
+ + +
+
+
+
+

+ + Community Central + +

+
+ +
+ + + + + + + + + + + + + + + +
+
    +
    + +
    + +
    + +
    + Watchlist Random page Recent changes
    +
    + + + + + + +
    +
    +
    +

    Talk:Theme designer

    + + +
    + 86,821pages on
    this wiki
    +

    Back to page

    +
    + + + +
    + +
    +
    + + +
    + + + + +
    +

    Got a question about this topic?
    Head on over to the Forum! +

    +
    +
    • You can use this space to suggest improvements to the help page. +
    +    		 + +
    + +


    +

    +

    Early, uncategorized threads Edit

    +
    I can not do that! My problem is when the My Tools, there is nothing "Special: Themedesigner. For the love of God: HEEELLP MEEE!!!--User:LigadajustiçaSmallville 22:01, October 11, 2010 (UTC) +
    +
    Are you looking on a wiki that you are an admin on? You need to be an admin to use this tool. If you want to test it out, you can on the theme designer wiki. If not, please give me more details. --Sarah@Wikia (Help Forum) (blog) 02:40, October 12, 2010 (UTC) +
    +
    +

    The theme designer is a new tool for the "new wiki look". You can change to it by going to your preferences and chosing the new theme instead of monaco. I have figured that much out. I'm having a major problem with this new tool. The three top buttons: theme, custom, and wordmark are all unresponsive to a click. Also, the wiki preview picture is only a tiny box with scrollbars not a full-page view that would be expected. Is this an error in the system that is being fixed? Or am I doing it wrong? Mortemer2112 01:10, October 12, 2010 (UTC) +

    +
    What browser are you on? Can you send more details and a screenshot to Special:contact and we will work to get this fixed right away. Best, --Sarah@Wikia (Help Forum) (blog) 02:40, October 12, 2010 (UTC) +
    +

    I'm using Internet explorer, I've sent a screenshot in I hope it helps. Mortemer2112 19:31, October 14, 2010 (UTC) +

    +
    Hi, thanks for helping me! I'm on Google Chrome. Yes I am admin of a wiki. I'm trying to change the background and the designer of the Wiki. Since then I am very confused. We see the "My Tools" on the page: Wikia.com, right? I see, but had no "History". I tried to edit the My Tools, but found no Special: Themedesigner. We see the "My Tools" on the page: http://www.wikia.com/Wikia, right? I see, but had no "History". I tried to edit the My Tools, but found no Special: Themedesigner. Understood, I wish you, Sarah, explain to me how blessed I would find the "Special: Themedesigner".--LigadajustiçaSmallville 14:35, October 12, 2010 (UTC) +
    +
    What wiki are you having a problem on?--Sarah@Wikia (Help Forum) (blog) 17:09, October 12, 2010 (UTC) +
    +
    +
    The Wiki is in Portuguese of Brazil. The link is this: T.I.M.E Wiki! Thank you Sarah! --LigadajustiçaSmallville 21:49, October 12, 2010 (UTC) +
    +
    Hi, I just checked your account on chrome, and it appeared that you did not have the new look selected in your preferences. Once I chose this option, it gave your account access to theme designer. Let me know if this continues not to work. Best, --Sarah@Wikia (Help Forum) (blog) 23:20, October 12, 2010 (UTC) +
    +
    +
    Pardon my delay in responding, was busy. Hi, I prefer the old view of the Wiki. But if the problem is this, I change. But I have no access to Special: Themedesigner. That's my objective was to create a new theme for my Wiki. A blue tone. I wanted to access what is appearing in the picture.--LigadajustiçaSmallville 23:23, October 13, 2010 (UTC) +
    +
    Theme designer
    +

    Hello LigadajustiçaSmallville, You still did not change your settings, so I did it for you. You can now access the theme designer here. Best, --Sarah@Wikia (Help Forum) (blog) 16:26, October 14, 2010 (UTC) +

    +
    Hi Sarah, thanks for helping me in the Theme designer! I was the editing and arrived at the "wordmark" and not know the size and scale of the graphic image but the image is ready, lack fit the size available to post on the graphic.--LigadajustiçaSmallville 22:42, October 20, 2010 (UTC) +
    +
    The wordmark must be an png file and 250 x 65 px. You can use a tool like http://pixlr.com/editor/ to adjust it. Cheers, --Sarah@Wikia (Help Forum) (blog) 23:56, October 20, 2010 (UTC) +
    +
    +
    Oh Sarah. I turned the 250 px image, but could not put in the wordmark. You can change the picture for me?
    Ooopleaa
    And how do I put for unregistered users when they come to my Wiki page, see the new look of Wikia.--LigadajustiçaSmallville 16:15, October 26, 2010 (UTC) +
    +
    Hello, I resized and added the wordmark for you, so enjoy! Now all users will see the new look. Happy editing, --Sarah@Wikia (Help Forum) (blog) 18:39, October 26, 2010 (UTC) +
    +
    +

    Help Edit

    +

    Hi, I'm an administrator at Logopedia (Logos Wikia), and I'm wondering about how to change this new wiki look back to the regular Monaco Sapphire skin. I want this for every user, not just for me. Please tell me how to change it back. Thanks! --Alxeedo TALK 20:04, October 18, 2010 (UTC) +

    +
    Hi Alxeedo, you can change your personal preferences to show Monaco until Nov 3rd for just you. It is not possible to change it back for other users. On Nov 3rd, Monaco will be removed as an option. The "Oasis" theme in the theme designer is similar to sapphire, so you may want to experiment with using that. Best, --Sarah@Wikia (Help Forum) (blog) 23:15, October 18, 2010 (UTC) +
    +

    Dimensions Edit

    +

    What dimensions would work best for the custom background graphic (not the wordmark)? --れび (talk to Lavi!) 01:21, October 28, 2010 (UTC) +

    +
    Hi. It depends a bit on what you want for a background. If it is a pattern that can be tiled, then you can use any size image that you like. If you would like a single image, the vertical height would depend on how far down the page you want it to extend. In terms of width, a single image needs to be wider than 960px, which is the width of the content and sidebar area, or it won't be seen around them. How much wider depends again on what you want to put there (and what you can fit within the filesize limit of 100 kilobytes). As an example, something in the realm of 1400x600 would be reasonable -- it would extend at least halfway down most users' screens; just choose a background color that will look good below it and to the sides on bigger screens. -- Wendy (talk)@Wikia 01:01, October 29, 2010 (UTC) +
    +

    Invisible Wordmark Edit

    +

    Hi I am having a problem with my wordmark. I have repeatedly tried using a 250x65 .PNG image and this is what happens. It appears fine in the theme designer so I click save. But then it doesn't show up after I save. I re-open theme designer and the image preview of it (besides the box that says: dont use a graphic) is only a small box with a red X, hinting that the image file is invalid even though it shows up fine when first uploaded. +

    +
    I'm having a similar problem. I have uploaded a background image and a wordmark and it is fine in the designer but when saved doesn't show. I even deleted the old File:Wiki-wordmark.png but it still displays, what is with that? It's a bit worrying that the new skin is supposed to go live in the next couple of days but I can't get a theme working. --Squilibob 22:54, October 29, 2010 (UTC) (sealonline.wikia.com) +
    Hi, I see a wordmark on sealonline.wikia.com, is that not the right one? Or has it showed up now? --Sarah@Wikia (Help Forum) (blog) 18:47, November 1, 2010 (UTC) +
    The bug seems to have been fixed, thanks. --Squilibob 23:41, November 1, 2010 (UTC) +
    +
    +

    You can not do that! Edit

    +

    It is now that we can not choose and change the style sheet for Monaco Custom? There must be freedom of choice! --DCamer 21:16, October 30, 2010 (UTC) +

    I tried to put any graphic background and wordmark and nothing. NOTHING!!! What's up? Danielvalle2222 04:20, February 13, 2012 (UTC) +

    +
    Please send in a link to the wiki, your browser info and any screenshots via Special:Contact. Also, in the future, best to start a new section at the bottom when posting these types of questions there. Sarah@Wikia (help forum | blog) 18:53, February 13, 2012 (UTC) +
    +

    Advanced CSS Edit

    +

    Hello, the main wiki I edit on, the Calvin and Hobbes Wiki, has advanced CSS coding in the Common.css file for use on the main page and other pages. I was wondering if I could somehow transfer that code from the Common.css file to another file so that it will also work in the new wikia theme. Thank you, Brovie talk 06:31, November 6, 2010 (UTC) +

    +
    Yes you can add it to Wikia.css --Sarah@Wikia (Help Forum) (blog) 00:48, November 9, 2010 (UTC) +
    Thank you! --Brovie talk 04:53, November 9, 2010 (UTC) +
    +
    +

    Multiple Themes Edit

    +

    Maybe this is a stupid question, but, as a new Administrator on the Mortal Kombat Wiki, I'm curious if there's a way to make multiple themes. That is to say, allow registered users to choose which theme they'd like to see the wikia displayed as. Is there a way? —CavalierTunestalk 17:40, December 24, 2010 (UTC) +

    +
    Users can use their personal preference to choose between the admin theme or monobook, but there is not a way to set a different theme for different types of users. --Sarah@Wikia (Help Forum) (blog) 18:59, December 27, 2010 (UTC) +
    Thank you, but I think I asked my question incorrectly. I was trying to ask if there was a way to create multiple admin themes that users can choose from under their personal preferences. For instance, if a random user went to his or her personal preferences, could they choose between "Monobook," "Admin Theme 1," "Admin Theme 2," etc. Or can there be only one admin theme? I'm not asking if I can create different themes for different types of users, just if I can create multiple themes for users to choose from under their personal preferences. I hope that was less ambiguous. —CavalierTunestalk 17:56, December 31, 2010 (UTC) +
    +
    +

    TablesEdit

    +

    When I changed the customizable page color (for the background), it messed up the wikitables, making the borders a different color and making the backgroungs match. While the matching/transparent background is fine, is there a way to make the borders stay the default black? +Neocarleen • Talk 09:12, January 17, 2011 (UTC) +

    +

    Background image does not load properly Edit

    +

    Hello, I'm experiencing some problems with my wiki's background image. For some reason, whenever I update the background image, the old one still presents as the current image. I have tried to switch back and forth between old and new images but nothing changed. The small preview images in the file's history indicated that there was nothing wrong with the upload process or the file (at least that is what I think), so I guess there must be some problems with displaying system. Is there anyway to overcome this? Venom00 05:52, February 16, 2011 (UTC) +

    - Hi, I am not a offical STAFF but I can help you -βᵒᵇ βʳᶦᶜᵏˢ™ Ʈᵃᶫᵏ · βᶫᵒᵍ +

    +
      • You need to use MediaWiki:Wikia.css, put this coding down but switch BACKGROUND with the background image link +
      +
    +

    { +

    +
     body.mediawiki  {
+ background:#0F5835 url(BACKGROUND) fixed center top repeat !important;
+ background-repeat-x: repeat !important;
+ background-repeat-y: repeat !important;
+ background-color: #0F5835!important
+
    +

    { +

    +

    Recent changes to Theme Designer Edit

    +

    Do the recent changes to the Theme Designer related to the  WikiaPage  and  WikiaPageBackground  elements require any changes to the instructions/explanations provided here? I am specifically thinking of notes on how to use the new transparency bar under the page background settings. Thanks! — SpikeToronto 05:40, July 30, 2011 (UTC) +

    +
    I have a problem with the transparency bar. I tried to adjust it to make the foreground transparent but after I clicked "I'm done", nothing has changed and when I went back to the Theme designer, it bar was where it was before I changed it. EpsilonIndi 16:20, July 30, 2011 (UTC) +
    +
    Epsilon: You should post your question at the Community Central forum. You should do this for several reasons:
    1. These Help Wiki talk pages have very little traffic, so no one with an answer is likely to come by here for some time (weeks or months even)
    2. These Help Wiki talk pages are for discussing changes/improvements to the Help Wiki articles.
    3. The Community Central forums are the central location for solving problems
    4. The Community Central forums have very high traffic of people who have the expertise to answer your question within hours
    Thanks! — SpikeToronto 20:08, July 30, 2011 (UTC) +
    +
    +

    How to change Wordmark?Edit

    +

    Hi, I am the founder of Isle of Tune Wiki and the wiki is already created. How can I change the Wordmark?
    Sam Wang 05:03, October 11, 2011 (UTC) +

    +
    You can use the wordmark section of the theme designer. If you need further help, try and post in the founder & admin forum. Cheers, --Sarah@Wikia (help forum | blog) 16:08, October 21, 2011 (UTC) +
    +

    I have a question: I have a wiki name logo for Charlie Bone Wiki, [1] and there are no local admins on that wiki. So what can I do? --Lao 21:11, October 20, 2011 (UTC) +

    +
    You can apply to adopt it - which will give you admin rights. You can apply here. Best, --Sarah@Wikia (help forum | blog) 16:08, October 21, 2011 (UTC) +
    +

    Help Edit

    +

    On my Wiki, eveytime time I try to upload a favicon or wordmark it keeps saying "File extension does not match MIME type," and it's becoming very frustrating. Can someone please help? +

    --A hero boy named Finn +

    +
    Are you sure you are using the right file types? For the workmark it has to be a png file and for the favicon it has to be ico. If not, they won't work. Sarah@Wikia (help forum | blog) 19:18, December 24, 2011 (UTC) +
    +

    Help :cEdit

    +

    How I can put images on the header? +

    Archivo:Th_mlp_rainbow_dash_sprite_2_by_blaze33193-d3lmwke.png Rainbow Dash!~ 03:41, January 1, 2012 (UTC) +

    +

    Theme Choosing. Edit

    +

    Recently on DragonVale Wiki was a theme change, which, personally, I hate. Now, I was looking into this regarless, but I would love to know: is there a tool to let people choose their own themes from some set ones? I think the desired parchment effect is just no. With the love this theme is getting I don't think it will be changed, but I would love to change to another theme, hate to stick with this one, and giving other users and anons the choice would make everyone happy. To clarify, a tool to let everyone decide from set themes for their personal browsing. The new one I find literally unbearable. I can't look at it without saying, "ew!". Really, I am just considering leaving since I'm not very active anyway and I hate the design, downing my motivation to visit. If this tool does not exist, please, create it. It would be a great item to add everywhere, because it would cut down on any possible skin arguments and let everybody set the look they want. Also, is it in production at all, if not already available? ~Nibe - DragonVale Bureaucrat - Talk | Blog 06:34, February 3, 2012 (UTC) +

    +
    Hello, There isn't an easy way to create your own personal view of the wiki. You can modify your personal css and js file to adjust your look, but this can be a bit complicated. I would recommend posting on our support forum to see if other users are able to share with you the modifications they have made. Best of luck, Sarah@Wikia (help forum | blog) 17:17, February 3, 2012 (UTC) +
    +

    Upload wont work in theme desginerEdit

    +

    Hi, im currently working on two wikis as an admin. I'm having a lot of trouble with uploading my own images for backgorunds, wordmarks etc. +

    All my grapghics fit the size and file type criteria, but when i click upload, nothing happens. +

    The first wiki, baldursgategame.wiki.com and the second is jackass.wiki.com +

    These are the two graphic wordmarks i'm trying to upload.... +

    +
    Bg wordmark
    Wordmark2
    +

    I'm also having the same problem with my jackass facIcon. +

    Thanks in advance... Thomaslove92 22:39, March 1, 2012 (UTC) +

    +
    • EDIT: I've fixed this..i was using google chrome. Works fine in Firefox. This maybe should be addressed however. Thomaslove92 01:53, March 2, 2012 (UTC)
      +
    +

    Thanks for letting us know and happy to hear its fixed. What wikis exactly was it happening on? Sarah@Wikia (help forum | blog) 17:02, March 2, 2012 (UTC) +

    +
    • It was on baldursgategame.wiki.com and jackass.wiki.com - it appears just to be an issue with google chrome. Thomaslove92 22:18, March 3, 2012 (UTC) +
    +

    Background not uploading?Edit

    +

    I believe I have experienced this problem before, but I can't get around it this time. While helping a friend at w:c:thelegospace with the theme, I am unable to make the background image show. Instead of this, it keeps the old background. I tried deleting the wiki-background file and whatnot, but it still doesn't show the new one. Any .css I could add to make the background show? If so, I'd appreciate it. Thanks, - Bug +

    +
    This is happening with the wordmark on my wiki. The thing Talk 22:53, April 1, 2012 (UTC) +
    +

    - Hi, I am not a offical STAFF but I can help you -βᵒᵇ βʳᶦᶜᵏˢ™ Ʈᵃᶫᵏ · βᶫᵒᵍ +

    +
      • You need to use MediaWiki:Wikia.css, put this coding down but switch BACKGROUND with the background image link +
      +
    +

    { +

    +
     body.mediawiki  {
+ background:#0F5835 url(BACKGROUND) fixed center top repeat !important;
+ background-repeat-x: repeat !important;
+ background-repeat-y: repeat !important;
+ background-color: #0F5835!important
+
    +

    { +

    +

    So, Wait...Edit

    +

    If I change the theme, does it only change it for MY account, or does everyone else see it too? Like, If I make everything purple, will everyone else see purple, or only me? +

    {{subst:User:JessiPhan/Sig}} 02:14, April 12, 2012 (UTC) JessiPhan +

    + +

    Help Saving ThemeEdit

    +

    I'm an admin on 28 Days Later Wiki, and I'm trying to change the background image, but whenever I save the change, it just keeps the original background image! Could someone please help me with this. TroopDude 12:40, May 12, 2012 (UTC) +

    - Hi, I am not a offical STAFF but I can help you -βᵒᵇ βʳᶦᶜᵏˢ™ Ʈᵃᶫᵏ · βᶫᵒᵍ +

    +
      • You need to use MediaWiki:Wikia.css, put this coding down but switch BACKGROUND with the background image link +
      +
    +

    { +

    +
     body.mediawiki  {
+ background:#0F5835 url(BACKGROUND) fixed center top repeat !important;
+ background-repeat-x: repeat !important;
+ background-repeat-y: repeat !important;
+ background-color: #0F5835!important
+
    +

    { +

    +

    Why does it keep cutting my background image? Edit

    +

    I have my picture, and it's at 1600px wide. It's 1100 tall and within the file size limits. I want it to be fixed. +

    +When I upload the image, it cuts off the outer edges. Why is this?
    WikiBackground3
    What it SHOULD look like
    Screen Shot 2012-06-19 at 6.25.45 PM
    What it ACTUALLY looks like
    +

    It's like Wikia "zooms in" on it. I just want the stuff on the edges to be showing, but it simply does work. What is causing this, and how can I fix it? +

    Thanks. +

    Bienvenue dans notre univers! Retrouve-nous au coeur de ta memoire. 22:29, June 19, 2012 (UTC) +

    - Hi, I am not a offical STAFF but I can help you, you can try using MediaWiki.css, you need to post this coding down in the wiki +

    { +

    +
      body.mediawiki  {
+  background:#0F5835 url(http://images3.wikia.nocookie.net/__cb20120619222302/tsclg/images/5/50/Wiki-background) fixed center top repeat !important;
+  background-repeat-x: repeat !important;
+  background-repeat-y: repeat !important;
+  background-color: #0F5835!important;
+
    +

    { +

    -βᵒᵇ βʳᶦᶜᵏˢ™ Ʈᵃᶫᵏ · βᶫᵒᵍ +

    +

    Favicon Edit

    +

    I've uploaded my favicon about 4 times. It's 16 x 16, it's a .ico file, it uploads fine, but when I save my changes, it doesn't change from the default. I've tried adding ?action=purge, too. No difference. --GwenhwyfarRaven (talk) 07:17, September 23, 2012 (UTC) +

    +

    Multi-Theme Edit

    +

    Is there a possible way to set up multiple themes for a wiki? Not where a user can select their own theme, but where a few pages have their own unique theme. Submachiner (talk) 21:27, October 2, 2012 (UTC)\ +

    +

    Editor problemEdit

    +

    Hi. +I have a big problem on my wiki, Warriorz Fanfiction Wiki. +

    I've gone to the theme designer page, but when I click one of the tabs (Theme, Customize and Wordmark) on the top left, the board on the right where all the editing options for the wiki's background and whatnot don't show up. The board stays blank. +

    PLEASE help me!!!! Kittycat79Melodyfrost: Mate of Hawkfrost 09:16, December 20, 2012 (UTC) +

    +
    Hey there, Can you send in all the details (link to wiki, your browswer info and a screenshot) to Special:Contact? This wil help us in figuring out the issue. Thanks, Sarah@Wikia (help forum | blog) 17:29, December 20, 2012 (UTC) +
    +
    K, I did that. Thanks. Kittycat79Melodyfrost: Mate of Hawkfrost 02:27, December 21, 2012 (UTC) +
    +

    Background problemEdit

    +

    So the basic problem is that the background only shows up on some of my pages. It mostly shows up on all, I think, but not my user page and not my main page. Now this might just be a bug, but it's highly annoying. The wiki in discussion is http://tiffanyalvord.wikia.com ~Xo MinniHowl (talk)  18:37, January 8, 2013 (UTC)  +

    +
    I see it on all pages. What shows when you don't see it? What browser and version are you on? Sarah@Wikia (help forum | blog) 18:55, January 8, 2013 (UTC) +
    +

    everytime i try to upload a new background picture it wont let me. i have the right file type and it's not over 150kb and it shows i have it selected but it doesn't do anything when i click upload. pleez help. ReichMuskrat15 (talk) 22:22, February 23, 2013 (UTC) +

    +
    Responded on your message wall. Sarah@Wikia (help forum | blog) 18:54, February 25, 2013 (UTC) +
    +

    Specific wordmark for specific pages Edit

    +

    Why the admin central of this wiki had a different wordmark? Can you make a specific wordmark dor specific pages?DarkTimes2013 (talk) 05:44, July 3, 2013 (UTC) +

    +

    Background issuesEdit

    +

    I am having trouble with the background on my wiki . The background is stuck on the previous imageupoaded, and won't change to the new one. Despite this, when I open up the theme designer it claims I am using the new image for my background. +

    Nototter (talk) 00:58, September 8, 2013 (UTC) +

    +
    I see it as tiled and appears to be the latest version - has it updated for you now? Sarah@Wikia (help forum | blog) 16:35, September 9, 2013 (UTC) +
    +

    What if..Edit

    +

    What if the founder of the wikia is no longer contributing and there are no admins? Some of us really want to change the BG and the title of It Girl Wikia , but we can't. The "administrator" role can be gived to another contributor(s)? +

    MaryJones (talk) 16:13, October 15, 2013 (UTC)MaryJones +

    You can fill in the adoption form on this wiki to get the rigths, so you can do these things. 2Actimv  talk  + + 17:39, October 16, 2013 (UTC) +

    +

    Dragon Ball Fanon problemsEdit

    +

    Some problems: +

    +
    1. The original theme just disappeared on us about halfway through the day of 11/7/2013. +
    2. Attempts to revert the theme to an earlier version did not bring up any theme. Only a dark purple background remained if any attempt was made. +
    3. After uploading a new theme (which was not the correct size), I am unable to upload any more. If I upload a new custom-made theme, it shows the original, incorrectly-sized one. Even if I upload a different picture, it will still show that first image. So I'm just completely unable to use this tool. Is there any possible way to get around this glitch-ridden system and get a theme for the wiki at all? -KidVegeta (talk) +
    +

    Ice Age WikiEdit

    +

    I've been the current admin of the Ice Age Wiki for some time. One thing I've wanted changed on there is the current background. I have found an image from the Ice Age films online that I cleaned up a little so as to make it blank and devoid of characters and promptly uploaded it. However, once I attempted to make that image in particular the Wiki's background, it got cornered to the upper left side of the screen; that is, it technically is the background, but it has shrunken horribly and doesn't cover even a slight bit of the area I intended. I've tried to make the image smaller, I've tried multiple uploads of the same thing over again, but none of it works. This said, might someone be able to lend a hand in correcting the thing once and for all?--Macrauchenia (talk) 02:58, November 15, 2013 (UTC) +

    + + + +
    +Retrieved from "http://community.wikia.com/wiki/Help_talk:Theme_designer?oldid=1142554"
    + +
    + + + +
    +
    + +
    +
    +
    +
    + +
    + + +
    + + + + + + + + + + + + + + + +
    +
      +
      + +
      + + +
      + +
      +
      +

      Around Wikia's network

      + Random Wiki +
      + + +
      +
      + +
      +
      + + +
      + +
      + + + + + + + + + + + + + + + + + + + + + + + + + +
      + + + + + diff --git a/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/text12.html b/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/text12.html new file mode 100644 index 00000000..c4977417 --- /dev/null +++ b/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/text12.html @@ -0,0 +1,117 @@ + + + + Karate notes + + + + + + + + + + + + + + + + +
      home + | articles | links | fun | about
      Up to: Quick math and science observations
      + +

      + Karate notes + +

      +

      Tue 05/29/01: breaking guard, mount, leg lock, oompa

      +For maintaining mount, swim arms when they try to push up +Front naked choke-push forearm into their neck (either side) and push them away. + +

      + When mounted: if their arm behind your head, pin their forearm to ground using back of your head. Then hook leg and + oompa. + Hip out-try and "crab out" like John did. More of a butt movement, keep legs in same position + +

      +

      + Break Guard: Get back. Turn a little sideways, hook your arm under and get their left leg off. Push their leg up to + their head, keep your hands in a frame as you turn. Go around, get into side mount. + +

      +

      + Also leglock: once you have broken guard, keep their leg in your frame. Lean backwards, roll onto back. Hook their + ankle under your armpit and in your arm. Twist your arm, break the ankle. + +

      +

      + Break Guard 2 (John's way): You're in guard. Get up, into deep horse stance with their legs on your hips/thighs. + Push their legs out with your elbows. Can climb up into mount from there. + +

      +

      + Alex's armbar: after they have fallen, you are in horse stance, they are on ground. Step directly over them, pull up + on their arm. Reverse armlock (they are on their stomach, you are on one side, pulling up the arm). + +

      +

      + Regular armbar: Remember to keep knees tight! Pull their hand to chest, turn their thumb to the sky. Pull their arm + down to the pinky side, not straight down on the nuts. + +

      +

      Thurs 05/31/01: San Chin, some grappling and throws

      +San Chin: don't rock back then forward. Load up on the leg, bend it slightly, then do thrust completely outward. Leaning +back then forward takes too much time. + +

      + Throws: 3 steps (don't do them at the same time, just do them smoothly). 1: Step in, pull their hand/elbow to your + chest. 2: Forearm strike to chest/neck, knock them off balance. 3: Step past them (hips go past them!) with your + closest leg, then reap. Keep their arm tight as they fall, then you can do the armlock. + +

      +

      + Steve's Aikido throw: They punch/stab at you w/right hand. Pull them in with your right, step back and to the side. + Also put your left hand behind their head and pull them along. As they begin to turn, put your right forearm on + their neck and crank them backwards toward the ground (like slamming the basketball on the ground). + +

      +

      + Breakfalling: to practice, step forward with right leg. Fall down, slapping right hand. Legs stay on ground as you + fall, don't roll back. + +

      +

      Tuesday June 5

      +Throw (on John): Be strong: push them with your left arm, pull in their left arm. Do leg sweep, be strong on down part. + +

      + Steve's choke: clasp hands together... twist hands. Monkey clap. Effective, good when have back. For rear choke, + grab own head, gets a tighter choke. + +

      + +
      + + + + + + + + + + + + +
      Kalid at kazad@princeton.edu
      + + + + + \ No newline at end of file diff --git a/tests/llm_web_kit/extractor/html/recognizer/test_para.py b/tests/llm_web_kit/extractor/html/recognizer/test_para.py index adb38c59..58bdbd20 100644 --- a/tests/llm_web_kit/extractor/html/recognizer/test_para.py +++ b/tests/llm_web_kit/extractor/html/recognizer/test_para.py @@ -23,7 +23,7 @@ def test_recognize_simple_para(self): result = self.recognizer.recognize('', [(html_to_element(html), html_to_element(html))], html) # 验证结果 - self.assertEqual(len(result), 2) # 应该识别出2个段落 + self.assertEqual(len(result), 4) # 应该识别出4个段落 # 验证第一个段落 first_para = result[0][0] @@ -40,7 +40,19 @@ def test_recognize_simple_para(self): self.assertEqual(jso[0]['t'], 'text') self.assertEqual(jso[1]['c'], 'E=MC^2') self.assertEqual(jso[1]['t'], 'equation-inline') - self.assertTrue(jso[2]['c'].endswith('请访问')) - self.assertEqual(jso[3]['c'], 'https://abc.com') - self.assertEqual(jso[3]['t'], 'code-inline') - self.assertEqual(jso[4]['c'], '.') + + # 验证第三个段落 + second_para = result[2][0] + text = second_para.text + jso = json.loads(text) + self.assertEqual(jso[0]['c'], '其中E是能量。') + self.assertEqual(jso[0]['t'], 'text') + + # 验证第四个段落 + second_para = result[3][0] + text = second_para.text + jso = json.loads(text) + self.assertTrue(jso[0]['c'].endswith('请访问')) + self.assertEqual(jso[1]['c'], 'https://abc.com') + self.assertEqual(jso[1]['t'], 'code-inline') + self.assertEqual(jso[2]['c'], '.') diff --git a/tests/llm_web_kit/extractor/html/recognizer/test_text.py b/tests/llm_web_kit/extractor/html/recognizer/test_text.py index d887c0b0..a7e48823 100644 --- a/tests/llm_web_kit/extractor/html/recognizer/test_text.py +++ b/tests/llm_web_kit/extractor/html/recognizer/test_text.py @@ -30,7 +30,7 @@ def test_text_1(self): '中共中央政治局召开会议审议《成-2020年10月16日新闻联播', 'zh') == '知识乱象\n中共中央政治局召开会议审议《成-2020年10月16日新闻联播' result = self.text_recognize.recognize('http://www.baidu.com', [(html_to_element(html_content), html_to_element(html_content))], html_content) - assert '知识乱象\\n\\n 中共中央政治局' in element_to_html(result[908][0]) + assert '知识乱象\\n\\n 中共中央政治局' in element_to_html(result[587][0]) def test_text_2(self): """ @@ -152,7 +152,7 @@ def test_text_7(self): with open(Path(__file__).parent.parent.parent / 'assets/extractor_chain_input/good_data/html/text7.html', 'r') as file: html_content = file.read() result = self.text_recognize.recognize('http://www.baidu.com', [(html_to_element(html_content), html_to_element(html_content))], html_content) - assert '1) A man takes 5 hrs and 45 mins to walk to a certain place and ride back' in element_to_html(result[51][0]) and BaseHTMLElementRecognizer.is_cc_html(result[51][0]) + assert '1) A man takes 5 hrs and 45 mins to walk to a certain place and ride back' in element_to_html(result[66][0]) and BaseHTMLElementRecognizer.is_cc_html(result[66][0]) def test_text_8(self): """ @@ -164,7 +164,7 @@ def test_text_8(self): with open(Path(__file__).parent.parent.parent / 'assets/extractor_chain_input/good_data/html/text8.html', 'r') as file: html_content = file.read() result = self.text_recognize.recognize('http://www.baidu.com', [(html_to_element(html_content), html_to_element(html_content))], html_content) - assert "40xy\' -ln(x^8) = 0\\n\\n\\n\\n Initial Condition: y(1)=31\\n\\n\\n\\n Work:" in element_to_html(result[54][0]) and BaseHTMLElementRecognizer.is_cc_html(result[54][0]) + assert "40xy\' -ln(x^8) = 0\\n\\n\\n\\n Initial Condition: y(1)=31\\n\\n\\n\\n Work:" in element_to_html(result[69][0]) and BaseHTMLElementRecognizer.is_cc_html(result[69][0]) def test_text_9(self): """ @@ -176,7 +176,7 @@ def test_text_9(self): with open(Path(__file__).parent.parent.parent / 'assets/extractor_chain_input/good_data/html/text9.html', 'r') as file: html_content = file.read() result = self.text_recognize.recognize('http://www.baidu.com', [(html_to_element(html_content), html_to_element(html_content))], html_content) - assert '1) Consider the formula f(x)=lim(n-->infinity)((x^n)/(1+x^n)).\\n\\n Let D={x:f(x) is an element of R}. Calculate f(x) for all x elements of D and determine where f: D-->R is continuous.\\n\\n\\n\\n 2) Let f: D-->R and suppose that f(x) greater than equal 0 for all x elements of D. Define sqrt(f)-->R by (sqrt(f))(x) = sqrt(f(x)). If f is continuous at c elements of D, prove that sqrt(f) is continuous at c.' in element_to_html(result[50][0]) and BaseHTMLElementRecognizer.is_cc_html(result[50][0]) + assert '1) Consider the formula f(x)=lim(n--&gt;infinity)((x^n)/(1+x^n)).\\n\\n Let D={x:f(x) is an element of R}. Calculate f(x) for all x elements of D and determine where f: D--&gt;R is continuous.\\n\\n\\n\\n 2) Let f: D--&gt;R and suppose' in element_to_html(result[63][0]) and BaseHTMLElementRecognizer.is_cc_html(result[63][0]) def test_text_10(self): """ @@ -199,3 +199,47 @@ def test_text_10(self): result = chain.extract(input_data) content_md = result.get_content_list().to_mm_md() assert 'So far I have 2 sets of questions (but I\'m onlin in the 2nd chapter now\n\n![:smile:](d80757e36ca9835f7237339959a1fa1d929bb5c5297acb457475459d6da12278 "Smile :smile:")\n\n)\n\n 1)\n\n In the book' in content_md + + def test_text_11(self): + """ + 测试11 s3://web-parse-huawei/CC/pre-dedup/v008/unique_html/CC-MAIN-2013-48/part-67a0f2d36291-000157.jsonl.gz?bytes=47048276,27451 + Returns: + + """ + chain = ExtractSimpleFactory.create(self.config) + self.assertIsNotNone(chain) + test_data = { + 'track_id': 'text_md', + 'dataset_name': 'text_md', + 'url': 'http://community.wikia.com/wiki/Help_talk:Theme_designer', + 'data_source_category': 'HTML', + 'path': 'text11.html', + 'file_bytes': 1000, + 'meta_info': {'input_datetime': '2020-01-01 00:00:00'} + } + input_data = DataJson(test_data) + result = chain.extract(input_data) + content_md = result.get_content_list().to_mm_md() + assert '86,821 pages on' in content_md + + def test_text_12(self): + """ + 测试11 s3://llm-pdf-text-1/qa/quyuan/CC3.0/v009/data/part-67e685f42e4c-000000.jsonl?bytes=101642,27293 + Returns: + + """ + chain = ExtractSimpleFactory.create(self.config) + self.assertIsNotNone(chain) + test_data = { + 'track_id': 'text_md', + 'dataset_name': 'text_md', + 'url': 'https://betterexplained.com/~kazad/resources/shorts/karate_old.html', + 'data_source_category': 'HTML', + 'path': 'text12.html', + 'file_bytes': 1000, + 'meta_info': {'input_datetime': '2020-01-01 00:00:00'} + } + input_data = DataJson(test_data) + result = chain.extract(input_data) + content_md = result.get_content_list().to_mm_md() + assert 'For maintaining mount' in content_md diff --git a/tests/llm_web_kit/extractor/test_extractor_chain.py b/tests/llm_web_kit/extractor/test_extractor_chain.py index 78b08fe7..ec098d09 100644 --- a/tests/llm_web_kit/extractor/test_extractor_chain.py +++ b/tests/llm_web_kit/extractor/test_extractor_chain.py @@ -450,7 +450,7 @@ def test_para_is_short(self): input_data = DataJson(test_data) result = chain.extract(input_data) content_txt = result.get_content_list().to_nlp_md() - assert len(content_txt) == 2022 + assert len(content_txt) == 1982 def test_xml_tag(self): """测试xml标签.""" @@ -572,7 +572,7 @@ def test_table_colspan_str_error(self): test_data = self.data_json[34] input_data = DataJson(test_data) result = chain.extract(input_data) - result_flag = result.get_content_list()._get_data()[0][37]['content']['is_complex'] + result_flag = result.get_content_list()._get_data()[0][28]['content']['is_complex'] assert result_flag is False def test_table_invalid_percent(self): From 8f9011828752d6ce77788ac6a29e49bc6a3ab837 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Mon, 14 Apr 2025 15:03:13 +0800 Subject: [PATCH 02/13] =?UTF-8?q?feat:=20element=5Fto=5Fhtml=E6=8D=A2?= =?UTF-8?q?=E6=88=90element=5Fto=5Fhtml=5Funescaped?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm_web_kit/extractor/html/recognizer/text.py | 6 +++--- .../extractor/html/recognizer/test_text.py | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/llm_web_kit/extractor/html/recognizer/text.py b/llm_web_kit/extractor/html/recognizer/text.py index 06d934c2..af01c79b 100644 --- a/llm_web_kit/extractor/html/recognizer/text.py +++ b/llm_web_kit/extractor/html/recognizer/text.py @@ -11,8 +11,8 @@ from llm_web_kit.extractor.html.recognizer.recognizer import ( BaseHTMLElementRecognizer, CCTag) from llm_web_kit.libs.doc_element_type import DocElementType, ParagraphTextType -from llm_web_kit.libs.html_utils import (element_to_html, html_to_element, - process_sub_sup_tags) +from llm_web_kit.libs.html_utils import (element_to_html_unescaped, + html_to_element, process_sub_sup_tags) special_symbols = [ # TODO 从文件读取 '®', # 注册商标符号 @@ -114,7 +114,7 @@ def __to_cctext_lst(self, lst: List[Tuple[HtmlElement | str, HtmlElement | str]] para_text = self.__get_paragraph_text(el_element) if para_text: - cctext_el = self._build_cc_element(CCTag.CC_TEXT, json.dumps(para_text, ensure_ascii=False, indent=4), '', html=element_to_html(raw_html_element)) + cctext_el = self._build_cc_element(CCTag.CC_TEXT, json.dumps(para_text, ensure_ascii=False, indent=4), '', html=element_to_html_unescaped(raw_html_element)) new_lst.append((cctext_el, raw_html_element)) return new_lst diff --git a/tests/llm_web_kit/extractor/html/recognizer/test_text.py b/tests/llm_web_kit/extractor/html/recognizer/test_text.py index a7e48823..f3832906 100644 --- a/tests/llm_web_kit/extractor/html/recognizer/test_text.py +++ b/tests/llm_web_kit/extractor/html/recognizer/test_text.py @@ -9,7 +9,8 @@ BaseHTMLElementRecognizer from llm_web_kit.extractor.html.recognizer.text import TextParagraphRecognizer from llm_web_kit.input.datajson import DataJson -from llm_web_kit.libs.html_utils import element_to_html, html_to_element +from llm_web_kit.libs.html_utils import (element_to_html_unescaped, + html_to_element) class TestTextParagraphRecognize(unittest.TestCase): @@ -30,7 +31,7 @@ def test_text_1(self): '中共中央政治局召开会议审议《成-2020年10月16日新闻联播', 'zh') == '知识乱象\n中共中央政治局召开会议审议《成-2020年10月16日新闻联播' result = self.text_recognize.recognize('http://www.baidu.com', [(html_to_element(html_content), html_to_element(html_content))], html_content) - assert '知识乱象\\n\\n 中共中央政治局' in element_to_html(result[587][0]) + assert '知识乱象\\n\\n 中共中央政治局' in element_to_html_unescaped(result[587][0]) def test_text_2(self): """ @@ -152,7 +153,7 @@ def test_text_7(self): with open(Path(__file__).parent.parent.parent / 'assets/extractor_chain_input/good_data/html/text7.html', 'r') as file: html_content = file.read() result = self.text_recognize.recognize('http://www.baidu.com', [(html_to_element(html_content), html_to_element(html_content))], html_content) - assert '1) A man takes 5 hrs and 45 mins to walk to a certain place and ride back' in element_to_html(result[66][0]) and BaseHTMLElementRecognizer.is_cc_html(result[66][0]) + assert '1) A man takes 5 hrs and 45 mins to walk to a certain place and ride back' in element_to_html_unescaped(result[66][0]) and BaseHTMLElementRecognizer.is_cc_html(result[66][0]) def test_text_8(self): """ @@ -164,7 +165,7 @@ def test_text_8(self): with open(Path(__file__).parent.parent.parent / 'assets/extractor_chain_input/good_data/html/text8.html', 'r') as file: html_content = file.read() result = self.text_recognize.recognize('http://www.baidu.com', [(html_to_element(html_content), html_to_element(html_content))], html_content) - assert "40xy\' -ln(x^8) = 0\\n\\n\\n\\n Initial Condition: y(1)=31\\n\\n\\n\\n Work:" in element_to_html(result[69][0]) and BaseHTMLElementRecognizer.is_cc_html(result[69][0]) + assert "40xy\' -ln(x^8) = 0\\n\\n\\n\\n Initial Condition: y(1)=31\\n\\n\\n\\n Work:" in element_to_html_unescaped(result[69][0]) and BaseHTMLElementRecognizer.is_cc_html(result[69][0]) def test_text_9(self): """ @@ -176,7 +177,7 @@ def test_text_9(self): with open(Path(__file__).parent.parent.parent / 'assets/extractor_chain_input/good_data/html/text9.html', 'r') as file: html_content = file.read() result = self.text_recognize.recognize('http://www.baidu.com', [(html_to_element(html_content), html_to_element(html_content))], html_content) - assert '1) Consider the formula f(x)=lim(n--&gt;infinity)((x^n)/(1+x^n)).\\n\\n Let D={x:f(x) is an element of R}. Calculate f(x) for all x elements of D and determine where f: D--&gt;R is continuous.\\n\\n\\n\\n 2) Let f: D--&gt;R and suppose' in element_to_html(result[63][0]) and BaseHTMLElementRecognizer.is_cc_html(result[63][0]) + assert '1) Consider the formula f(x)=lim(n-->infinity)((x^n)/(1+x^n)).\\n\\n Let D={x:f(x) is an element of R}. Calculate f(x) for all x elements of D and determine where f: D-->R is continuous.\\n\\n\\n\\n 2) Let f: D-->R and suppose' in element_to_html_unescaped(result[63][0]) and BaseHTMLElementRecognizer.is_cc_html(result[63][0]) def test_text_10(self): """ From 34dd4c5e393d16a7b598c38bbb40aded432ff361 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Mon, 14 Apr 2025 15:11:33 +0800 Subject: [PATCH 03/13] =?UTF-8?q?feat:=20=E8=A1=A5=E5=85=A8=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/llm_web_kit/extractor/html/recognizer/test_text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/llm_web_kit/extractor/html/recognizer/test_text.py b/tests/llm_web_kit/extractor/html/recognizer/test_text.py index f3832906..7825392a 100644 --- a/tests/llm_web_kit/extractor/html/recognizer/test_text.py +++ b/tests/llm_web_kit/extractor/html/recognizer/test_text.py @@ -177,7 +177,7 @@ def test_text_9(self): with open(Path(__file__).parent.parent.parent / 'assets/extractor_chain_input/good_data/html/text9.html', 'r') as file: html_content = file.read() result = self.text_recognize.recognize('http://www.baidu.com', [(html_to_element(html_content), html_to_element(html_content))], html_content) - assert '1) Consider the formula f(x)=lim(n-->infinity)((x^n)/(1+x^n)).\\n\\n Let D={x:f(x) is an element of R}. Calculate f(x) for all x elements of D and determine where f: D-->R is continuous.\\n\\n\\n\\n 2) Let f: D-->R and suppose' in element_to_html_unescaped(result[63][0]) and BaseHTMLElementRecognizer.is_cc_html(result[63][0]) + assert '1) Consider the formula f(x)=lim(n-->infinity)((x^n)/(1+x^n)).\\n\\n Let D={x:f(x) is an element of R}. Calculate f(x) for all x elements of D and determine where f: D-->R is continuous.\\n\\n\\n\\n 2) Let f: D-->R and suppose that f(x) greater than equal 0 for all x elements of D. Define sqrt(f)-->R by (sqrt(f))(x) = sqrt(f(x)). If f is continuous at c elements of D, prove that sqrt(f) is continuous at c.' in element_to_html_unescaped(result[63][0]) and BaseHTMLElementRecognizer.is_cc_html(result[63][0]) def test_text_10(self): """ From 2d9db6cfdbccf4c20f67c675794ca523d899167b Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Sun, 27 Apr 2025 12:39:59 +0800 Subject: [PATCH 04/13] =?UTF-8?q?feat:=20=E7=B2=BE=E7=AE=80HTML?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main_html_parser/parser/tag_simplifier.py | 18 +- .../simplify_html/__init__.py | 0 .../simplify_html/simplify_html.py | 737 ++++++++++++++++++ .../test_html_data/test_tah_simplifier.html | 482 ++++++++++++ .../parser/test_tag_simplifier.py | 25 + 5 files changed, 1257 insertions(+), 5 deletions(-) create mode 100644 llm_web_kit/main_html_parser/simplify_html/__init__.py create mode 100644 llm_web_kit/main_html_parser/simplify_html/simplify_html.py create mode 100644 tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_tah_simplifier.html create mode 100644 tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py diff --git a/llm_web_kit/main_html_parser/parser/tag_simplifier.py b/llm_web_kit/main_html_parser/parser/tag_simplifier.py index 5f031da6..ffdf0671 100644 --- a/llm_web_kit/main_html_parser/parser/tag_simplifier.py +++ b/llm_web_kit/main_html_parser/parser/tag_simplifier.py @@ -1,12 +1,15 @@ +from llm_web_kit.exception.exception import TagSimplifiedParserException from llm_web_kit.input.pre_data_json import PreDataJson, PreDataJsonKey from llm_web_kit.main_html_parser.parser.parser import BaseMainHtmlParser +from llm_web_kit.main_html_parser.simplify_html.simplify_html import \ + simplify_html class HtmlTagSimplifierParser(BaseMainHtmlParser): - """HTML标签简化处理器，用于合并和精简HTML标签，确保标签总数不超过限制.""" + """HTML标签简化处理器.""" def parse(self, pre_data: PreDataJson) -> PreDataJson: - """简化HTML结构，合并相似标签并确保总标签数不超过200. + """简化HTML结构. Args: pre_data (PreDataJson): 包含原始HTML的PreDataJson对象 @@ -19,10 +22,15 @@ def parse(self, pre_data: PreDataJson) -> PreDataJson: # layout_file_list = pre_data.get(PreDataJsonKey.LAYOUT_FILE_LIST, []) # 执行HTML标签简化逻辑 - # ... + try: + simplified_html, original_html, _ = simplify_html(typical_raw_html, is_xpath=False) + except TagSimplifiedParserException as e1: + raise e1 + except Exception as e2: + raise e2 # 设置输出数据 - pre_data[PreDataJsonKey.TYPICAL_RAW_TAG_HTML] = typical_raw_html # 保存原始标签HTML - pre_data[PreDataJsonKey.TYPICAL_SIMPLIFIED_HTML] = '' # 保存简化后的HTML + pre_data[PreDataJsonKey.TYPICAL_RAW_TAG_HTML] = original_html # 保存原始标签HTML + pre_data[PreDataJsonKey.TYPICAL_SIMPLIFIED_HTML] = simplified_html # 保存简化后的HTML return pre_data diff --git a/llm_web_kit/main_html_parser/simplify_html/__init__.py b/llm_web_kit/main_html_parser/simplify_html/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py new file mode 100644 index 00000000..d296f0a9 --- /dev/null +++ b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py @@ -0,0 +1,737 @@ +import copy +import re +import uuid +from typing import Dict, List, Tuple + +from lxml import etree, html + +# 行内标签 +inline_tags = { + 'map', 'optgroup', 'span', 'br', 'input', 'time', 'u', 'td', 'strong', 'textarea', 'small', 'sub', + 'samp', 'blink', 'b', 'code', 'nobr', 'strike', 'bdo', 'basefont', 'abbr', 'var', 'i', 'cccode-inline', + 'th', 'select', 's', 'pic', 'label', 'mark', 'object', 'dd', 'dt', 'ccmath-inline', 'svg', 'li', + 'button', 'a', 'font', 'dfn', 'sup', 'kbd', 'q', 'script', 'acronym', 'option', 'img', 'big', 'cite', + 'em' +} + +# 需要删除的标签 +tags_to_remove = { + 'head', 'header', 'footer', 'nav', 'style', 'script', 'select', 'noscript', 'link', 'meta', 'iframe', 'frame' +} + +# 需要保留的特殊标签（即使它们是行内标签） +EXCLUDED_TAGS = {'img', 'br', 'li', 'dt', 'dd', 'td', 'th'} + +# 需要删除的属性名模式（独立单词） +ATTR_PATTERNS_TO_REMOVE = { + 'nav', 'footer', 'header', # 独立单词 +} + +# 需要删除的属性名模式（特定前缀/后缀） +ATTR_SUFFIX_TO_REMOVE = { + '-nav', '_nav', + # '-footer', '_footer', # 有特例，可能dl列表一组最后一项添加了自定义footer属性，先注释 + # '-header', '_header', # 有特例，可能自定义的header中有标题，先注释 +} + + +def add_data_uids(dom: html.HtmlElement) -> None: + """为DOM所有节点添加data-uid属性（递归所有子节点）""" + for node in dom.iter(): + try: + node.set('data-uid', str(uuid.uuid4())) + except TypeError: + pass + + +def remove_all_uids(dom: html.HtmlElement) -> None: + """移除DOM中所有data-uid属性.""" + for node in dom.iter(): + if 'data-uid' in node.attrib: + del node.attrib['data-uid'] + + +def build_uid_map(dom: html.HtmlElement) -> Dict[str, html.HtmlElement]: + """构建data-uid到节点的映射字典.""" + return {node.get('data-uid'): node for node in dom.iter() if node.get('data-uid')} + + +def is_unique_attribute(tree, attr_name, attr_value): + """检查给定的属性名和值组合是否在文档中唯一。""" + elements = tree.xpath(f"//*[@{attr_name}='{attr_value}']") + return len(elements) == 1 + + +def get_relative_xpath(element): + path = [] + root_element = element.getroottree().getroot() + + while element is not None and element.getparent() is not None: + siblings = [sib for sib in element.getparent() if sib.tag == element.tag] + + found_unique_attr = False # 初始化变量 + + if len(siblings) > 1: + # 如果有多个同名兄弟节点，则尝试使用属性路径来区分 + candidate_attrs = [ + attr for attr in element.attrib + if not (attr.startswith('data-') or attr == 'style' or + attr == '_item_id' or + (element.attrib[attr].startswith('{') and element.attrib[attr].endswith('}'))) + ] + + for attr in candidate_attrs: + if is_unique_attribute(root_element.getroottree(), attr, element.attrib[attr]): + # 插入唯一属性的XPath片段 + path.insert(0, f'*[@{attr}="{element.attrib[attr]}"]') + found_unique_attr = True + break + + if not found_unique_attr: + index = siblings.index(element) + 1 + path.insert(0, f'{element.tag}[{index}]') + else: + # 如果没有同名兄弟节点，则直接使用标签名 + path.insert(0, element.tag) + + # 提前返回简化版本的XPath，如果当前元素有一个唯一属性 + if found_unique_attr: + simplified_path = f'//{"/".join(path)}' + return simplified_path + + element = element.getparent() + + # 返回完整路径 + return f'//{"/".join(path)}' + + +def extract_paragraphs(processing_dom: html.HtmlElement, uid_map: Dict[str, html.HtmlElement], + include_parents: bool = True) -> List[Dict[str, str]]: + """获取段落. + + content_type 字段：用于标识段落内容的类型，可能的值包括： + + 'block_element'：独立的块级元素 + + 'inline_elements'：纯内联元素组合 + + 'unwrapped_text'：未包裹的纯文本内容 + + 'mixed'：混合内容（包含文本和内联元素） + + :param processing_dom: + :param uid_map: + :param include_parents: + :return: 段落列表，每个段落包含html、content_type和_original_element字段 + """ + + def is_block_element(node) -> bool: + """判断是否为块级元素.""" + if node.tag in inline_tags: + return False + return isinstance(node, html.HtmlElement) + + def has_block_children(node) -> bool: + """判断是否有块级子元素.""" + return any(is_block_element(child) for child in node.iterchildren()) + + def clone_structure(path: List[html.HtmlElement]) -> Tuple[html.HtmlElement, html.HtmlElement]: + """克隆节点结构.""" + if not path: + raise ValueError('Path cannot be empty') + if not include_parents: + last_node = html.Element(path[-1].tag, **path[-1].attrib) + return last_node, last_node + root = html.Element(path[0].tag, **path[0].attrib) + current = root + for node in path[1:-1]: + new_node = html.Element(node.tag, **node.attrib) + current.append(new_node) + current = new_node + last_node = html.Element(path[-1].tag, **path[-1].attrib) + current.append(last_node) + return root, last_node + + paragraphs = [] + + def process_node(node: html.HtmlElement, path: List[html.HtmlElement]): + """递归处理节点.""" + current_path = path + [node] + inline_content = [] + content_sources = [] + + # 处理节点文本 + if node.text and node.text.strip(): + inline_content.append(('direct_text', node.text.strip())) + content_sources.append('direct_text') + + # 处理子节点 + for child in node: + if is_block_element(child): + # 处理累积的内联内容 + if inline_content: + try: + root, last_node = clone_structure(current_path) + merge_inline_content(last_node, inline_content) + + content_type = 'mixed' + if all(t == 'direct_text' for t in content_sources): + content_type = 'unwrapped_text' + elif all(t == 'element' for t in content_sources): + content_type = 'inline_elements' + + # 获取原始元素 + original_element = uid_map.get(node.get('data-uid')) + paragraphs.append({ + 'html': etree.tostring(root, encoding='unicode').strip(), + 'content_type': content_type, + '_original_element': original_element # 添加原始元素引用 + }) + except ValueError: + pass + inline_content = [] + content_sources = [] + + # 处理块级元素 + if not has_block_children(child): + try: + root, last_node = clone_structure(current_path + [child]) + last_node.text = child.text if child.text else None + for grandchild in child: + last_node.append(copy.deepcopy(grandchild)) + + # 获取原始元素 + original_element = uid_map.get(child.get('data-uid')) + paragraphs.append({ + 'html': etree.tostring(root, encoding='unicode').strip(), + 'content_type': 'block_element', + '_original_element': original_element # 添加原始元素引用 + }) + except ValueError: + pass + else: + process_node(child, current_path) + + # 处理tail文本 + if child.tail and child.tail.strip(): + inline_content.append(('tail_text', child.tail.strip())) + content_sources.append('tail_text') + else: + inline_content.append(('element', child)) + content_sources.append('element') + if child.tail and child.tail.strip(): + inline_content.append(('tail_text', child.tail.strip())) + content_sources.append('tail_text') + + # 处理剩余的内联内容 + if inline_content: + try: + root, last_node = clone_structure(current_path) + merge_inline_content(last_node, inline_content) + + content_type = 'mixed' + if all(t == 'direct_text' for t in content_sources): + content_type = 'unwrapped_text' + elif all(t == 'element' for t in content_sources): + content_type = 'inline_elements' + elif all(t in ('direct_text', 'tail_text') for t in content_sources): + content_type = 'unwrapped_text' + + # 获取原始元素 + original_element = uid_map.get(node.get('data-uid')) + paragraphs.append({ + 'html': etree.tostring(root, encoding='unicode').strip(), + 'content_type': content_type, + '_original_element': original_element # 添加原始元素引用 + }) + except ValueError: + pass + + def merge_inline_content(parent: html.HtmlElement, content_list: List[Tuple[str, str]]): + """合并内联内容.""" + last_inserted = None + for item_type, item in content_list: + if item_type in ('direct_text', 'tail_text'): + if last_inserted is None: + if not parent.text: + parent.text = item + else: + parent.text += ' ' + item + else: + if last_inserted.tail is None: + last_inserted.tail = item + else: + last_inserted.tail += ' ' + item + else: + parent.append(copy.deepcopy(item)) + last_inserted = item + + # 开始处理 + process_node(processing_dom, []) + + # 去重 + seen = set() + unique_paragraphs = [] + for p in paragraphs: + if p['html'] not in seen: + seen.add(p['html']) + unique_paragraphs.append(p) + + return unique_paragraphs + + +def remove_xml_declaration(html_string): + # 正则表达式匹配 或 （没有问号结尾的情况） + pattern = r'<\?xml\s+.*?\??>' + return re.sub(pattern, '', html_string, flags=re.DOTALL) + + +def post_process_html(html_content: str) -> str: + """对简化后的HTML进行后处理.""" + if not html_content: + return html_content + + # 1. 删除HTML注释 + html_content = re.sub(r'', '', html_content, flags=re.DOTALL) + + # 2. 处理标签外的空白（保留标签内文本的换行） + def replace_outside_tag_space(match): + """只替换标签外的连续空白.""" + if match.group(1): # 如果是标签内容 + return match.group(1) + elif match.group(2): # 如果是非标签内容 + # 将非标签内容中的连续空白替换为单个空格 + return re.sub(r'\s+', ' ', match.group(2)) + return match.group(0) # 默认返回整个匹配 + + # 使用正则匹配所有标签内容和非标签内容 + html_content = re.sub(r'(<[^>]+>)|([^<]+)', replace_outside_tag_space, html_content) + + return html_content.strip() + + +def remove_tags(dom): + """删除特定的标签. + + :param dom: + :return: + """ + for tag in tags_to_remove: + for node in dom.xpath(f'.//{tag}'): + parent = node.getparent() + if parent is not None: + parent.remove(node) + + +def remove_inline_tags_except_img_br(dom): + # 定义块级元素列表（不完全，可根据需要扩展） + + # 使用深度优先遍历DOM树 + for element in dom.iter(): + if element.tag not in inline_tags: + # 处理块元素中的子元素 + for child in list(element): # 使用list()创建副本，因为我们会修改原结构 + if child.tag in inline_tags and child.tag in ['br', 'img']: + # 移除行内元素，但保留其文本内容 + parent = child.getparent() + if child.text: + prev = child.getprevious() + if prev is not None: + prev.tail = (prev.tail or '') + child.text + else: + parent.text = (parent.text or '') + child.text + + # 处理子元素的子元素（如果有） + for subchild in child: + parent.insert(parent.index(child), subchild) + + if child.tail: + prev = child.getprevious() + if prev is not None: + prev.tail = (prev.tail or '') + child.tail + else: + parent.text = (parent.text or '') + child.tail + + parent.remove(child) + + +def is_meaningful_content(element) -> bool: + """严格判断元素是否包含有效内容.""" + if element.text and element.text.strip(): + return True + if element.tag == 'img': + src = element.get('src', '') + return bool(src and src.strip()) + for child in element: + if is_meaningful_content(child): + return True + if element.tail and element.tail.strip(): + return True + return False + + +def clean_attributes(element): + """清理元素属性，只保留图片的有效src.""" + if element.tag == 'img': + src = element.get('src', '').strip() + element.attrib.clear() + if src: + element.set('src', src) + else: + element.attrib.clear() + for child in element: + clean_attributes(child) + + +def _remove_nested_inline_tags_only(element): + """只移除嵌套的行内标签，保留当前元素本身（即使是行内标签） 用于处理inline_elements和mixed类型的内容.""" + # 先处理子元素（深度优先） + for child in list(element.iterchildren()): + _remove_nested_inline_tags_only(child) + + # 如果子元素是需要移除的行内标签 + if child.tag in inline_tags and child.tag not in EXCLUDED_TAGS: + parent = child.getparent() + if parent is None: + continue + + # 转移内容到父元素 + text = child.text or '' + tail = child.tail or '' + grandchildren = list(child.iterchildren()) + + # 处理元素文本 + if text: + prev = child.getprevious() + if prev is not None: + prev.tail = (prev.tail or '') + text + else: + parent.text = (parent.text or '') + text + + # 转移子节点 + index = parent.index(child) + for grandchild in reversed(grandchildren): + parent.insert(index, grandchild) + + # 处理尾部文本 + if tail: + prev = child.getprevious() + if prev is not None: + prev.tail = (prev.tail or '') + tail + else: + parent.text = (parent.text or '') + tail + + # 移除当前子元素 + parent.remove(child) + + +def remove_inline_tags(element): + """递归移除所有指定的行内标签（包括嵌套情况），保留img和br标签 优化点：确保文本顺序正确，正确处理嵌套标签的文本转移.""" + # 先处理子元素（深度优先） + for child in list(element.iterchildren()): + remove_inline_tags(child) + + # 如果当前元素是需要移除的行内标签 + if element.tag in inline_tags and element.tag not in EXCLUDED_TAGS: + parent = element.getparent() + if parent is None: + return + + # 保存当前元素的各部分内容 + leading_text = element.text or '' # 元素开始前的文本 + trailing_text = element.tail or '' # 元素结束后的文本 + children = list(element) # 子元素列表 + + # 获取当前元素在父元素中的位置 + element_index = parent.index(element) + + # 1. 处理leading_text（元素开始前的文本） + if leading_text: + if element_index == 0: # 如果是第一个子元素 + parent.text = (parent.text or '') + leading_text + else: + prev_sibling = parent[element_index - 1] + prev_sibling.tail = (prev_sibling.tail or '') + leading_text + + # 2. 转移子元素到父元素中 + for child in reversed(children): + parent.insert(element_index, child) + + # 3. 处理trailing_text（元素结束后的文本） + if trailing_text: + if len(children) > 0: # 如果有子元素，追加到最后一个子元素的tail + last_child = children[-1] + last_child.tail = (last_child.tail or '') + trailing_text + elif element_index == 0: # 如果没有子元素且是第一个子元素 + parent.text = (parent.text or '') + trailing_text + else: # 如果没有子元素且不是第一个子元素 + prev_sibling = parent[element_index - 1] if element_index > 0 else None + if prev_sibling is not None: + prev_sibling.tail = (prev_sibling.tail or '') + trailing_text + else: + parent.text = (parent.text or '') + trailing_text + + # 4. 移除当前元素 + parent.remove(element) + + +def simplify_list(element): + """简化列表元素，只保留第一组和最后一组（对于dl列表保留完整的dt+所有dd）""" + if element.tag in ('ul', 'ol'): + # 处理普通列表(ul/ol) + items = list(element.iterchildren()) + if len(items) > 2: + # 保留第一个和最后一个子元素 + for item in items[1:-1]: + element.remove(item) + + # 在第一个和最后一个之间添加省略号 + ellipsis = etree.Element('span') + ellipsis.text = '...' + items[-1].addprevious(ellipsis) + + elif element.tag == 'dl': + # 处理定义列表(dl) + items = list(element.iterchildren()) + if len(items) > 2: + # 找出所有dt元素 + dts = [item for item in items if item.tag == 'dt'] + + if len(dts) > 1: + # 获取第一组dt和所有后续dd + first_dt_index = items.index(dts[0]) + next_dt_index = items.index(dts[1]) + first_group = items[first_dt_index:next_dt_index] + + # 获取最后一组dt和所有后续dd + last_dt_index = items.index(dts[-1]) + last_group = items[last_dt_index:] + + # 清空dl元素 + for child in list(element.iterchildren()): + element.remove(child) + + # 添加第一组完整内容 + for item in first_group: + element.append(item) + + # 添加省略号 + ellipsis = etree.Element('span') + ellipsis.text = '...' + element.append(ellipsis) + + # 添加最后一组完整内容 + for item in last_group: + element.append(item) + + # 递归处理子元素 + for child in element: + simplify_list(child) + + +def should_remove_element(element) -> bool: + """判断元素的class或id属性是否匹配需要删除的模式.""" + # 检查class属性 + class_name = element.get('class', '') + if class_name: + class_parts = class_name.strip().split() + for part in class_parts: + # 检查是否完全匹配独立单词 + if part in ATTR_PATTERNS_TO_REMOVE: + return True + # 检查是否包含特定前缀/后缀 + for pattern in ATTR_SUFFIX_TO_REMOVE: + if pattern in part: + return True + + # 检查id属性 + id_name = element.get('id', '') + if id_name: + id_parts = id_name.strip().split('-') # id通常用连字符分隔 + for part in id_parts: + # 检查是否完全匹配独立单词 + if part in ATTR_PATTERNS_TO_REMOVE: + return True + # 检查是否包含特定前缀/后缀 + for pattern in ATTR_SUFFIX_TO_REMOVE: + if pattern in part: + return True + return False + + +def remove_specific_elements(element): + """删除class或id名匹配特定模式的标签及其内容.""" + for child in list(element.iterchildren()): + remove_specific_elements(child) + + if should_remove_element(element): + parent = element.getparent() + if parent is not None: + parent.remove(element) + + +def truncate_text_content(element, max_length=500): + """递归处理元素及其子元素的文本内容，总长度超过max_length时截断 但保持标签结构完整.""" + # 首先收集所有文本节点（包括text和tail） + text_nodes = [] + + # 收集元素的text + if element.text and element.text.strip(): + text_nodes.append(('text', element, element.text)) + + # 递归处理子元素 + for child in element: + truncate_text_content(child, max_length) + # 收集子元素的tail + if child.tail and child.tail.strip(): + text_nodes.append(('tail', child, child.tail)) + + # 计算当前元素下的总文本长度 + total_length = sum(len(text) for (typ, node, text) in text_nodes) + + # 如果总长度不超过限制，直接返回 + if total_length <= max_length: + return + + # 否则进行截断处理 + remaining = max_length + for typ, node, text in text_nodes: + if remaining <= 0: + # 已经达到限制，清空剩余的文本内容 + if typ == 'text': + node.text = None + else: + node.tail = None + continue + + if len(text) > remaining: + # 需要截断这个文本节点 + if typ == 'text': + node.text = text[:remaining] + '...' + else: + node.tail = text[:remaining] + '...' + remaining = 0 + else: + remaining -= len(text) + + +def process_paragraphs(paragraphs: List[Dict[str, str]], uid_map: Dict[str, html.HtmlElement], is_xpath: bool = True) -> Tuple[str, html.HtmlElement]: + """处理段落并添加 _item_id，同时在原始DOM的对应元素上添加相同ID. + + Args: + paragraphs: 段落列表，每个段落包含html、content_type和_original_element + original_dom: 原始DOM树 + + Returns: + Tuple[简化后的HTML, 标记后的原始DOM] + """ + result = [] + item_id = 1 + + for para in paragraphs: + try: + # print(f"para: {para['html']}") + # 解析段落HTML + root = html.fromstring(post_process_html(para['html'])) + root_for_xpath = copy.deepcopy(root) + content_type = para.get('content_type', 'block_element') + + # 公共处理步骤 + clean_attributes(root) + simplify_list(root) + remove_inline_tags(root) + + # 跳过无意义内容 + if not is_meaningful_content(root): + continue + + # 截断过长的文本内容 + truncate_text_content(root, max_length=1000) + + # 为当前段落和原始元素添加相同的 _item_id + current_id = str(item_id) + root.set('_item_id', current_id) + para['_original_element'].set('_item_id', current_id) + + para_xpath = [] + if is_xpath: + if content_type in ('inline_elements', 'mixed'): + for child in root_for_xpath.iterchildren(): + original_element = uid_map.get(child.get('data-uid')) + try: + _xpath = get_relative_xpath(original_element) + except Exception: + _xpath = None + para_xpath.append(_xpath) + elif content_type == 'block_element': + try: + _xpath = get_relative_xpath(para['_original_element']) + except Exception: + _xpath = None + para_xpath.append(_xpath) + else: + try: + _xpath = get_relative_xpath(para['_original_element']) + except Exception: + _xpath = None + para_xpath.append(_xpath) + + item_id += 1 + + # 保存处理结果 + cleaned_html = etree.tostring(root, encoding='unicode').strip() + result.append({ + 'html': cleaned_html, + '_item_id': current_id, + '_xpath': para_xpath, + 'content_type': content_type + }) + + except Exception: + # import traceback + # print(f'处理段落出错: {traceback.format_exc()}') + continue + + # 组装最终HTML + simplified_html = '' + ''.join( + p['html'] for p in result) + '' + + return simplified_html, result + + +def simplify_html(html_str, is_xpath: bool = True) -> etree.Element: + """ + :return: + simplified_html: 精简HTML + original_html: 添加_item_id的原始HTML + _xpath_mapping: xpath映射 + """ + # 预处理 + preprocessed_html = remove_xml_declaration(html_str) + + # 解析原始DOM + original_dom = html.fromstring(preprocessed_html) + add_data_uids(original_dom) + original_uid_map = build_uid_map(original_dom) + + # 创建处理用的DOM（深拷贝） + processing_dom = copy.deepcopy(original_dom) + # 清理DOM + remove_tags(processing_dom) + remove_specific_elements(processing_dom) + + # 提取段落（会记录原始元素引用） + paragraphs = extract_paragraphs(processing_dom, original_uid_map, include_parents=False) + + # 处理段落（同步添加ID） + simplified_html, result = process_paragraphs(paragraphs, original_uid_map, is_xpath) + + remove_all_uids(original_dom) + original_html = etree.tostring(original_dom, pretty_print=True, encoding='unicode') + + _xpath_mapping = {item['_item_id']: { + '_xpath': item['_xpath'], + 'content_type': item['content_type'] + } for item in result} + + return simplified_html, original_html, _xpath_mapping diff --git a/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_tah_simplifier.html b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_tah_simplifier.html new file mode 100644 index 00000000..b1e759b7 --- /dev/null +++ b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_tah_simplifier.html @@ -0,0 +1,482 @@ + + + + + + + + + + + + + + + + + + + + + Accelerating the Deployment of RAN Open Source + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + +
      +
      + +
      +
      +

      AT&T and Nokia Accelerate the Deployment of RAN Open Source

      +
      +
      + + +
      + + +
      + +
      +
      +
      + + +
      + + +
      + + +
      + + +
      + + DALLAS, January 08, 2019 + +
      +
      + + + + + + + + + + +
      + + + +
      + +
      +

      AT&T and Nokia Accelerate the Deployment of RAN Open Source

      +
      +
      + + +
      +
      + + + + +
      + + +
      +
      + +
      +

      The companies contribute code to provide open access to 5G under the O-RAN Alliance

      +

      In support of 5G, AT&T* will begin development of a software platform for the RAN Intelligent Controller (RIC), to enable the creation of open source software that is aligned with the O-RAN target architecture. AT&T and Nokia are co-creating the platform code to accelerate the deployment of open source software for the 5G RAN.

      +

      The RIC platform will provide a set of functions and interfaces that allow for increased optimizations through policy-driven closed loop automation and for faster, more flexible service deployments and programmability within the RAN. The intent is to enable an intelligent rapidly evolvable radio network by fostering the creation of a multi-vendor open ecosystem of interoperable components for the disaggregated RAN.

      +

      “We’re delighted to work with Nokia on co-creating an open source implementation of the RIC platform to accelerate innovations and interoperability in the RAN,” said Mazin Gilbert, vice president of advanced technology and systems at AT&T Labs. “We continue to look for opportunities to drive open platforms and open interfaces in the community."

      +

      “Nokia is a strong proponent of RAN network openness and has been active in numerous open source communities, contributing code and defining open interface specifications,” said Mike Murphy, CTO for North America, Nokia. “We are excited to be partnering with AT&T to co-create RIC software and share with the open-source community to foster further collaboration and innovation.”

      +

      The platform will be architected in the form of an extensible real-time microservices framework coupled with a radio information database and key open control plane interfaces for mobility management, spectrum management, load balancing, radio resource control and RAN slicing to name a few. This will allow implementations of these functions, sourced from multiple vendors, to be mixed and matched on a single network infrastructure. The platform will also enable interfaces to third party applications for enhanced mobility functions such as cross layer optimization and machine learning inferences.

      +

      To further support the company’s 5G initiatives, AT&T will increase its engagement in Akraino Edge Stack, a Linux Foundation project focused on building production ready cloud infrastructure for edge deployment in open source. In particular, AT&T has also signed a multiyear co- development agreement with Nokia to further expand Akraino Edge Stack capabilities supporting the needs of the RIC and other edge cloud platform deployments. This work will support the growing need for virtualized edge environments of various sizes, deployment locations and capabilities.

      + +
      + + +
      +
      +
      +
      + +
       
      +
      + +
      +
      +

      *About AT&T Communications

      +

      We help family, friends and neighbors connect in meaningful ways every day. From the first phone call 140+ years ago to mobile video streaming, we innovate to improve lives. We have the best network according to America’s biggest test.** We’re building FirstNet just for first responders and creating next-generation mobile 5G. With DIRECTV and DIRECTV NOW, we deliver entertainment people love to talk about. Our smart, highly secure solutions serve over 3 million global businesses – nearly all of the Fortune 1000. And worldwide, our spirit of service drives employees to give back to their communities.

      +

      AT&T Communications is part of AT&T Inc. (NYSE:T). Learn more at att.com/CommunicationsNews.

      +

      AT&T products and services are provided or offered by subsidiaries and affiliates of AT&T Inc. under the AT&T brand and not by AT&T Inc. Additional information about AT&T products and services is available at about.att.com. Follow our news on Twitter at @ATT, on Facebook at facebook.com/att and on YouTube at youtube.com/att.

      +

      © 2018 AT&T Intellectual Property. All rights reserved. AT&T, the Globe logo and other marks are trademarks and service marks of AT&T Intellectual Property and/or AT&T affiliated companies. All other marks contained herein are the property of their respective owners.

      +

      **Based on GWS OneScore Sept. 2018. Excludes crowd sourced studies.

      + +
      + + +
      + View More +
      + +
      +
      + +
      + +
      +
      +
      +
      +
      + + +
      +
      + + +
      + + + + + + +
      +
      +
      + + + + + + + +
      +
      +
      +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      First in the U.S. to Mobile 5G – What’s Next? Defining AT&T’s Network Path in 2019 and Beyond

      + +
      +

      We plan for nationwide 5G Evolution coverage by spring 2019 and nationwide mobile 5G coverage by early 2020.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      More Than 50,000 Square Miles of LTE Coverage Added Nationwide to Support AT&T and FirstNet Users

      + +
      +

      We increased the LTE coverage area for the AT&T network and FirstNet public safety communications platform nationwide.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      Las Vegas Pilots Smart Lighting Solution with AT&T, Ubicquia

      + +
      +

      The City of Las Vegas will pilot a smart lighting solution to help improve public safety and increase energy savings.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + +
      +
      + +
      +
      + + + + + + + + + +
      + + +
      + + + +
      + + + + + +
      + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py b/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py new file mode 100644 index 00000000..08e46eae --- /dev/null +++ b/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py @@ -0,0 +1,25 @@ +import unittest +from pathlib import Path + +from llm_web_kit.input.pre_data_json import PreDataJson, PreDataJsonKey +from llm_web_kit.main_html_parser.parser.tag_simplifier import \ + HtmlTagSimplifierParser + +base_dir = Path(__file__).resolve().parent + + +class MyTestCase(unittest.TestCase): + def test_tag_simplifier(self): + file_path = base_dir / 'assets/test_html_data/test_tah_simplifier.html' + with open(file_path, 'r', encoding='utf-8') as file: + raw_html = file.read() + data_dict = {PreDataJsonKey.TYPICAL_RAW_HTML: raw_html} + pre_data = PreDataJson(data_dict) + pre_data_result = HtmlTagSimplifierParser({}).parse(pre_data) + simplifier_raw_html = pre_data_result.get(PreDataJsonKey.TYPICAL_SIMPLIFIED_HTML, '') + _item_id_count = simplifier_raw_html.count('_item_id') + self.assertEqual(_item_id_count, 34) + + +if __name__ == '__main__': + unittest.main() From ad7c7e143a4c5246885e42b003e8b40a128a5842 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Sun, 27 Apr 2025 13:26:23 +0800 Subject: [PATCH 05/13] =?UTF-8?q?feat:=20=E7=B2=BE=E7=AE=80HTML?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm_web_kit/main_html_parser/simplify_html/simplify_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py index d296f0a9..b8024505 100644 --- a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py +++ b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py @@ -16,7 +16,7 @@ # 需要删除的标签 tags_to_remove = { - 'head', 'header', 'footer', 'nav', 'style', 'script', 'select', 'noscript', 'link', 'meta', 'iframe', 'frame' + 'head', 'header', 'footer', 'nav', 'aside', 'style', 'script', 'select', 'noscript', 'link', 'meta', 'iframe', 'frame' } # 需要保留的特殊标签（即使它们是行内标签） From 0a26cf8a08b75c0b4dafe48bb14cf2ed02caffec Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Sun, 27 Apr 2025 14:37:18 +0800 Subject: [PATCH 06/13] =?UTF-8?q?feat:=20HTML=E4=BB=A3=E8=A1=A8=E6=80=A7?= =?UTF-8?q?=E7=BD=91=E9=A1=B5=E9=80=89=E6=8B=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../parser/typical_html_selector.py | 14 +- .../main_html_parser/typical_html/__init__.py | 0 .../typical_html/typical_html.py | 78 ++ .../test_typical_html_data/0.html | 482 +++++++++++++ .../test_typical_html_data/1.html | 570 +++++++++++++++ .../test_typical_html_data/2.html | 522 ++++++++++++++ .../test_typical_html_data/3.html | 676 ++++++++++++++++++ .../test_typical_html_data/4.html | 542 ++++++++++++++ .../test_typical_html_data/5.html | 503 +++++++++++++ .../test_typical_html_data/6.html | 548 ++++++++++++++ .../test_typical_html_data/7.html | 568 +++++++++++++++ .../test_typical_html_data/8.html | 563 +++++++++++++++ .../test_typical_html_data/9.html | 626 ++++++++++++++++ .../parser/test_tag_simplifier.py | 2 +- .../parser/test_typical_html_selector.py | 30 + 15 files changed, 5720 insertions(+), 4 deletions(-) create mode 100644 llm_web_kit/main_html_parser/typical_html/__init__.py create mode 100644 llm_web_kit/main_html_parser/typical_html/typical_html.py create mode 100644 tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/0.html create mode 100644 tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/1.html create mode 100644 tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/2.html create mode 100644 tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/3.html create mode 100644 tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/4.html create mode 100644 tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/5.html create mode 100644 tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/6.html create mode 100644 tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/7.html create mode 100644 tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/8.html create mode 100644 tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/9.html create mode 100644 tests/llm_web_kit/main_html_parser/parser/test_typical_html_selector.py diff --git a/llm_web_kit/main_html_parser/parser/typical_html_selector.py b/llm_web_kit/main_html_parser/parser/typical_html_selector.py index 6538f50e..698bea1d 100644 --- a/llm_web_kit/main_html_parser/parser/typical_html_selector.py +++ b/llm_web_kit/main_html_parser/parser/typical_html_selector.py @@ -1,5 +1,8 @@ +from llm_web_kit.exception.exception import TypicalHtmlSelectorParserException from llm_web_kit.input.pre_data_json import PreDataJson, PreDataJsonKey from llm_web_kit.main_html_parser.parser.parser import BaseMainHtmlParser +from llm_web_kit.main_html_parser.typical_html.typical_html import \ + select_representative_html class TypicalHtmlSelectorParser(BaseMainHtmlParser): @@ -13,9 +16,14 @@ def parse(self, pre_data: PreDataJson) -> PreDataJson: PreDataJson: 包含代表性网页的PreDataJson对象 """ # 选择代表性网页逻辑 - # ... - + layout_file_list = pre_data.get(PreDataJsonKey.LAYOUT_FILE_LIST, []) + try: + typical_html = select_representative_html(layout_file_list) + except TypicalHtmlSelectorParserException as e1: + raise e1 + except Exception as e2: + raise e2 # 设置输出数据 - pre_data[PreDataJsonKey.TYPICAL_RAW_HTML] = '' + pre_data[PreDataJsonKey.TYPICAL_RAW_HTML] = typical_html return pre_data diff --git a/llm_web_kit/main_html_parser/typical_html/__init__.py b/llm_web_kit/main_html_parser/typical_html/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/llm_web_kit/main_html_parser/typical_html/typical_html.py b/llm_web_kit/main_html_parser/typical_html/typical_html.py new file mode 100644 index 00000000..8ed5d189 --- /dev/null +++ b/llm_web_kit/main_html_parser/typical_html/typical_html.py @@ -0,0 +1,78 @@ +from io import StringIO + +from lxml import html + +REQUIRED_TAGS = {'', ''} + + +def has_essential_tags(html_str): + """检查是否包含关键标签.""" + lower_html = html_str.lower() + return all(tag in lower_html for tag in REQUIRED_TAGS) + + +def select_representative_html(html_content_list): + """从多个HTML文件中选择最具代表性的一个.""" + # 存储所有页面的XPath和复杂度信息 + page_data = [] + global_xpaths = set() + + # 第一遍：收集所有页面的XPath和基本复杂度 + for item in html_content_list: + try: + html_str = item['html'] + track_id = item['track_id'] + + if not has_essential_tags(html_str): + continue + + if len(html_str) < 100: + continue + + file_obj = StringIO(html_str) + tree = html.parse(file_obj) + + # 收集本页所有XPath + page_xpaths = set() + total_tags = 0 + for element in tree.iter(): + xpath = tree.getpath(element) + page_xpaths.add(xpath) + total_tags += 1 + + # 记录页面数据 + page_data.append({ + 'track_id': track_id, + 'xpaths': page_xpaths, + 'tag_count': total_tags, + 'original_data': item, + }) + + # 更新全局XPath集合 + global_xpaths.update(page_xpaths) + + except Exception: + continue + + if not page_data: + return None + + # 第二遍：计算每个页面的代表得分 + best_score = -1 + representative_path = None + + for page in page_data: + # 计算XPath覆盖率（该页面包含的全局XPath比例） + coverage = len(page['xpaths'] & global_xpaths) / len(global_xpaths) + + # 计算复杂度得分（基于标签数量） + complexity = page['tag_count'] / max(p['tag_count'] for p in page_data) + + # 综合得分（可根据需要调整权重） + score = 0.6 * coverage + 0.4 * complexity + + if score > best_score: + best_score = score + representative_path = page['original_data'] + + return representative_path diff --git a/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/0.html b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/0.html new file mode 100644 index 00000000..ee52df7c --- /dev/null +++ b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/0.html @@ -0,0 +1,482 @@ + + + + + + + + + + + + + + + + + + + + + Accelerating the Deployment of RAN Open Source + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + +
      +
      + +
      +
      +

      AT&T and Nokia Accelerate the Deployment of RAN Open Source

      +
      +
      + + +
      + + +
      + +
      +
      +
      + + +
      + + +
      + + +
      + + +
      + + DALLAS, January 08, 2019 + +
      +
      + + + + + + + + + + +
      + + + +
      + +
      +

      AT&T and Nokia Accelerate the Deployment of RAN Open Source

      +
      +
      + + +
      +
      + + + + +
      + + +
      +
      + +
      +

      The companies contribute code to provide open access to 5G under the O-RAN Alliance

      +

      In support of 5G, AT&T* will begin development of a software platform for the RAN Intelligent Controller (RIC), to enable the creation of open source software that is aligned with the O-RAN target architecture. AT&T and Nokia are co-creating the platform code to accelerate the deployment of open source software for the 5G RAN.

      +

      The RIC platform will provide a set of functions and interfaces that allow for increased optimizations through policy-driven closed loop automation and for faster, more flexible service deployments and programmability within the RAN. The intent is to enable an intelligent rapidly evolvable radio network by fostering the creation of a multi-vendor open ecosystem of interoperable components for the disaggregated RAN.

      +

      “We’re delighted to work with Nokia on co-creating an open source implementation of the RIC platform to accelerate innovations and interoperability in the RAN,” said Mazin Gilbert, vice president of advanced technology and systems at AT&T Labs. “We continue to look for opportunities to drive open platforms and open interfaces in the community."

      +

      “Nokia is a strong proponent of RAN network openness and has been active in numerous open source communities, contributing code and defining open interface specifications,” said Mike Murphy, CTO for North America, Nokia. “We are excited to be partnering with AT&T to co-create RIC software and share with the open-source community to foster further collaboration and innovation.”

      +

      The platform will be architected in the form of an extensible real-time microservices framework coupled with a radio information database and key open control plane interfaces for mobility management, spectrum management, load balancing, radio resource control and RAN slicing to name a few. This will allow implementations of these functions, sourced from multiple vendors, to be mixed and matched on a single network infrastructure. The platform will also enable interfaces to third party applications for enhanced mobility functions such as cross layer optimization and machine learning inferences.

      +

      To further support the company’s 5G initiatives, AT&T will increase its engagement in Akraino Edge Stack, a Linux Foundation project focused on building production ready cloud infrastructure for edge deployment in open source. In particular, AT&T has also signed a multiyear co- development agreement with Nokia to further expand Akraino Edge Stack capabilities supporting the needs of the RIC and other edge cloud platform deployments. This work will support the growing need for virtualized edge environments of various sizes, deployment locations and capabilities.

      + +
      + + +
      +
      +
      +
      + +
       
      +
      + +
      +
      +

      *About AT&T Communications

      +

      We help family, friends and neighbors connect in meaningful ways every day. From the first phone call 140+ years ago to mobile video streaming, we innovate to improve lives. We have the best network according to America’s biggest test.** We’re building FirstNet just for first responders and creating next-generation mobile 5G. With DIRECTV and DIRECTV NOW, we deliver entertainment people love to talk about. Our smart, highly secure solutions serve over 3 million global businesses – nearly all of the Fortune 1000. And worldwide, our spirit of service drives employees to give back to their communities.

      +

      AT&T Communications is part of AT&T Inc. (NYSE:T). Learn more at att.com/CommunicationsNews.

      +

      AT&T products and services are provided or offered by subsidiaries and affiliates of AT&T Inc. under the AT&T brand and not by AT&T Inc. Additional information about AT&T products and services is available at about.att.com. Follow our news on Twitter at @ATT, on Facebook at facebook.com/att and on YouTube at youtube.com/att.

      +

      © 2018 AT&T Intellectual Property. All rights reserved. AT&T, the Globe logo and other marks are trademarks and service marks of AT&T Intellectual Property and/or AT&T affiliated companies. All other marks contained herein are the property of their respective owners.

      +

      **Based on GWS OneScore Sept. 2018. Excludes crowd sourced studies.

      + +
      + + +
      + View More +
      + +
      +
      + +
      + +
      +
      +
      +
      +
      + + +
      +
      + + +
      + + + + + + +
      +
      +
      + + + + + + + +
      +
      +
      +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      First in the U.S. to Mobile 5G – What’s Next? Defining AT&T’s Network Path in 2019 and Beyond

      + +
      +

      We plan for nationwide 5G Evolution coverage by spring 2019 and nationwide mobile 5G coverage by early 2020.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      More Than 50,000 Square Miles of LTE Coverage Added Nationwide to Support AT&T and FirstNet Users

      + +
      +

      We increased the LTE coverage area for the AT&T network and FirstNet public safety communications platform nationwide.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      Las Vegas Pilots Smart Lighting Solution with AT&T, Ubicquia

      + +
      +

      The City of Las Vegas will pilot a smart lighting solution to help improve public safety and increase energy savings.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + +
      +
      + +
      +
      + + + + + + + + + +
      + + +
      + + + +
      + + + + + +
      + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/1.html b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/1.html new file mode 100644 index 00000000..7b4ec8c5 --- /dev/null +++ b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/1.html @@ -0,0 +1,570 @@ + + + + + + + + + + + + + + + + + + + + + New Effort Transforms Golf Tourney ‘Trash’ to Backpacks | AT&T + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + + +
      + + +
      + +
      +
      +
      + + +
      + + +
      + + +
      + + +
      + + May 14, 2018 + +
      +
      + + + + + + + + + + +
      + + + +
      + +
      +

      New Effort Transforms Tourney ‘Trash’ to Backpacks

      +
      +
      + +
      + +
      + + +
      +
      + + + + +
      +
      +
      + + +
      +
      + +
      +

      By Andy Morgan +

      +

      Soda cans and water bottles aren’t the only things destined for recycling at this year’s AT&T Byron Nelson.

      +

      Thousands of yards of branded cloth material – also known as “scrim” -- used for tournament signs and to help designate spectator areas will be recycled and made into some 1,000 student backpacks.

      +

      Nicole Anderson, an AT&T associate vice president for citizenship and sustainability, calls the process “upcycling.”

      +

      “We’ll end up with about 4 tons of material from the AT&T Byron Nelson,” Anderson said. “We’ll upcycle a portion of that, turning it into backpacks in time for school next fall, and recycle the rest.”

      +

      Anderson hopes to work with AT&T employee groups to stuff the backpacks with school supplies before they’re distributed to students in southern Dallas.

      +

      “This upcycling effort meshes well with the sustainability efforts of Trinity Forest Golf Club, the new home of the AT&T Byron Nelson,” said Shiz Suzuki, AT&T assistant vice president-sponsorships. “The course just won a major environmental award, so we’re proud to do our part to keep the focus on sustainability.”

      +

      It’s not the first time that our golf tournament collateral has been given a new use. Caddy bibs, or the aprons the pro caddies wear during tournament play, from the 2015 AT&T Byron Nelson have been reborn as luggage tags.

      +

      AT&T is giving out about 2,500 of the tags to AT&T customers, Pro-Am golfers and members of the media at this year’s tournament.

      +

      “As a company, our environmental sustainability efforts are growing and expanding into several areas, including our sponsored events, like the AT&T Byron Nelson,” Anderson said. “As part of that effort, whenever we see an opportunity to transform ‘trash’ into something useful that can benefit the local community, we’re taking a look at it. And that’s exciting.”

      + +
      + + +
      +
      +
      +
      + + + + + + + +
      +
      +
      + + Slide 0 alt text + + Slide 1 alt text + +
      + + +
      +
      +
      + + Slide 0 alt text + + Slide 1 alt text + +
      +
      + + +
      +
      +
      + + +
      +
      + + +
      +
      + +
      +

      AT&T Citizenship & Sustainability employees Amy Ramos, left, and Nicole Anderson check over a box of luggage tags made from “upcycled” signage from the 2015 AT&T Byron Nelson tournament. This year, tournament signage will be reused to make about 1,000 backpacks for students in southern Dallas.

      + +
      + + +
      +
      +
      + +
      +
      +
      + + +
      +
      + + +
      + + + + + + +
      +
      +
      + + + + + + + +
      +
      +
      +
      + + + +
      + +
      +
      +
      + +
      + + + +
      + +
      +
      +
      + + + +

      Building Climate Resilience

      + +
      +

      From out of control wildfires in California to crippling hurricanes on the Gulf coasts, disasters are increasing.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      Lowe’s Teams Up with AT&T and HydroPoint Smart Water Management

      + +
      +

      AT&T IoT solutions and HydroPoint are helping improve Lowe’s commitment to environmental sustainability via the use of connected smart irrigation controllers.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      AT&T and Synchronoss Simplify Building Management for Businesses

      + +
      +

      Our new offering will allow business owners to proactively monitor, manage and maintain their building’s energy use.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + +
      +
      + +
      +
      + + + + + + + + + +
      + + +
      + + + +
      + + + + + + + + +
      + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/2.html b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/2.html new file mode 100644 index 00000000..f6b9cac0 --- /dev/null +++ b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/2.html @@ -0,0 +1,522 @@ + + + + + + + + + + + + + + + + + + + + + Basketball Stadium Data Metrics | AT&T + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + + +
      + + +
      + +
      +
      +
      + + +
      + + +
      + + +
      + + +
      + + November 10, 2015 + +
      +
      + + + + + + + + + + +
      + + + +
      + +
      +

      Pro Basketball Home Opener Metrics

      +
      +
      + + +
      + generic_basketball_court_946x432gif.gif +
      + +
      +
      + + + + +
      +
      +
      + + +
      +
      + +
      +

      In the offseason, our network team fine-tuned its game plan to give our customers a great experience at each arena – and we delivered in the clutch. This season we are using a venue-specific Distributed Antenna System (DAS) at 24 different pro basketball arenas.

      +

      We saw some interesting traffic metrics across our network at the season openers from these 24 arenas:

      +
        +
      • Overall, more than 3.1TB of data was used across these arenas. This marked a 70% increase in traffic when comparing the data used during this season’s home opener to the average game during March of last season. 3.1TB is equal to more than 9M social media posts with photos
        • +
      +
        +
      • The greatest increase in mobile data usage was in Minnesota, where we saw a 390% increase in traffic when comparing the data used during this season’s home opener to the average home game in March of last season.
        • +
      +
        +
      • The highest mobile data usage occurred in Miami with more than 260GB of data used. 260GB is equal to more than 743K social media posts with photos.
        • +
      +
      +

      + +
      + + +
      +
      +
      + +
      +
      +
      + + +
      +
      + + +
      + + + + + + +
      +
      +
      + + + + + + + +
      +
      +
      +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      The Dead Must Die Is Coming to Tribeca Film Festival

      + +
      +

      Game of Thrones fans will have a chance to vanquish the army north of the wall at the Tribeca Hub Spring Studios in NY.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      + +
      +

      For the third consecutive year AT&T & Tribeca will bring one filmmaker’s story to life through AT&T: Presents Untold Stories.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      Elizabeth Banks, Mayim Bialik and Geena Davis Join 4th Annual AT&T SHAPE

      + +
      +

      AT&T announced multi-hyphenate actresses Elizabeth Banks as first headlining speaker of the Los Angeles based event.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + +
      +
      + +
      +
      + + + + + + + + + +
      + + +
      + + + +
      + + + + + + + + +
      + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/3.html b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/3.html new file mode 100644 index 00000000..90147427 --- /dev/null +++ b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/3.html @@ -0,0 +1,676 @@ + + + + + + + + + + + + + + + + + + + + + Introducing the AT&T Corporate Community Impact Award Winner, Sheila Meyers + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + +
      +
      + +
      +
      +

      AT&T Corporate Community Impact Award

      +
      +
      + + +
      + + +
      + +
      +
      +
      + + +
      + + +
      + + +
      + + +
      + + November 19, 2019 + +
      +
      + + + + + + + + + + +
      + + + +
      + +
      +

      AT&T Corporate Community Impact Award

      +
      +
      + + +
      +
      + + + + +
      + + +
      +
      + +
      +

      Spotlight on Sheila Meyers: donating life-saving blood

      +

      This year, we launched the Community Impact Award to recognize employees who make a positive impact in their world. We received hundreds of nominations and stories of employees who live our value to make a difference. Our employees voted for the story that resonated with them most. We’re pleased to introduce you to the AT&T Corporate Community Impact Award winner.

      + +
      + + +
      +
      +
      +
      + + +
      + + + + + +
      + + + + + + + + +
      +
      + +
      + + +
      +

      Sheila Meyers

      +

      AT&T Corporate

      +

      Associate Director – Financial Systems

      +

      Bedminster, NJ

      +

      +
      + +
      + + + + +
      + +
      +
      +
      + + +
      +
      + +
      +

      Sheila Meyers’ cousin was diagnosed with leukemia when Sheila was in high school. Throughout treatment, blood transfusions kept her cousin alive until the cancer went into remission. That’s when Sheila started donating blood.

      +

      During the birth of her first child, Shelia was the beneficiary of a plasma donation which gave doctors time to save Sheila's life and her ability to have more children. That’s when Shelia decided to do even more.

      +

      To make it easier for others to donate blood and help save lives like hers and her cousin’s, Shelia works with a team of volunteers and the New Jersey Blood Services to run blood drives at the AT&T office in Bedminster, NJ. In addition to setting up the drives, Shelia and her team of volunteers make the drives enjoyable and entertaining for their colleagues. They provide refreshments and baked goodies to help stabilize blood sugar and fluid levels after donating. The refreshments often follow a theme, for instance shark-related food in the summer and typical tailgate fare in the Fall. People will stop Sheila in the hallways to ask what food will be at the next drive. Some of Shelia’s colleagues also donate raffle prizes for their fellow donors, such as tickets to professional sports games. “Pint Clubs” track life-to-date donations with top donors belonging to the 100+ Pints Club. Friendly competitions develop as donors give more blood and move up in the clubs. Shelia is quick to recognize her colleagues and fellow volunteers who help make the drives successful. “It’s really created a sense of community and fun in the office,” Sheila remarked.

      +

      The team of volunteers hosts a blood drive every other month so people can donate the maximum amount of blood each year (healthy donors may safely donate every 56 days). Since blood is perishable, frequent donations help hospitals and other organizations maintain their supply for the unexpected – from natural disasters to everyday incidents that require transfusions.

      +

      Together, employees in the AT&T Bedminster office have donated about 600 to 700 units of blood annually, impacting around 2,000 families each year. “You never know – the one time you donate could be the one time it’s really needed, and you save someone’s life,” Sheila said. “I encourage everyone to consider donating blood and help families like mine.”

      + +
      + + +
      +
      +
      +
      + + +
      +
      + + + + + + + +
      +
      + + + +
      +
      +
      +
      + + + + + +
      + +
      + +
      +
      +
      +

      Bedminster, NJ Blood Drive Captains

      +
      +
      +       +
      + + + +
      +
      +
      +
      + + + + + + + +
      +
      +
      +
      + + + + + +
      + +
      + +
      +
      +
      +

      Sheila Meyers

      +
      +
      +       +
      + + + +
      +
      +
      +
      + + + + + + + +
      +
      +
      +
      + + + + + +
      + +
      + +
      +
      +
      +

      Some of the 100+ Pints Club Employees

      +
      +
      +       +
      + + + +
      +
      +
      +
      + + +
      + + + + + + + + +
      +
      +
      + +
      +
      +
      +
      +
      + + +
      +
      + + +
      + + + + + + +
      +
      +
      + + + + + + + +
      +
      +
      +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      Pitches with Purpose 2018 Winner MindRight

      + +
      +

      Learn how MindRight expands offerings to teens who want to talk with MindRight Health.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      Blue Star Families’ Spouseforce Program

      + +
      +

      AT&T continues to help serve the loved ones of those who serve the country by supporting Blue Star Families’ Spouseforce program.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      2019 Community Impact Award Winners

      + +
      +

      The AT&T Community Impact Award recognizes employees who make a positive impact in their world.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + +
      +
      + +
      +
      + + + + + + + + + +
      + + +
      + + + +
      + + + + + +
      + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/4.html b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/4.html new file mode 100644 index 00000000..11e72ab3 --- /dev/null +++ b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/4.html @@ -0,0 +1,542 @@ + + + + + + + + + + + + + + + + + + + + + Samsung Galaxy S® 4 mini: The First Smartphone with HD Voice from AT&T | AT&T + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + + +
      + + +
      + +
      +
      +
      + +
      + +
      + +
      + + +
      + + May 19, 2014 + +
      +
      + + + + + +
      + +
      + +
      +

      Samsung Galaxy S® 4 mini: The First Smartphone with HD Voice from AT&T

      +
      +
      + +
      + +
      + + +
      +
      + + + + +
      +
      +
      + + + + + +
      +like +0I found this story helpful + +
      +
      + + +
      +
      + +
      +

       

      +

      Just when you thought it couldn’t get any better, the Samsung Galaxy S® 4 mini with HD Voice from AT&T is now available, enabled by Voice Over LTE. Beginning May 23, 2014 you can visit our online store or an AT&T store location to purchase the Galaxy S 4 mini in Black Mist or Valentine Pink with zero down and no commitment. Customers can also get the premium smartphone at $14.24 on Next 18 or $18.50 on Next 12. The cost of the device is $49.99 with a two-year agreement.  

      +

      The Galaxy S 4 mini is the first device on AT&T’s most reliable 4G LTE Network to utilize HD Voice. HD Voice from AT&T provides crystal clear conversations, reduced background noise, and the ability to talk while surfing the Web – all on the nation’s most reliable 4G LTE network.

      +

      The Galaxy S 4 mini has some of the most popular software features from the Galaxy S 4 in a compact form factor to enrich the user experience. AT&T was the first to deliver Samsung’s Galaxy family of devices; and the introduction of HD Voice in collaboration with Samsung marks yet another key milestone in offering the best selection of Samsung Galaxy series in the U.S.

      +

      Read the complete consumer blog to learn more about the HD Voice for the Samsung Galaxy S® 4 mini.

      +

      For more information about the Samsung Galaxy S 4 mini visit: att.com/galaxys4mini. For more information about HD Voice from AT&T visit: att.com/hdvoice.

      + +
      + + +
      +
      +
      + +
      +
      +
      + + +
      +
      + +
      + + + +
      +
      +
      + + + + + + + +
      +
      +
      +
      + + + +
      +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + +

      Samsung Galaxy Z Flip Available Feb. 14

      + +
      +

      AT&T is proud to lead the way in innovative folding devices.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      +
      + + +
      +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + +

      Google Pixel 4, 4XL Available October 16

      + +
      +

      Starting October 16, AT&T customers can order the new Google Pixel 4 and Pixel 4XL for use on the Nation’s Best Wireless Network.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      +
      + + +
      +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + +

      New Samsung Product Coming to AT&T

      + +
      +

      Starting September 27th, customers can order the Samsung Galaxy Tab S5e for use on the Nation’s Best Wireless Network.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      +
      + +
      +
      + +
      +
      + + + + + + + + + +
      + + +
      + + + +
      + + + + + + + + +
      + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/5.html b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/5.html new file mode 100644 index 00000000..a836bf14 --- /dev/null +++ b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/5.html @@ -0,0 +1,503 @@ + + + + + + + + + + + + + + + + + + + + + AT&T Announces Intent to Carry LG G3 | AT&T + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + + +
      + + +
      + +
      +
      +
      + + +
      + + +
      + + +
      + + +
      + + Dallas, TX, May 27, 2014 + +
      +
      + + + + + + + + + + +
      + + + +
      + +
      +

      AT&T Announces Intent to Carry LG G3

      +
      +
      + + +
      +
      + + + + +
      +
      +
      +
      +
      + + +
      +
      + +
      +

      DALLAS, May 27, 2014 – AT&T1 today confirmed that it will offer the LG G3 this year.

      +

      The LG G3 is set to join the nation’s most reliable 4G LTE network2. “We are excited to add the LG G3 to AT&T’s leading device portfolio this year,” said Jeff Bradley, senior vice president – Devices, AT&T. “With its razor sharp 5.5 inch Quad HD display to its laser auto camera focus, the LG G3 is an innovative smartphone for consumers looking for the latest technologies all running on the network that covers more than 99 percent of all Americans3.”

      +

      For more information, visit att.com/g3.

      +

      1AT&T products and services are provided or offered by subsidiaries and affiliates of AT&T Inc. under the AT&T brand and not by AT&T Inc.

      +

      2Reliability based on 3rd party data regarding nationwide carriers’ 4G LTE. LTE is a trademark of ETSI. 4G LTE not available everywhere.

      +

      3Based on coverage in U.S. licensed/roaming areas. Compatible device required. Coverage not available everywhere.

      +

      About AT&T

      +

      AT&T Inc. (NYSE:T) is a premier communications holding company and one of the most honored companies in the world. Its subsidiaries and affiliates – AT&T operating companies – are the providers of AT&T services in the United States and internationally. With a powerful array of network resources that includes the nation’s most reliable 4G LTE network, AT&T is a leading provider of wireless, Wi-Fi, high speed Internet, voice and cloud-based services. A leader in mobile Internet, AT&T also offers the best wireless coverage worldwide of any U.S. carrier, offering the most wireless phones that work in the most countries.  It also offers advanced TV service with the AT&T U-verse® brand. The company’s suite of IP-based business communications services is one of the most advanced in the world.

      +

      Additional information about AT&T Inc. and the products and services provided by AT&T subsidiaries and affiliates is available at www.att.com/aboutus or follow our news on Twitter at @ATT, on Facebook at www.facebook.com/att and YouTube at www.youtube.com/att.

      +

      © 2014 AT&T Intellectual Property. All rights reserved. AT&T, the AT&T logo and all other marks contained herein are trademarks of AT&T Intellectual Property and/or AT&T affiliated companies. All other marks contained herein are the property of their respective owners.

      + +
      + + +
      +
      +
      + +
      + + +
      +
      + + +
      + + + + + + +
      +
      +
      + + + + + + + +
      +
      +
      +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      AT&T to Add Second Samsung Smartphone to Mobile 5G Portfolio in 2019

      + +
      +

      AT&T expands mobile 5G portfolio confirming 3 mobile 5G devices to date.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      AT&T to Offer More Mobile 5G Devices

      + +
      +

      We will grow our mobile 5G device lineup in the spring of 2019 with the launch of the first Samsung mobile 5g smartphone.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      Step Up Your Game and Get the Razer Phone 2 Only from AT&T on Nov. 16

      + +
      +

      Starting this Friday AT&T customers can get the Razer Phone 2 at select AT&T stores. Learn how you can get the new powerhouse gaming phone here.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + +
      +
      + +
      +
      + + + + + + + + + +
      + + +
      + + + +
      + + + + + + + + +
      + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/6.html b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/6.html new file mode 100644 index 00000000..2f41a7d4 --- /dev/null +++ b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/6.html @@ -0,0 +1,548 @@ + + + + + + + + + + + + + + + + + + + + + AT&T Selected to Help Federal Communications Commission Use Cloud Technology | AT&T + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + + +
      + + +
      + +
      +
      +
      + +
      + +
      + +
      + + +
      + + Vienna, Virginia, January 12, 2016 + +
      +
      + + + + + +
      + +
      + +
      +

      AT&T Selected to Help Federal Communications Commission Use Cloud Technology

      +
      +
      + + +
      +
      + + + + +
      +
      +
      +
      +
      + + +
      +
      + +
      +

      AT&T* Corp. will help the Federal Communications Commission (FCC) transform its business with cloud services.

      +

      Under the terms of a 5-year agreement, AT&T will provide the FCC a cost-effective IP solution that will support mobile and cloud-based applications. The service will link a number of offices and data centers and let the FCC change network and Internet needs on demand. Using AT&T’s services, the FCC can connect to diverse cloud service providers.

      +

      The solution is compliant with federal government security rules. That includes the Managed Trusted Internet Protocol Service (MTIPS) requirement for improved security. It is highly secure and reliable with built-in redundancy to keep business operations up and running.

      +

      Our set of strategic services can help the FCC modernize its technology and use cloud services in a highly secure manner,” said Mike Leff, vice president-Civilian, AT&T Government Solutions.

      +

      *AT&T products and services are provided or offered by subsidiaries and affiliates of AT&T Inc. under the AT&T brand and not by AT&T Inc.

      + +
      + + +
      +
      +
      +
      +
      +
      +

      About AT&T

      +

      AT&T Inc. (NYSE:T) helps millions around the globe connect with leading entertainment, mobile, high speed Internet and voice services. We’re the world’s largest provider of pay TV. We have TV customers in the U.S. and 11 Latin American countries. In the U.S., our wireless network has the nation’s most reliable 4G LTE. We offer the best global coverage of any U.S. wireless provider*. And we help businesses worldwide serve their customers better with our mobility and highly secure cloud solutions.

      +

      Additional information about AT&T products and services is available at http://about.att.com. Follow our news on Twitter at @ATT, on Facebook at http://www.facebook.com/att and YouTube at http://www.youtube.com/att.

      +

      © 2016 AT&T Intellectual Property. All rights reserved. AT&T, the Globe logo and other marks are trademarks and service marks of AT&T Intellectual Property and/or AT&T affiliated companies. All other marks contained herein are the property of their respective owners.

      +

      Reliability claim based on nationwide carriers’ 4G LTE. 4G LTE not available everywhere.

      +

      *Global coverage claim based on offering discounted voice and data roaming; LTE roaming; voice roaming; and world-capable smartphone and tablets in more countries than any other U.S. based carrier. International service required.  Coverage not available in all areas. Coverage may vary per country and be limited/restricted in some countries.

      +

      + +
      +
      +
      + + + +

      View more

      + + + + + + + + + + + + + + +
      + +
      + + +
      +
      + +
      + + + +
      +
      +
      + + + + + + + +
      +
      +
      +
      + + + +
      +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + +

      Tech Mahindra and AT&T Announce Multi-Year Agreement

      + +
      +

      This agreement will help to deliver flexibility across IT operations including movement to the cloud.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      +
      + + +
      +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + +

      AT&T and Dell Technologies Collaborate on Open Source Edge Computing and 5G Software

      + +
      +

      Automated, programmable and intelligent bare-metal cloud operations to speed new service delivery.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      +
      + + +
      +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + +

      AT&T and Microsoft Announce Strategic Alliance

      + +
      +

      AT&T and Microsoft are applying cloud, AI, and 5G technology to improve how people live and work.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      +
      + +
      +
      + +
      +
      + + + + + + + + + +
      + + +
      + + + +
      + + + + + + + + +
      + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/7.html b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/7.html new file mode 100644 index 00000000..7ed84707 --- /dev/null +++ b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/7.html @@ -0,0 +1,568 @@ + + + + + + + + + + + + + + + + + + + + + Boston Flagship Store | AT&T + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + + +
      + + +
      + +
      +
      +
      + +
      + +
      + +
      + + +
      + + April 17, 2015 + +
      +
      + + + + + +
      + +
      + +
      +

      Boston's Store of the Future

      +
      +
      + +
      + +
      + + +
      +
      + + + + +
      +
      +
      + + + + + + + +
      +
      + + +
      +
      + +
      +
      + + +
      +
      + +
      +

      The new 9,000 square-foot AT&T store in Boston is one of our largest retail stores in the country. The new Boylston Street store is much like our award-winning flagship store on Michigan Avenue in Chicago. You’ll enjoy a bright and open café-style shopping environment. You can even "try before you buy.” Whether you’re looking for smartphones, tablets, fitness accessories or home automation, it’s easy for you to get the feel for all the merchandise. To learn more about this exciting new store opening, check out the news release.

      +

      + +
      + + +
      +
      +
      + +
      +
      +
      + + +
      +
      + +
      + + + +
      +
      +
      + + + + + + + +
      +
      +
      +
      + + + +
      +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + +

      Samsung Galaxy Z Flip Available Feb. 14

      + +
      +

      AT&T is proud to lead the way in innovative folding devices.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      +
      + + +
      +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + +

      AT&T Boosts Miami Area Coverage for Football’s Big Game

      + +
      +

      AT&T invests more than $85 million to improve wireless capacity and coverage for a better customer experience.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      +
      + + +
      +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + +

      Laying the Foundation for 5G and Public Safety

      + +
      +

      We’re investing in our network by enhancing reliability, coverage, and speed by building out our LTE network and laying the foundation for 5G.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      +
      + +
      +
      + +
      +
      + + + + + + + + + +
      + + +
      + + + +
      + + + + + + + + +
      + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/8.html b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/8.html new file mode 100644 index 00000000..f4878ec1 --- /dev/null +++ b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/8.html @@ -0,0 +1,563 @@ + + + + + + + + + + + + + + + + + + + + + "If You Love Cars, You Want His Job | AT&T" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + + +
      + + +
      + +
      +
      +
      + +
      + +
      + +
      + + +
      + + March 25, 2015 + +
      +
      + + + + + +
      + +
      + +
      +

      If You Love Cars, You Want His Job

      +
      +
      + +
      + +
      + + +
      +
      + + + + +
      +
      +
      + + +
      +
      + +
      +

      Brian Greaves grew up in Detroit's car culture. He went to work for a local automaker.

      +

      Then, four years ago, we lured him away – from his hometown, but not from the cars he loves.

      +

      Brian leads all new products developed on the Drive Platform and manages our AT&T Drive Studio in Atlanta.

      +

      Brian's team focuses on research and development of new technologies and services that make connected-car driving safer, more secure and more convenient for passengers.

      +

      Today, AT&T provides technology to leading automakers such as GM, BMW, Ford, Nissan, Tesla, Audi, Volvo and Subaru. This year, nearly half of the newly connected U.S. passenger vehicles will be added to our network. In the last quarter of 2014, we added 800,000 cars.

      +

      Brian talked recently about his lifelong love of cars, moving across the country – in his car – and the sci-fi-like potential of connected cars.

      +

      +

      Some employees say they have been waiting for connected cars since "The Jetsons" was on television. Do you see excitement like this every day?

      +

      Yes, but we have just begun to scratch the surface of what can be done behind the wheel of a connected car. Customers in connected cars today have an experience similar to being on plain cellphones. With the recent growth in data-rich integrated services, customers will soon have an experience more like being on smartphones.

      +

      What will connected cars be like in 2020?

      +

      I think we will see some forms of semi-autonomous vehicles. So you'll be able to get on the highway and just relax while your car responds to the environment and its surroundings. Technologically, we are not that far from fully autonomous vehicles, where cars will be able to drive themselves. Imagine when you will have the ability to send your car to the grocery store or dry cleaners to pick up everything.

      +

      Vehicles also will be able speak with one another to avoid collisions and adjust to traffic patterns. If another car is coming around the corner, your vehicle will automatically brake for you to avoid accidents. The technology is there and will be rolling out shortly.

      +

      We've heard concerns about privacy issues and cars being hacked. Should we be worried?

      +

      Right now, there is more concern out there than there probably should be. We have a whole team working with all vendor partners to make sure we protect both the security and privacy of our customers behind the wheel.

      +

      AT&T, as well as the automotive manufacturers, takes the privacy concerns as top-of-mind in everything we do. It is no different from how we protect the privacy of our smartphone users.

      +

      A blog writer recently took a six-week spin in a 2015 Audi A3, the first 4G LTE car in the U.S. market. It was connected to AT&T's network. The blogger concluded that a connected car really is a smarter car. Is that true?

      +

      A connected car absolutely is a smarter car. It's no different than a laptop that is connected or not connected. You can get so much more information about your drive, or your commute, that makes it much more safe, convenient, relevant – and better overall for the customer.

      +

      You're from Detroit. Were you born with a love of cars?

      +

      You kind of have to be. There are so many things in and around Detroit that are centered on cars. And most people have family members who work for the Big Three. I was born into a Ford family. My grandfather retired from Ford Motor Company after about 40 years.

      +

      I grew up loving cars and all kinds of gadgets. Now I love my job. I love coming in to work every day and working with an all-star team of passionate, dedicated individuals who have a similar vision and interest.

      +

      +

      How did you get into the car business?

      +

      Connected cars are part of the Internet of Things, a major trend powering our journey to Vision 2020. Our goal is to deliver a future transformed by our high-speed, mobile and video-centric network.

      +

      I actually began working as a college intern at the largest telematics company out of Detroit. ("Telematics" is an industry term for the transfer of vehicle data to and from the vehicle. The modern term is the connected car.) And it was a heck of an opportunity. Following graduation, they asked me back in a full-time role.

      +

      What kind of work did you do in Detroit?

      +

      I supported the launches of new products, everything from turn-by-turn navigation to newly launched safety and security services like stolen vehicle slowdown. If the car is stolen and the police are behind the car, we are able to basically kill the throttle so that the vehicle comes to a halt and law enforcement can arrest the thief.

      +

      How did you find your job at AT&T?

      +

      I was looking and they were looking. The stars seemed to align. I liked Detroit and I liked my job, but it was a really gray time in the automotive industry.

      +

      When I met with Cameron Coursey, now AT&T's vice president of Internet of Things, he said, "Our leadership is 100 percent behind this. We will do whatever it takes to succeed."

      +

      He convinced me to come over. I kind of took a leap of faith. It was an opportunity to do something different. And make a difference. So I packed up my car and headed down South.

      +

      I was happy to stay in Detroit. I love the people, the blue-collar attitude, and am a huge Lions, Pistons and, of course, Red Wings fan. Those are my teams, it's my town and where my roots are.

      +

      But I am starting to love Georgia – and, of course, the weather. Also, my fiancé, who's from Virginia, just completed her medical residency. She joined an ob-gyn practice in Atlanta where she is very happy, so it looks like we will be staying here.

      +

      What was your first impression of AT&T?

      +

      I came into the room and they said, "We are going to show you the automotive team" and there were five people. Tops. I was like: "Where is the rest of the team?" But that was it.

      +

      Connected cars represent a billion-dollar business opportunity for AT&T. And it's already paying off.

      +

      "We posted 1.9 million total wireless net adds, led by gains in tablets and connected cars," AT&T Chairman and CEO Randall Stephenson wrote in last year's fourth-quarter letter.

      +

      When I started, Emerging Devices focused on tablets, gaming devices and e-readers. Now we have kind of pivoted and most of Emerging Devices – which is now called Internet of Things – is focused on connected cars. We have to be up to about 50 or 60 people working on automotive. That's collectively across business development and product development.

      +

      What did you do first?

      +

      I worked on this huge, 100-page deck. I basically started writing everything you could do in the telematics/connected car space. We pulled from this to actually go in and talk to manufacturers.

      +

      Did you help create the Drive Studio, AT&T's new connected car headquarters?

      +

      I helped with the design, layout, location, everything down to the color of the chairs. My team was also important in understanding the tools we needed. For instance, if you run a car in one of the bays, you need to make sure a vehicle exhaust removal system is hooked up to the vehicle's exhaust to pull the carbon monoxide out of there.

      +

      How often do you have visitors?

      +

      We opened Jan. 16 (in 2014) and we had over 3,300 visitors and just over 500 events. That includes various car manufacturers and tier-one suppliers, as well as community outreach to the Boy Scouts of America, for example.

      +

      What is the most interesting thing in the works at the Drive Studio now?

      +

      We recently built a huge wall that replicates a Digital Life home. And we tied that to a "connected car" – which is basically a fake car that you can sit in. Through this application, you can say: "turn my home lights on" and "open my garage door" and you can literally look to the side and see the garage door open and the lights turn on.

      +

      We've also recently built multimode navigation. So, if you were planning to meet up at a tailgate event, you could navigate to the university and park in some big giant parking deck three blocks down from your friend. Then, when you get out of the car, the navigation will then transfer over to a wearable device. You will be able to walk right to the tailgate and meet exactly where your friends are.

      +

      What's in this for the auto manufacturers?

      +

      Cars have become a commodity where quality has improved greatly over the last 10 years. So if I am an automaker, how do I differentiate and get new customers to buy my car, instead of the other guys'? If you watch any commercial on TV today, their focus is around infotainment, connected-car services – and fuel economy.

      +

      Will you buy your next car based on its connected car abilities?

      +

      It will play a huge role in my purchasing decision. There are obviously other things that have to be taken into account: style, safety, fuel efficiency. I might have a family by then. My boss jokes that my next car is going to be a minivan.

      +

      Are you interested in all kinds of transportation?

      +

      Yes, things that go fast – sports cars, fast boats. I have a performance boat that I have out on Lake Lanier in Georgia and I still have snowmobiles up in Michigan that I hit the trails with. I love working on and enjoying all kinds of vehicles.

      +

      Since cars have become your work, do you still enjoy driving?

      +

      Yes, I love driving. I'm passionate about my car. Heck, I even like washing my car.

      +

      Learn more about AT&T and Connected Car in our 2014 Annual Report. +

      +

       

      + +
      + + +
      +
      +
      + +
      +
      +
      + + +
      +
      + +
      + + + +
      +
      +
      + + + + + + + +
      +
      +
      +
      + + + +
      +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + +

      AT&T Boosts Miami Area Coverage for Football’s Big Game

      + +
      +

      AT&T invests more than $85 million to improve wireless capacity and coverage for a better customer experience.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      +
      + + +
      +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + +

      Laying the Foundation for 5G and Public Safety

      + +
      +

      We’re investing in our network by enhancing reliability, coverage, and speed by building out our LTE network and laying the foundation for 5G.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      +
      + + +
      +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + +

      Google Pixel 4, 4XL Available October 16

      + +
      +

      Starting October 16, AT&T customers can order the new Google Pixel 4 and Pixel 4XL for use on the Nation’s Best Wireless Network.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      +
      + +
      +
      + +
      +
      + + + + + + + + + +
      + + +
      + + + +
      + + + + + + + + +
      + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/9.html b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/9.html new file mode 100644 index 00000000..dec83e5e --- /dev/null +++ b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/test_typical_html_data/9.html @@ -0,0 +1,626 @@ + + + + + + + + + + + + + + + + + + + + + Reese Witherspoon’s Spotlight on Female Creators + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + + +
      + + +
      + +
      +
      +
      + + +
      + + +
      + + +
      + + +
      + + EL SEGUNDO, July 10, 2018 + +
      +
      + + + + + + + + + + +
      + + + +
      + +
      +

      AT&T and Reese Witherspoon’s Hello Sunshine Announce Launch of VOD Channel Spotlighting Female Creators and Their Stories

      +
      +
      + + +
      +
      + + + + +
      +
      +
      +
      +
      + + +
      +
      + + +
      +
      + 768x475_story_level_comms_hello_sunshine.jpg +
      + + +
      + + + +
      +
      +
      +
      + + +
      +
      + +
      +

      The Hello Sunshine VOD Channel to Feature Witherspoon’s First-Ever Unscripted Series “Shine On with Reese” Premiering July 17

      +

      +

      Comedic Home Design Series “Master the Mess” From The Home Edit Arrives This September on DIRECTV, DIRECTV NOW and U-Verse

      +

      +

      AT&T Hello Sunshine Filmmaker Lab Running July 30–Aug. 6 in Los Angeles

      +

      +

      AT&T* and Hello Sunshine is debuting 2 new female-driven series on DIRECTV, DIRECTV NOW and U-Verse with the launch of the Hello Sunshine Video On Demand (VOD) channel. Hello Sunshine produces engaging and distinctive new narratives with women at the center.

      +

      +

      The VOD channel will premiere 2 female-driven original series:

      +
        +
      • Witherspoon’s first unscripted series, “Shine On with Reese,” premiering July 17.
        • +
      • The Home Edit’s “Master the Mess,” premiering Sept. 4.
        • +
      +

      Viewers can also watch curated original content including short-form and library film titles. Go to att.com/hellosunshine for a sneak peek at what’s to come.

      +

      +

      Hello Sunshine was launched in November 2016 by Reese Witherspoon in partnership with The Chernin Group and AT&T’s Otter Media portfolio.

      +

      +

      “AT&T was one of our first partners in the quest to bring female-centric storytelling to the forefront and we are especially proud of how these original series push that mission ahead,” said Sarah Harden, CEO, Hello Sunshine. “We’re looking forward to working with AT&T to develop more opportunities for women to see themselves and their stories reflected in entertainment.”

      +

      +

      To watch, search “Hello Sunshine” on DIRECTV NOW, go to channel 1112 on DIRECTV or channel 1530 on U-verse. New DIRECTV NOW subscribers can get their first 3 months for $10 a month with the promo code SUNSHINE3 – go to att.com/hellosunshine to learn more and sign up.1

      +

      +

      “Providing an empowering platform for diverse voices is a priority for us,” said Valerie Vargas, senior vice president – Advertising and Creative Services, AT&T. “Our work with Hello Sunshine gives stories about women the spotlight they deserve, which inspires our viewers and drives authentic conversation.”

      +

      +

      “Shine On with Reese” – debuts July 17

      +

      Witherspoon’s first unscripted series celebrates the stories of extraordinary women who have created their own unique paths to success. Watch as she uncovers what inspires, motivates and gives joy to each of these trailblazers, and hear their perspectives on ambition, work, family and hopes for the future.

      +

      +

      “I am thrilled to have such like-minded, creative collaborators at AT&T to help Hello Sunshine pursue our mission of elevating and showcasing the voices of women that we have such admiration and respect for,” said Witherspoon. “I am so excited for the world to experience the stories from our partnership with AT&T, which are set to be equal parts entertaining, inspiring, thought-provoking and unabashedly real."

      +

      +

      Viewers can look forward to weekly episodes with:

      +
        +
      • Dolly Parton (singer/songwriter)
        • +
      • Ava DuVernay (filmmaker/producer)
        • +
      • Pink (singer/songwriter/performer)
        • +
      • Cleo Wade (poet) and Elaine Welteroth (journalist)
        • +
      • Candace Nelson (Sprinkles founder) and Sara Blakely (Spanx founder)
        • +
      • Glennon Doyle (author/activist) and Abby Wambach (athlete/author)
        • +
      • America Ferrara (actress/activist)
        • +
      • Kacey Musgraves (singer/songwriter)
        • +
      • Simone Askew (First Captain, USMA at West Point) and cadets
        • +
      +

      +

      “’Shine On with Reese’ is about experiencing with Reese the stories of exceptional artists, entrepreneurs and leaders who happen to be women, and reveals how their personal journeys are unique yet relatable,” said Charlotte Koh, head of Digital Media and Programming, Hello Sunshine. “It epitomizes our goal of telling unexpected stories from interesting women’s perspectives.”

      +

      +

      “Shine On with Reese” will premiere on DIRECTV, DIRECTV NOW and U-Verse via the Hello Sunshine VOD channel Tuesday, July 17. Viewers can go to att.com/hellosunshine for a sneak peek at the first episode.

      +

      +

      “Master the Mess” series from The Home Edit – debuts Sept. 4

      +

      Decluttering gurus Clea Shearer and Joanna Teplin are The Home Edit, a Nashville-based business that blends functionality with their signature clean aesthetic. In a series that blends comedy with organization, we’ll see The Home Edit tackle the pantries, closets, bathrooms and laundry rooms of families who are about to experience a transformation in their lives. Shearer and Teplin are intelligent, funny and charmingly self-deprecating problem-solvers who make life better for each client in surprising ways. For more information on The Home Edit, go to www.thehomeedit.com, Twitter and Instagram.

      +

      +

      AT&T Hello Sunshine Filmmaker Lab – July 30-Aug. 6

      +

      AT&T and Hello Sunshine collaborate with Dreaming Tree Foundation (Fresh Films LLC) to create an unforgettable opportunity for the next generation of female storytellers to attend a summer filmmaker lab in Los Angeles. The custom program will host 20 girls for an eight-day immersive program where they will work with industry experts – and Reese herself – to create content for the Hello Sunshine VOD channel.

      +

      +

      1After 3 months, service renews at then-prevailing rate (currently min. $35/mo.) unless you cancel. Price for "Live a Little" package after discount. New subscribers only. Limited time offer. Voids 7-day trial. Restrictions apply.

      +

      + +
      + + +
      +
      +
      +
      +
      +
      +

      About Hello Sunshine

      +

      Hello Sunshine is a media brand and digital content company dedicated to female authorship and storytelling across all platforms. Hello Sunshine is producing feature films, scripted and unscripted television, audio storytelling and social series under the Hello Sunshine brand, all anchored by a singular mission: to change the narrative for women. Hello Sunshine is also the home for Reese’s Book Club, fast-growing in reach and influence. Hello Sunshine has also partnered with Audible to highlight the female-centered books from Reese’s Book Club, as well as to produce upcoming original audio content. Some of the film and TV projects that have already been announced include the limited series from Reese Witherspoon and Kerry Washington, based on Celeste Ng’s 2017 bestseller Little Fires Everywhere, as a Hulu Original Series; a 10-episode comedy series for Apple that was created by Colleen McGuinness (30 Rock) and inspired by Curtis Sittenfeld's short-story collection You Think It, I'll Say It; Are You Sleeping, executive produced by and starring Octavia Spencer, based on the true-crime novel by Kathleen Barber, created and written by Nichelle Tramble Spellman (The Good Wife, Justified) for Apple; and an Untitled Morning Show Project produced by and starring Jennifer Aniston and Reese Witherspoon, which was given a two-season, straight-to-series order by Apple with Kerry Ehrin (Bates Motel) serving as showrunner. Hello Sunshine launched its first original podcast series, How It Is, featuring powerful, personal stories told by a diverse group of high-profile women and hosted by actor and author Diane Guerrero (Orange Is The New Black, Jane the Virgin) and a series on Facebook Watch, Meet My Mom, showcasing honest and entertaining chats between high-profile celebrities and their moms.

      +

      +

      *About AT&T Communications

      +

      We help family, friends and neighbors connect in meaningful ways every day. From the first phone call 140+ years ago to mobile video streaming, we innovate to improve lives. We have the nation’s largest and most reliable network and the nation’s best network for video streaming.*** We’re building FirstNet just for first responders and creating next-generation mobile 5G. With DIRECTV and DIRECTV NOW, we deliver entertainment people love to talk about. Our smart, highly secure solutions serve over 3 million global businesses – nearly all of the Fortune 1000. And worldwide, our spirit of service drives employees to give back to their communities.

      +

      +

      AT&T Communications is part of AT&T Inc. (NYSE:T). Learn more at att.com/CommunicationsNews.

      +

      +

      AT&T products and services are provided or offered by subsidiaries and affiliates of AT&T Inc. under the AT&T brand and not by AT&T Inc. Additional information about AT&T products and services is available at about.att.com. Follow our news on Twitter at @ATT, on Facebook at facebook.com/att and on YouTube at youtube.com/att.

      +

      +

      © 2018 AT&T Intellectual Property. All rights reserved. AT&T, the Globe logo and other marks are trademarks and service marks of AT&T Intellectual Property and/or AT&T affiliated companies. All other marks contained herein are the property of their respective owners.

      +

      +

      ***Coverage not available everywhere. Based on overall coverage in U.S. licensed/roaming areas. Reliability based on voice and data performance from independent 3rd party data.

      +

      + +
      +
      +
      + + + +

      View more

      + + + + + + + + + + + + + + +
      + +
      + + +
      +
      + + +
      + + + + + + +
      +
      +
      + + + + + + + +
      +
      +
      +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      AT&T Debuts Two Slimmer Live TV Packages with HBO Included

      + +
      +

      AT&T’s DIRECTV NOW debuts two new live TV packages that include premium access to HBO.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      AT&T Delivers Crime Thriller ‘Outlaws’ from Australian Filmmaker Stephen McCallum

      + +
      +

      AT&T is giving DIRECTV customers exclusive access to the crime thriller, “Outlaws.” Learn how you can watch it on DIRECTV CINEMA.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + + +
      + +
      +
      +
      + +
      + + + Get Faster Streaming + + +
      + +
      +
      +
      + + + +

      Cheddar To Roll Out Across AT&T Video Platforms in December 2018

      + +
      +

      The leading post-cable networks company, Cheddar, is beginning rollout across multiple AT&T video platforms including DIRECTV, DIRECTV NOW and U-verse TV starting this December.

      +
      + +
      + Read More +
      + +
      +
      +
      +
      + +
      + + +
      +
      + +
      +
      + + + + + + + + + +
      + + +
      + + + +
      + + + + + + + + +
      + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py b/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py index 08e46eae..75b883a1 100644 --- a/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py +++ b/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py @@ -18,7 +18,7 @@ def test_tag_simplifier(self): pre_data_result = HtmlTagSimplifierParser({}).parse(pre_data) simplifier_raw_html = pre_data_result.get(PreDataJsonKey.TYPICAL_SIMPLIFIED_HTML, '') _item_id_count = simplifier_raw_html.count('_item_id') - self.assertEqual(_item_id_count, 34) + self.assertEqual(_item_id_count, 32) if __name__ == '__main__': diff --git a/tests/llm_web_kit/main_html_parser/parser/test_typical_html_selector.py b/tests/llm_web_kit/main_html_parser/parser/test_typical_html_selector.py new file mode 100644 index 00000000..aa33eb63 --- /dev/null +++ b/tests/llm_web_kit/main_html_parser/parser/test_typical_html_selector.py @@ -0,0 +1,30 @@ +import unittest +from pathlib import Path + +from llm_web_kit.input.pre_data_json import PreDataJson, PreDataJsonKey +from llm_web_kit.main_html_parser.parser.typical_html_selector import \ + TypicalHtmlSelectorParser + +base_dir = Path(__file__).resolve().parent + + +class MyTestCase(unittest.TestCase): + def test_typical_html_selector(self): + file_dir = base_dir / 'assets/test_html_data/test_typical_html_data' + html_files = [f'{file_dir}/{i}.html' for i in range(10)] + html_content = [] + for file_path in html_files: + with open(file_path, 'r', encoding='utf-8') as f: + html_content.append({ + 'track_id': Path(file_path).name, + 'html': f.read() + }) + data_dict = {PreDataJsonKey.LAYOUT_FILE_LIST: html_content} + pre_data = PreDataJson(data_dict) + pre_data_result = TypicalHtmlSelectorParser({}).parse(pre_data) + typical_html = pre_data_result.get(PreDataJsonKey.TYPICAL_RAW_HTML, '') + self.assertEqual(typical_html['track_id'], '9.html') + + +if __name__ == '__main__': + unittest.main() From 1d944afdf4d14562fc23596b956333b1eaa3b3de Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Sun, 27 Apr 2025 15:04:10 +0800 Subject: [PATCH 07/13] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main_html_parser/parser/tag_simplifier.py | 2 +- .../parser/assets/test_html_data/1.html | 972 +++++++++++++ .../assets/test_html_data/normal_dl.html | 1238 +++++++++++++++++ .../assets/test_html_data/normal_table.html | 235 ++++ .../test_html_data/special_table_1.html | 479 +++++++ .../parser/test_tag_simplifier.py | 44 + 6 files changed, 2969 insertions(+), 1 deletion(-) create mode 100644 tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/1.html create mode 100644 tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/normal_dl.html create mode 100644 tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/normal_table.html create mode 100644 tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/special_table_1.html diff --git a/llm_web_kit/main_html_parser/parser/tag_simplifier.py b/llm_web_kit/main_html_parser/parser/tag_simplifier.py index ffdf0671..28ceac0b 100644 --- a/llm_web_kit/main_html_parser/parser/tag_simplifier.py +++ b/llm_web_kit/main_html_parser/parser/tag_simplifier.py @@ -23,7 +23,7 @@ def parse(self, pre_data: PreDataJson) -> PreDataJson: # 执行HTML标签简化逻辑 try: - simplified_html, original_html, _ = simplify_html(typical_raw_html, is_xpath=False) + simplified_html, original_html, _ = simplify_html(typical_raw_html) except TagSimplifiedParserException as e1: raise e1 except Exception as e2: diff --git a/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/1.html b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/1.html new file mode 100644 index 00000000..22eb01d9 --- /dev/null +++ b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/1.html @@ -0,0 +1,972 @@ + + + + + + + + + + Klapka wlewu paliwa Ford Transit Custom 2012 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + +
      + +
        + +
      • Zaloguj się
      +
      + +
      +
      + +
      + +

      SKLEP INTERNETOWY Z CZĘŚCIAMI DO FORDA TRANSITA

      +
      + +
      + + + + + + + +
      +
      +
      + + + +
      +
      +
      + + + + + + + + + + + +
      +
      +
      +
      +
      +Klapka wlewu paliwa Ford Transit Custom 2012
      + +
      +
      + +
      +
      +

      + + Klapka wlewu paliwa Ford Transit Custom 2012 + +

      +
      +
      Ocena4.9
      + + +
      +

      + +

      +
      + + + 123zł + + + +
      + + +
      + +
      + + + + + +
      + +
      +
      + + +
      +
      + + +
      +
      + +
      +
      +
      + + + +
      +
      + + + +
      +
      +
      + + + +
      +
      + + +

      + +
      +
      +
      + +
      + +
      +
      + +
      +
      + +
      +
      +
      Produkt dostępny
      Ten produkt kupiono 93 razy.
      Cena wysyłki : 16,00złCzas wysyłki : 1 dzieńZapytaj o produkt
      + + + + + +
      +
      +
      + +
      + +
      +
      +
      + +

      Klapka wlewu paliwa Ford Transit Custom 2012

      + + Klapka wlewu paliwa pasuje do Forda Transita Customa z lat 2012-2016. Osłona wlewu paliwa jest umieszczona przy lewych drzwiach przednich. Klapka wlewu paliwa do Ford Transit Custom jest wyposażona w niezbędne zaczepy oraz kształt tego elementu są identyczne jak oryginalny produkt.

      Pasuje do:
      Ford Transit Custom MK1 2012-2016 +
      Ford Tourneo Custom MK1 2012-2016

      NR CZĘŚCI:

      BK21-V405A02-CDXWAA , BK21V405A02CDXWAA +

      BK21-V405A02-CBXWAA , BK21V405A02CBXWAA +

      BK21-V405A02-CAXWAA , BK21V405A02CAXWAA


      FINIS:

      1837417 , 1771170 +

      1770054

      + + +       +
      + + Nowy produkt:
      Tak

      Skala trudności montażu (1-5):
      2 - zamontujesz tę część z użyciem podstawowych narzędzi

      Jakość produktu:
      część zamienna

      Producent części:

       +
      +
      +
      + + + +

      Opinie klientów którzy kupili produkt:

       
      +
      + + +  Grazyna +
      +
      +
      + WSZYSTKO OK.GORĄCO POLECAM
      + + + +
      +
      + +
      +
      + + +  Sporea +
      +
      +
      + este ca originalul.
      + + + +
      +
      + +
      +
      + + +  Sporea +
      +
      +
      + Este ca originalul,perfect.
      + + + +
      +
      + +
      +
      + + +  Jan   Data dodania: +
      +
      + +
      +
      +
      + +
      +

      Podobne produkty:

       +
      + + + + + + + + + +
      + Klamka drzwi Ford Transit 2014 Custom Connect przód oryginalna
      Klamka drzwi Ford Transit 2014 Custom Connect przód oryginalna
      + Klamka drzwi przód Ford Transit MK8 2014 Custom Connect
      Klamka drzwi przód Ford Transit MK8 2014 Custom Connect
      + Obudowa korka wlewu paliwa Ford Transit Custom 2012
      Obudowa korka wlewu paliwa Ford Transit Custom 2012
      + Zamek drzwi lewy przedni Ford Transit Custom 2012 LHD 8PIN
      Zamek drzwi lewy przedni Ford Transit Custom 2012 LHD 8PIN
      +
      + + + + +
      + +

      Klienci, którzy kupili ten produkt zakupili również:

      + + + +
      Szkło lusterka Ford Transit Custom prawe manualne
      + +
      Szkło lusterka Ford Transit Custom prawe manualne + +
      Obudowa korka wlewu paliwa Ford Transit Custom 2016 Ecoblue Trateo
      + +
      Obudowa korka wlewu paliwa Ford Transit Custom 2016 Ecoblue Trateo + +
      Kołpak osłona śrub przód podwójne koła Ford Transit 1992
      + +
      Kołpak osłona śrub przód podwójne koła Ford Transit 1992 + +
      Kołpak Ford Transit podwójne koła
      + +
      Kołpak Ford Transit podwójne koła + +
      + + +
      + + + + +
      +
      Tagi

      Klapka wlewu paliwa Transit

       
      Podaj słowo kluczowe
      Zaawansowane wyszukiwanie
      +
      Informacje
      Koszty Wysyłki
      Płatności
      Korzystanie z serwisu
      Kontakt
      +
      Transit Center in Europe
      +
      + + + + + +
      + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/normal_dl.html b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/normal_dl.html new file mode 100644 index 00000000..cbd81a5b --- /dev/null +++ b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/normal_dl.html @@ -0,0 +1,1238 @@ + + + + + + + + + + + + + + + + + + + + + Bits & Pieces: CAS(E) This Sketch #400 + + + + + + + + + + + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +

      Pages

      +
      + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      + +
      + +

      Tuesday, December 8, 2020

      + +
      + +
      +
      + + + + +

      +CAS(E) This Sketch #400 +

      +
      +
      +
      +
      +

      Wow!  Number 400.  What a milestone for CAS(E) This Sketch.  Congratulations!  I used this week's sketch for another Christmas card.

      The images are from Altenew's Holiday Bow and the gray glitter strips are from a paper pack by DCWV.  I used a darker gray from the same pack for the branches.  The sentiment is from HLS.

      Thanks for dropping by.  I love it when you do.




      +
      +
      +
      +
      + +Posted by + + + +at + + + + + + + + + + + + +
      + +
      +
      +
      + +Labels: +, +, + + +
      +
      + + +
      +
      +
      +
      + +

      4 comments:

      +
      +
      +
      + +
      + +
      +I Card Everyone +said... +
      +
      +

      +Bobby, Michelle may have used your card to inspire her Sketch!! It's perfection - and now I must remember to use that pad of glitter paper you gave me!
      =] +

      +
      +
      + + +December 9, 2020 at 7:26 AM + + + + + + + +
      +
      + +
      + +
      +Jeanne H +said... +
      +
      +

      +Another lovely card, Bobby. I love seeing your cards. Thanks. :) +

      +
      +
      + + +December 9, 2020 at 10:51 AM + + + + + + + +
      +
      + +
      + +
      +Lisa Elton +said... +
      +
      +

      +Oh goodness, what a great use of the sketch, so pretty! +

      +
      +
      + + +December 9, 2020 at 9:17 PM + + + + + + + +
      +
      + +
      + +
      +TK +said... +
      +
      +

      +Very pretty, love the gray stripes! +

      +
      +
      + + +December 9, 2020 at 9:50 PM + + + + + + + +
      +
      +
      +

      +Post a Comment +

      +
      +
      + +
      + +
      +
      + +Newer Post + + +Older Post + +Home +
      +
      +
      +
      +Subscribe to: +Post Comments (Atom) +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      + +
      +
      +
      +
      + +
      + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + + + + + + + + +
      + + + +
      + + +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      +
      + + + + + + \ No newline at end of file diff --git a/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/normal_table.html b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/normal_table.html new file mode 100644 index 00000000..24a95cca --- /dev/null +++ b/tests/llm_web_kit/main_html_parser/parser/assets/test_html_data/normal_table.html @@ -0,0 +1,235 @@ + + + + + + + + + 🇷🇴 | Show hub - •NICE DOWNLOAD• DcHub DC++ Dchublist NMDC and ADCs хабов Huburi Хаблист + + + + + + + + + + + + + + + + + + + + + + + + +
      Hubs +
        +
      + +

      •NICE DOWNLOAD• DcHub

      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      Client + https://dchublists.com/client-ro/FlylinkDC_x64.exe +
      StatusOffline | ID: 691
      URL + https://dchublists.com/nice +
      Address +NMDC | dchub://nice.ddnsking.eu:4411 +
      ASN + OVH SAS +
      Failover + Not available +
      Name•NICE DOWNLOAD• DcHub
      Topic + WELCOME ! WELCOME !! WELCOME !! /FAV TO MAINCHAT !! +
      Description + •NICE DOWNLOAD• Download Constanta R3volutionSTAR Romania Bucuresti Shooter Rar Apps mpeg Punk Download avi Covers Games Pictures Comics Rap International Windows HipHop porn Trance TT Chat images Techno Mp3 Music Prison Break Movies Network +
      Category + Not available +
      Software + Verlihub 1.2.0.21 +
      Owner + Self +
      Location + RO Romania +
      Users + 0 | 271 +
      Clones2
      Share + 0 B | 83.10 TB +
      User limit6000
      Share limit0 B
      Slot limit0
      Hub limit0
      Reliability98.23%
      Checked + 2024-12-05 15:11:23 | 2018-02-04 +
      Votes + +0 | -0 | 0 +
      Website + Not available +
      Email + Not available +
      +
      +

      Online users

      + + Requested hub is not online.

      + +

      Comments

      + There are no comments for this hub, you can write one here. +
      +
      + + + + + + Книги для детей купить в Москве | Аделанта + + + + + + + + + + + + + + + + + + + + + +
      + + + + + + + + + + + + +
      + + + + + + + + +
      +
      + +       		+ + + Из книжного собрания
      + Александра Лугачева
      +
      + +
      + + +
      +
      + + + + + + + + + + + + + + + + + + + + + + + +
      ГлавнаяКаталог книгДревние книгиИстория древних книгСтаринные книгиАнтикварные книгиКупимДоставкаАрхив сделок    
      +
      +
      Путь:
      Главная / Старинные книги купить / Антикварные книги / Книги для детей
      + + + +
      + +
      +
      +
      +
      +
      +
      + + Корзина + 0 позиций +
      + + На сумму 0 руб. +
      +
      + +
      +
      +
      +
      +
      + + +
      +
      + + + + + +
      + Подснежник, стихотворения для детей и юношества +
      + Подснежник, стихотворения для детей и юношества + Подснежник, стихотворения для детей и юношества + Подснежник, стихотворения для детей и юношества +
      +
      +

      Вас могут заинтересовать:
      + • все книги автора Плещеев А.Н.
      + • ещё книги из раздела "Книги для детей"

      +
      + +
      + + + + +
      + + +
      7 900 РУБ.
      + + +
      +
      +
      + + +

      Подснежник, стихотворения для детей и юношества (1897 г.)

      + +
      + Автор:

      + + +

      П Р О Д А Н О

      + При согласии нового собственника книги можем предоставить его координаты для переговоров +

      + + + + + + + + Антикварная книга в твердом издательском коленкоровом переплете. На обложке и корешке тисненные золотом название книги и узорные орнаменты. На форзаце - дарственная надпись: "Оле Викторовой на память от крестного", датированная 1906 годом. Лисьи пятна и карандашные пометы на страницах. Значительные потертости корешка. Мраморный обрез страниц. На 31-й странице печать Комендатуры Выборгского района Революционной охраны города Петрограда.
      +
      Издательство:
      Типография Суворина А.С. (Санкт-Петербург)

      + Язык:
      русский

      + Сохранность книги: хорошая
      + Год издания: 1897
      + + Число страниц:
      119

      + Формат книги: 150 х 220 мм
      + Твердый переплет +

      +

      Плещеев Алексей Николаевич (1825-1893 гг.) - русский писатель, поэт, переводчик; литературный и театральный критик. В 1846 году первый же сборник стихов сделал Плещеева знаменитым в революционной молодежной среде; как участник кружка Петрашевского он был в 1849 году арестован и некоторое время спустя отправлен в ссылку, где провел на военной службе почти десять лет. По возвращении из ссылки Плещеев продолжил литературную деятельность; пройдя через годы бедности и лишений, он стал авторитетным литератором, критиком, издателем, а в конце жизни и меценатом. Многие произведения поэта (особенно - стихи для детей) стали хрестоматийными, считаются классикой. На стихи Плещеева известнейшими русскими композиторами написаны более ста романсов.
      +
      +Купив старинную книгу, изданную более 100 лет назад, вы станете обладателем антикварного издания, которое не подлежит вывозу за пределы Российской Федерации
      +
      + + + + +
      + + +
      7 900 РУБ.
      + +
      + +
      + + + + + + + + + + + + + + + + + + + +
      +

      Похожие товары

      + + +
      + + + + + + + + + + + + + + + + + + + + + + +
      + + Мученики Колизея. Исторический рассказ для детей + Антикварное издание
      +

      Мученики Колизея. Исторический рассказ для детей (1905 г.)

      + Автор: Тур Евгения

      + + + + + + + + + + Антикварная книга во владельческом составном переплете. Золотое тиснением по корешку. Потертости переплета. Следы реставрации двух страниц (изображения Колизея). На титульном листе старой книги библиотечный штамп. Следы загрязнения страниц. Лисьи пятна. Следы загрязнения страниц от перелистывания. Редкие следы залития. Мелкие надрывы и фрагментарные утраты по краям некоторых страниц (текст не задет). Книготорговые штампы и пометы на нахзаце. Реставрация бумажной полосой стр. 53 (текст незначительно задет). Издание шестое. Исторический рассказ для детей. Орфография и пунктуация приближены к современным нормам. Первая публикация произведения состоялась в 1890 году, "Мученики Колизея" тогда были напечатаны в Университетской типографии.
      +

      + Издательство: Типография Г. Лисснера и Д. Собко (Москва)

      + Язык: русский

      + Сохранность книги: хорошая

      + Год издания: 1905

      + Число страниц: 164

      + Формат книги: 145 х 215 мм

      + Твердый переплет + +

      + Цена: 11 650 руб. +      Купить       		+ + Детская энциклопедия, или Сокращение всех наук. С гравированными картинками, ч.1-4 + Антикварное издание
      +

      Детская энциклопедия, или Сокращение всех наук. С гравированными картинками, ч.1-4 (1835 г.)

      + Автор:

      + + + + + + + + + + Антикварная книга в цельнокожаном переплете эпохи. Представляет собой конволют из первых четырех частей 9-томного издания. Имеются потертости переплета, надрывы уголков; бледные разводы от воды по нижним полям. Из двадцати пяти гравюр, которые должных присутствовать в первых четырех частях "Детской энциклопедии...", в нашем экземпляре сохранились восемь гравированных картинок. Текст в антикварной книге дан параллельно на французском и русском языках. Содержание томов: Ч. 1: 52 стр., Ч. 2: 82 стр., 8 л. грав., Ч. 3: 72 стр., Ч. 4: 93 стр. Библиография старинной книги: Обольянинов № 763, Сопиков № 3582.

      + Издательство: Типография Семена Августа (Москва)

      + Язык: русский

      + Сохранность книги: хорошая

      + Год издания: 1835

      + Число страниц: 299

      + Формат книги: 135 х 225 мм

      + Твердый переплет + +

      + Цена: 61 900 руб. +      Купить       		+ + Прилежный ученик (новое издание). Le nouvel ecolier vertueux + Антикварное издание
      +

      Прилежный ученик (новое издание). Le nouvel ecolier vertueux (1821 г.)

      + Автор: Лемер Анри

      + + + + + + + + + + Антикварная книга карманного формата начала XIX века в цельном кожаном переплете, золотое и красное тиснение по корешку. Старинное издание отпечатано на бумаге верже. Мраморный узор по срезу страниц. С одной черно-белой гравюрой на авантитуле. Посмертное издание французского автора. По мнению современников Анри Лемера, "...книга для воспитания сердец юношества, призванная уберечь их от главных ошибок, совершаемых во время учебного процесса, в которой собраны моральные поучения от самых лучших авторов..."

      + Издательство: Ledentu Librarie (Париж)

      + Язык: французский

      + Сохранность книги: хорошая

      + Год издания: 1821

      + Число страниц: 258

      + Формат книги: 85 х 130 мм

      + Твердый переплет + +

      + Цена: 7 150 руб. +      Купить
      + + +
      + + + +

      +
      +
      +
      +
      + + + + + + + + + + + + + + + + + +
      Реставрация старых книгОценка старинных книгЭнциклопедия букинистаРусские писателиБиблиотека Ивана ГрозногоИстория русских книг    
      +
      +
      +
      +

      © 2024 Adelanta Group

      + + +
      +
      +

      + Интернет-магазин антикварных книг. Собрание старинных книг и антикварных изданий. История древних книг.
      + Свои вопросы по покупке или продаже старинных книг Вы можете задать по телефону: (495) 725-89-08 +

      +
      + +

      + +

      +
      +
      +
      + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py b/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py index 75b883a1..64524660 100644 --- a/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py +++ b/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py @@ -20,6 +20,50 @@ def test_tag_simplifier(self): _item_id_count = simplifier_raw_html.count('_item_id') self.assertEqual(_item_id_count, 32) + def test_tag_simplifier1(self): + file_path = base_dir / 'assets/test_html_data/normal_dl.html' + with open(file_path, 'r', encoding='utf-8') as file: + raw_html = file.read() + data_dict = {PreDataJsonKey.TYPICAL_RAW_HTML: raw_html} + pre_data = PreDataJson(data_dict) + pre_data_result = HtmlTagSimplifierParser({}).parse(pre_data) + simplifier_raw_html = pre_data_result.get(PreDataJsonKey.TYPICAL_SIMPLIFIED_HTML, '') + _item_id_count = simplifier_raw_html.count('_item_id') + self.assertEqual(_item_id_count, 18) + + def test_tag_simplifier2(self): + file_path = base_dir / 'assets/test_html_data/normal_table.html' + with open(file_path, 'r', encoding='utf-8') as file: + raw_html = file.read() + data_dict = {PreDataJsonKey.TYPICAL_RAW_HTML: raw_html} + pre_data = PreDataJson(data_dict) + pre_data_result = HtmlTagSimplifierParser({}).parse(pre_data) + simplifier_raw_html = pre_data_result.get(PreDataJsonKey.TYPICAL_SIMPLIFIED_HTML, '') + _item_id_count = simplifier_raw_html.count('_item_id') + self.assertEqual(_item_id_count, 30) + + def test_tag_simplifier3(self): + file_path = base_dir / 'assets/test_html_data/special_table_1.html' + with open(file_path, 'r', encoding='utf-8') as file: + raw_html = file.read() + data_dict = {PreDataJsonKey.TYPICAL_RAW_HTML: raw_html} + pre_data = PreDataJson(data_dict) + pre_data_result = HtmlTagSimplifierParser({}).parse(pre_data) + simplifier_raw_html = pre_data_result.get(PreDataJsonKey.TYPICAL_SIMPLIFIED_HTML, '') + _item_id_count = simplifier_raw_html.count('_item_id') + self.assertEqual(_item_id_count, 5) + + def test_tag_simplifier4(self): + file_path = base_dir / 'assets/test_html_data/1.html' + with open(file_path, 'r', encoding='utf-8') as file: + raw_html = file.read() + data_dict = {PreDataJsonKey.TYPICAL_RAW_HTML: raw_html} + pre_data = PreDataJson(data_dict) + pre_data_result = HtmlTagSimplifierParser({}).parse(pre_data) + simplifier_raw_html = pre_data_result.get(PreDataJsonKey.TYPICAL_SIMPLIFIED_HTML, '') + _item_id_count = simplifier_raw_html.count('_item_id') + self.assertEqual(_item_id_count, 37) + if __name__ == '__main__': unittest.main() From 4393ea4d5311bdc85aa437c354a405bde6640c53 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Sun, 27 Apr 2025 15:14:08 +0800 Subject: [PATCH 08/13] =?UTF-8?q?feat:=20=E5=8E=BB=E6=8E=89=E5=A4=9A?= =?UTF-8?q?=E4=BD=99=E5=86=85=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../simplify_html/simplify_html.py | 74 ------------------- 1 file changed, 74 deletions(-) diff --git a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py index b8024505..12b08940 100644 --- a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py +++ b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py @@ -323,38 +323,6 @@ def remove_tags(dom): parent.remove(node) -def remove_inline_tags_except_img_br(dom): - # 定义块级元素列表（不完全，可根据需要扩展） - - # 使用深度优先遍历DOM树 - for element in dom.iter(): - if element.tag not in inline_tags: - # 处理块元素中的子元素 - for child in list(element): # 使用list()创建副本，因为我们会修改原结构 - if child.tag in inline_tags and child.tag in ['br', 'img']: - # 移除行内元素，但保留其文本内容 - parent = child.getparent() - if child.text: - prev = child.getprevious() - if prev is not None: - prev.tail = (prev.tail or '') + child.text - else: - parent.text = (parent.text or '') + child.text - - # 处理子元素的子元素（如果有） - for subchild in child: - parent.insert(parent.index(child), subchild) - - if child.tail: - prev = child.getprevious() - if prev is not None: - prev.tail = (prev.tail or '') + child.tail - else: - parent.text = (parent.text or '') + child.tail - - parent.remove(child) - - def is_meaningful_content(element) -> bool: """严格判断元素是否包含有效内容.""" if element.text and element.text.strip(): @@ -383,48 +351,6 @@ def clean_attributes(element): clean_attributes(child) -def _remove_nested_inline_tags_only(element): - """只移除嵌套的行内标签，保留当前元素本身（即使是行内标签） 用于处理inline_elements和mixed类型的内容.""" - # 先处理子元素（深度优先） - for child in list(element.iterchildren()): - _remove_nested_inline_tags_only(child) - - # 如果子元素是需要移除的行内标签 - if child.tag in inline_tags and child.tag not in EXCLUDED_TAGS: - parent = child.getparent() - if parent is None: - continue - - # 转移内容到父元素 - text = child.text or '' - tail = child.tail or '' - grandchildren = list(child.iterchildren()) - - # 处理元素文本 - if text: - prev = child.getprevious() - if prev is not None: - prev.tail = (prev.tail or '') + text - else: - parent.text = (parent.text or '') + text - - # 转移子节点 - index = parent.index(child) - for grandchild in reversed(grandchildren): - parent.insert(index, grandchild) - - # 处理尾部文本 - if tail: - prev = child.getprevious() - if prev is not None: - prev.tail = (prev.tail or '') + tail - else: - parent.text = (parent.text or '') + tail - - # 移除当前子元素 - parent.remove(child) - - def remove_inline_tags(element): """递归移除所有指定的行内标签（包括嵌套情况），保留img和br标签 优化点：确保文本顺序正确，正确处理嵌套标签的文本转移.""" # 先处理子元素（深度优先） From 615678acd185f0b429d0beb9c0bad6c734b00fdf Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Fri, 9 May 2025 11:17:34 +0800 Subject: [PATCH 09/13] =?UTF-8?q?fix:=20=E4=BC=98=E5=8C=96=E7=B2=BE?= =?UTF-8?q?=E7=AE=80v1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm_web_kit/input/pre_data_json.py | 1 + .../main_html_parser/parser/tag_simplifier.py | 1 + .../simplify_html/simplify_html.py | 318 +++++++++++++++--- .../typical_html/typical_html.py | 147 ++++++-- .../parser/test_tag_simplifier.py | 4 +- 5 files changed, 393 insertions(+), 78 deletions(-) diff --git a/llm_web_kit/input/pre_data_json.py b/llm_web_kit/input/pre_data_json.py index 6625b89b..3e2dd7dd 100644 --- a/llm_web_kit/input/pre_data_json.py +++ b/llm_web_kit/input/pre_data_json.py @@ -15,6 +15,7 @@ class PreDataJsonKey: TYPICAL_RAW_HTML = 'typical_raw_html' TYPICAL_RAW_TAG_HTML = 'typical_raw_tag_html' + XPATH_MAPPING = 'xpath_mapping' TYPICAL_SIMPLIFIED_HTML = 'typical_simplified_html' LLM_RESPONSE = 'llm_response' HTML_ELEMENT_LIST = 'html_element_list' diff --git a/llm_web_kit/main_html_parser/parser/tag_simplifier.py b/llm_web_kit/main_html_parser/parser/tag_simplifier.py index 28ceac0b..1d705c1b 100644 --- a/llm_web_kit/main_html_parser/parser/tag_simplifier.py +++ b/llm_web_kit/main_html_parser/parser/tag_simplifier.py @@ -32,5 +32,6 @@ def parse(self, pre_data: PreDataJson) -> PreDataJson: # 设置输出数据 pre_data[PreDataJsonKey.TYPICAL_RAW_TAG_HTML] = original_html # 保存原始标签HTML pre_data[PreDataJsonKey.TYPICAL_SIMPLIFIED_HTML] = simplified_html # 保存简化后的HTML + pre_data[PreDataJsonKey.XPATH_MAPPING] = _ # 保存xpath return pre_data diff --git a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py index 12b08940..efb4ac31 100644 --- a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py +++ b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py @@ -3,15 +3,17 @@ import uuid from typing import Dict, List, Tuple +from bs4 import BeautifulSoup from lxml import etree, html # 行内标签 inline_tags = { - 'map', 'optgroup', 'span', 'br', 'input', 'time', 'u', 'td', 'strong', 'textarea', 'small', 'sub', + 'map', 'optgroup', 'span', 'br', 'input', 'time', 'u', 'strong', 'textarea', 'small', 'sub', 'samp', 'blink', 'b', 'code', 'nobr', 'strike', 'bdo', 'basefont', 'abbr', 'var', 'i', 'cccode-inline', - 'th', 'select', 's', 'pic', 'label', 'mark', 'object', 'dd', 'dt', 'ccmath-inline', 'svg', 'li', + 'select', 's', 'pic', 'label', 'mark', 'object', 'dd', 'dt', 'ccmath-inline', 'svg', 'li', 'button', 'a', 'font', 'dfn', 'sup', 'kbd', 'q', 'script', 'acronym', 'option', 'img', 'big', 'cite', - 'em' + 'em', + # 'td', 'th' } # 需要删除的标签 @@ -63,46 +65,83 @@ def is_unique_attribute(tree, attr_name, attr_value): def get_relative_xpath(element): - path = [] - root_element = element.getroottree().getroot() - - while element is not None and element.getparent() is not None: - siblings = [sib for sib in element.getparent() if sib.tag == element.tag] - - found_unique_attr = False # 初始化变量 - - if len(siblings) > 1: - # 如果有多个同名兄弟节点，则尝试使用属性路径来区分 - candidate_attrs = [ - attr for attr in element.attrib - if not (attr.startswith('data-') or attr == 'style' or - attr == '_item_id' or - (element.attrib[attr].startswith('{') and element.attrib[attr].endswith('}'))) - ] - - for attr in candidate_attrs: - if is_unique_attribute(root_element.getroottree(), attr, element.attrib[attr]): - # 插入唯一属性的XPath片段 - path.insert(0, f'*[@{attr}="{element.attrib[attr]}"]') - found_unique_attr = True - break - - if not found_unique_attr: - index = siblings.index(element) + 1 - path.insert(0, f'{element.tag}[{index}]') + root_tree = element.getroottree() + current_element = element + path_from_element = [] + found_unique_ancestor = False + + # 从当前元素开始向上查找 + while current_element is not None and current_element.getparent() is not None: + siblings = [sib for sib in current_element.getparent() if sib.tag == current_element.tag] + + # 检查当前元素是否有唯一属性 + unique_attr = None + candidate_attrs = [ + attr for attr in current_element.attrib + if not (attr.startswith('data-') or attr == 'style' or + attr == '_item_id' or + (current_element.attrib[attr].startswith('{') and current_element.attrib[attr].endswith('}'))) + ] + + for attr in candidate_attrs: + if is_unique_attribute(root_tree, attr, current_element.attrib[attr]): + unique_attr = attr + break + + # 如果有唯一属性，构建相对路径并停止向上查找 + if unique_attr is not None: + path_from_element.insert(0, f'*[@{unique_attr}="{current_element.attrib[unique_attr]}"]') + found_unique_ancestor = True + break else: - # 如果没有同名兄弟节点，则直接使用标签名 - path.insert(0, element.tag) + # 没有唯一属性，使用常规方式 + if len(siblings) > 1: + index = siblings.index(current_element) + 1 + path_from_element.insert(0, f'{current_element.tag}[{index}]') + else: + path_from_element.insert(0, current_element.tag) + + current_element = current_element.getparent() + + # 构建最终的XPath + if found_unique_ancestor: + return f'//{"/".join(path_from_element)}' + else: + # 如果没有找到唯一属性祖先，返回完整路径 + return f'//{"/".join(path_from_element)}' + + +def is_data_table(table_element: html.HtmlElement) -> bool: + """判断表格是否是数据表格而非布局表格.""" + # 检查表格是否有 caption 标签 + if table_element.xpath('.//caption'): + return True + + # 检查是否有 th 标签 + if table_element.xpath('.//th'): + return True + + # 检查是否有 thead 或 tfoot 标签 + if table_element.xpath('.//thead') or table_element.xpath('.//tfoot'): + return True - # 提前返回简化版本的XPath，如果当前元素有一个唯一属性 - if found_unique_attr: - simplified_path = f'//{"/".join(path)}' - return simplified_path + # 检查是否有 colgroup 或 col 标签 + if table_element.xpath('.//colgroup') or table_element.xpath('.//col'): + return True + + # 检查是否有 summary 属性 + if table_element.get('summary'): + return True - element = element.getparent() + # 检查是否有 role="table" 或 data-table 属性 + if table_element.get('role') == 'table' or table_element.get('data-table'): + return True - # 返回完整路径 - return f'//{"/".join(path)}' + # 检查单元格是否有 headers 属性 + if table_element.xpath('.//*[@headers]'): + return True + + return False def extract_paragraphs(processing_dom: html.HtmlElement, uid_map: Dict[str, html.HtmlElement], @@ -125,8 +164,33 @@ def extract_paragraphs(processing_dom: html.HtmlElement, uid_map: Dict[str, html :return: 段落列表，每个段落包含html、content_type和_original_element字段 """ + # 创建表格类型映射，记录每个表格是数据表格还是布局表格 + table_types = {} + + # 先分析所有表格的类型 + for table in processing_dom.xpath('.//table'): + table_types[table.get('data-uid')] = is_data_table(table) + def is_block_element(node) -> bool: """判断是否为块级元素.""" + # 处理表格单元格特殊情况 + if node.tag in ('td', 'th'): + # 找到最近的祖先table元素 + table_ancestor = node + while table_ancestor is not None and table_ancestor.tag != 'table': + table_ancestor = table_ancestor.getparent() + + # 如果是表格单元格，根据表格类型决定是否为块级元素 + if table_ancestor is not None: + table_uid = table_ancestor.get('data-uid') + if table_types.get(table_uid, False): + # 数据表格的td/th不作为块级元素 + return False + else: + # 布局表格的td/th作为块级元素 + return True + + # 默认处理其他元素 if node.tag in inline_tags: return False return isinstance(node, html.HtmlElement) @@ -283,7 +347,10 @@ def merge_inline_content(parent: html.HtmlElement, content_list: List[Tuple[str, def remove_xml_declaration(html_string): # 正则表达式匹配 或 （没有问号结尾的情况） pattern = r'<\?xml\s+.*?\??>' - return re.sub(pattern, '', html_string, flags=re.DOTALL) + html_content = re.sub(pattern, '', html_string, flags=re.DOTALL) + # 1. 删除HTML注释 + html_content = re.sub(r'', '', html_content, flags=re.DOTALL) + return html_content def post_process_html(html_content: str) -> str: @@ -352,7 +419,7 @@ def clean_attributes(element): def remove_inline_tags(element): - """递归移除所有指定的行内标签（包括嵌套情况），保留img和br标签 优化点：确保文本顺序正确，正确处理嵌套标签的文本转移.""" + """递归移除所有指定的行内标签（包括嵌套情况），保留img和br等EXCLUDED_TAGS标签.""" # 先处理子元素（深度优先） for child in list(element.iterchildren()): remove_inline_tags(child) @@ -363,6 +430,16 @@ def remove_inline_tags(element): if parent is None: return + # 检查是否包含需要保留的标签（如img、br等） + has_excluded_tags = any( + child.tag in EXCLUDED_TAGS + for child in element.iterdescendants() + ) + + # 如果包含需要保留的标签，则不移除当前元素 + if has_excluded_tags: + return + # 保存当前元素的各部分内容 leading_text = element.text or '' # 元素开始前的文本 trailing_text = element.tail or '' # 元素结束后的文本 @@ -482,6 +559,13 @@ def should_remove_element(element) -> bool: for pattern in ATTR_SUFFIX_TO_REMOVE: if pattern in part: return True + + # 检查style属性 + style_attr = element.get('style', '') + if style_attr: + if 'display: none' in style_attr or 'display:none' in style_attr: + return True + return False @@ -556,9 +640,9 @@ def process_paragraphs(paragraphs: List[Dict[str, str]], uid_map: Dict[str, html for para in paragraphs: try: - # print(f"para: {para['html']}") + html_content = re.sub(r'', '', para['html'], flags=re.DOTALL) # 解析段落HTML - root = html.fromstring(post_process_html(para['html'])) + root = html.fromstring(html_content) root_for_xpath = copy.deepcopy(root) content_type = para.get('content_type', 'block_element') @@ -574,11 +658,6 @@ def process_paragraphs(paragraphs: List[Dict[str, str]], uid_map: Dict[str, html # 截断过长的文本内容 truncate_text_content(root, max_length=1000) - # 为当前段落和原始元素添加相同的 _item_id - current_id = str(item_id) - root.set('_item_id', current_id) - para['_original_element'].set('_item_id', current_id) - para_xpath = [] if is_xpath: if content_type in ('inline_elements', 'mixed'): @@ -602,10 +681,141 @@ def process_paragraphs(paragraphs: List[Dict[str, str]], uid_map: Dict[str, html _xpath = None para_xpath.append(_xpath) + # 为当前段落和原始元素添加相同的 _item_id + current_id = str(item_id) + root.set('_item_id', current_id) + + # 对于非块级元素（inline_elements, unwrapped_text, mixed） + original_parent = para['_original_element'] # 原网页中直接子元素的父节点 + if content_type != 'block_element': + if original_parent is not None: + # root_for_xpath有子元素 + if len(root_for_xpath) > 0: + if root_for_xpath.tag in inline_tags and uid_map.get(root_for_xpath.get('data-uid')).tag != 'body': + original_element = uid_map.get(root_for_xpath.get('data-uid')) + original_element.set('_item_id', current_id) + else: + # 收集需要包裹的子元素 + children_to_wrap = [] + for child in root_for_xpath.iterchildren(): + child_uid = child.get('data-uid') + if child_uid and child_uid in uid_map: + original_child = uid_map[child_uid] + children_to_wrap.append(original_child) + + if children_to_wrap: + # 确定包裹范围 + first_child = children_to_wrap[0] + last_child = children_to_wrap[-1] + + # 获取在父节点中的位置 + start_idx = original_parent.index(first_child) + end_idx = original_parent.index(last_child) + + # 收集所有需要移动的节点 + nodes_to_wrap = [] + for i in range(start_idx, end_idx + 1): + nodes_to_wrap.append(original_parent[i]) + + # 处理前面的文本 + leading_text = original_parent.text if start_idx == 0 else original_parent[ + start_idx - 1].tail + + # 处理后面的文本 + # trailing_text = last_child.tail + + # 创建wrapper元素 + wrapper = etree.Element('cc-alg-uc-tex') + wrapper.set('_item_id', current_id) + + # 设置前面的文本 + if leading_text: + wrapper.text = leading_text + if start_idx == 0: + original_parent.text = None + else: + original_parent[start_idx - 1].tail = None + + # 移动节点到wrapper中 + for node in nodes_to_wrap: + original_parent.remove(node) + wrapper.append(node) + + # 插入wrapper + original_parent.insert(start_idx, wrapper) + + # 设置后面的文本 + # if trailing_text: + # wrapper.tail = trailing_text + # last_child.tail = None + else: + if content_type == 'inline_elements': + original_element = uid_map.get(root_for_xpath.get('data-uid')) + original_element.set('_item_id', current_id) + else: + # root_for_xpath只有文本内容 + if root_for_xpath.text and root_for_xpath.text.strip(): + # 1. 在原始DOM中查找匹配的文本节点 + found = False + + # 检查父节点的text + if original_parent.text and original_parent.text.strip() == root_for_xpath.text.strip(): + # 创建wrapper + wrapper = etree.Element('cc-alg-uc-tex') + wrapper.set('_item_id', current_id) + wrapper.text = original_parent.text + + # 替换父节点的text + original_parent.text = None + + # 插入wrapper作为第一个子节点 + if len(original_parent) > 0: + original_parent.insert(0, wrapper) + else: + original_parent.append(wrapper) + + found = True + + # 检查子节点的tail + if not found: + for child in original_parent.iterchildren(): + if child.tail and child.tail.strip() == root_for_xpath.text.strip(): + # 创建wrapper + wrapper = etree.Element('cc-alg-uc-tex') + wrapper.set('_item_id', current_id) + wrapper.text = child.tail + + # 替换tail + child.tail = None + + # 插入wrapper到子节点之后 + parent = child.getparent() + index = parent.index(child) + parent.insert(index + 1, wrapper) + + found = True + break + + # 如果没有找到匹配的文本节点，使用父节点作为包裹对象 + if not found: + wrapper = etree.Element('cc-alg-uc-tex') + wrapper.set('_item_id', current_id) + wrapper.text = root_for_xpath.text + # 将父节点的内容移动到wrapper中 + for child in list(original_parent.iterchildren()): + wrapper.append(child) + original_parent.remove(child) + + # 添加wrapper到父节点 + original_parent.append(wrapper) + else: + # 块级元素直接设置属性 + original_parent.set('_item_id', current_id) + item_id += 1 # 保存处理结果 - cleaned_html = etree.tostring(root, encoding='unicode').strip() + cleaned_html = etree.tostring(root, method='html', encoding='unicode').strip() result.append({ 'html': cleaned_html, '_item_id': current_id, @@ -622,7 +832,7 @@ def process_paragraphs(paragraphs: List[Dict[str, str]], uid_map: Dict[str, html simplified_html = '' + ''.join( p['html'] for p in result) + '' - return simplified_html, result + return post_process_html(simplified_html), result def simplify_html(html_str, is_xpath: bool = True) -> etree.Element: @@ -635,8 +845,12 @@ def simplify_html(html_str, is_xpath: bool = True) -> etree.Element: # 预处理 preprocessed_html = remove_xml_declaration(html_str) + # 用 BeautifulSoup 修复未闭合标签，lxml 无法完全修复 + soup = BeautifulSoup(preprocessed_html, 'html.parser') + fixed_html = str(soup) + # 解析原始DOM - original_dom = html.fromstring(preprocessed_html) + original_dom = html.fromstring(fixed_html) add_data_uids(original_dom) original_uid_map = build_uid_map(original_dom) @@ -653,7 +867,7 @@ def simplify_html(html_str, is_xpath: bool = True) -> etree.Element: simplified_html, result = process_paragraphs(paragraphs, original_uid_map, is_xpath) remove_all_uids(original_dom) - original_html = etree.tostring(original_dom, pretty_print=True, encoding='unicode') + original_html = etree.tostring(original_dom, pretty_print=True, method='html', encoding='unicode') _xpath_mapping = {item['_item_id']: { '_xpath': item['_xpath'], diff --git a/llm_web_kit/main_html_parser/typical_html/typical_html.py b/llm_web_kit/main_html_parser/typical_html/typical_html.py index 8ed5d189..72e9e24b 100644 --- a/llm_web_kit/main_html_parser/typical_html/typical_html.py +++ b/llm_web_kit/main_html_parser/typical_html/typical_html.py @@ -1,8 +1,17 @@ +import math +import re +from collections import defaultdict from io import StringIO from lxml import html -REQUIRED_TAGS = {'', ''} +REQUIRED_TAGS = {''} + + +def remove_xml_declaration(html_string): + # 正则表达式匹配 或 （没有问号结尾的情况） + pattern = r'<\?xml\s+.*?\??>' + return re.sub(pattern, '', html_string, flags=re.DOTALL) def has_essential_tags(html_str): @@ -11,17 +20,17 @@ def has_essential_tags(html_str): return all(tag in lower_html for tag in REQUIRED_TAGS) -def select_representative_html(html_content_list): - """从多个HTML文件中选择最具代表性的一个.""" +def select_representative_html(html_strings): + """从多个HTML字符串中选择最具代表性的一个（仅计算body内内容）""" # 存储所有页面的XPath和复杂度信息 page_data = [] global_xpaths = set() - # 第一遍：收集所有页面的XPath和基本复杂度 - for item in html_content_list: + # 第一遍：收集所有页面的XPath和基本复杂度（仅body内） + for html_dict in html_strings: try: - html_str = item['html'] - track_id = item['track_id'] + html_str = remove_xml_declaration(html_dict['html']) + track_id = html_dict['track_id'] if not has_essential_tags(html_str): continue @@ -29,50 +38,140 @@ def select_representative_html(html_content_list): if len(html_str) < 100: continue - file_obj = StringIO(html_str) + # 将字符串转换为文件对象 + file_obj = StringIO(html_dict['html']) tree = html.parse(file_obj) - # 收集本页所有XPath + # 找到body元素 + body_element = tree.find('.//body') + if body_element is None: + continue + + # 收集body内所有XPath page_xpaths = set() total_tags = 0 - for element in tree.iter(): + tag_types = set() + max_depth = 0 + current_depth = 0 + depth_stack = [] + total_width = 0 # 总标签宽度（所有元素的子元素数量之和） + max_width = 0 # 最大单个标签宽度 + width_counts = defaultdict(int) # 各宽度级别的计数 + + for element in body_element.iter(): + # 计算当前深度 + if depth_stack and element in depth_stack[-1].getchildren(): + current_depth += 1 + else: + while depth_stack and element not in depth_stack[-1].getchildren(): + depth_stack.pop() + current_depth -= 1 + if depth_stack: + current_depth += 1 + depth_stack.append(element) + + # 更新最大深度 + if current_depth > max_depth: + max_depth = current_depth + + # 计算标签宽度（子元素数量） + children = list(element.getchildren()) + width = len(children) + total_width += width + if width > max_width: + max_width = width + # 记录宽度分布 + width_level = min(width, 10) # 将宽度分组，大于10的算作同一组 + width_counts[width_level] += 1 + xpath = tree.getpath(element) page_xpaths.add(xpath) total_tags += 1 + tag_types.add(element.tag) + + # 计算宽度多样性（使用熵来衡量） + width_entropy = 0 + total_elements = sum(width_counts.values()) + if total_elements > 0: + for count in width_counts.values(): + probability = count / total_elements + if probability > 0: + width_entropy -= probability * (probability and math.log(probability, 2)) # 记录页面数据 page_data.append({ 'track_id': track_id, 'xpaths': page_xpaths, 'tag_count': total_tags, - 'original_data': item, + 'max_depth': max_depth, + 'tag_types': tag_types, + 'tag_diversity': len(tag_types), # 标签多样性 + 'total_width': total_width, # 总宽度 + 'avg_width': total_width / total_tags if total_tags > 0 else 0, # 平均宽度 + 'max_width': max_width, # 最大宽度 + 'width_entropy': width_entropy, # 宽度分布熵 + 'original_data': html_dict, }) - # 更新全局XPath集合 + # 更新全局XPath集合（仅body内） global_xpaths.update(page_xpaths) except Exception: + # import traceback + # print(f'Error processing HTML: {traceback.format_exc()}') continue if not page_data: return None - # 第二遍：计算每个页面的代表得分 + # 第二遍：计算每个页面的代表得分（仅基于body内内容） best_score = -1 - representative_path = None + best_base_score = -1 # 用于存储最佳基础得分 + representative_html = None + + # 获取最大值用于归一化 + max_depth_all = max(p['max_depth'] for p in page_data) if page_data else 1 + max_tags_all = max(p['tag_count'] for p in page_data) if page_data else 1 + max_diversity_all = max(p['tag_diversity'] for p in page_data) if page_data else 1 + max_avg_width = max(p['avg_width'] for p in page_data) if page_data else 1 + max_max_width = max(p['max_width'] for p in page_data) if page_data else 1 + max_width_entropy = max(p['width_entropy'] for p in page_data) if page_data else 1 for page in page_data: - # 计算XPath覆盖率（该页面包含的全局XPath比例） - coverage = len(page['xpaths'] & global_xpaths) / len(global_xpaths) + # 计算XPath覆盖率（该页面body包含的全局XPath比例） + coverage = len(page['xpaths'] & global_xpaths) / len(global_xpaths) if global_xpaths else 0 + + # 计算复杂度得分（基于body内标签数量） + complexity = page['tag_count'] / max_tags_all + + # 计算标签多样性得分 + diversity = page['tag_diversity'] / max_diversity_all + + # 计算深度得分 + depth_score = page['max_depth'] / max_depth_all if max_depth_all > 0 else 0 + + # 计算宽度相关得分 + avg_width_score = page['avg_width'] / max_avg_width if max_avg_width > 0 else 0 + max_width_score = page['max_width'] / max_max_width if max_max_width > 0 else 0 + width_entropy_score = page['width_entropy'] / max_width_entropy if max_width_entropy > 0 else 0 + + # 计算基础得分（覆盖率和多样性） + base_score = 0.7 * coverage + 0.3 * diversity + + # 计算结构得分（复杂度、深度和宽度特征） + structure_score = 0.2 * complexity + 0.1 * depth_score + 0.4 * avg_width_score + 0.3 * max_width_score - # 计算复杂度得分（基于标签数量） - complexity = page['tag_count'] / max(p['tag_count'] for p in page_data) + # 计算分布得分（宽度分布的均匀性） + distribution_score = width_entropy_score - # 综合得分（可根据需要调整权重） - score = 0.6 * coverage + 0.4 * complexity + # 综合得分 + total_score = 0.4 * base_score + 0.3 * structure_score + 0.3 * distribution_score - if score > best_score: - best_score = score - representative_path = page['original_data'] + # 选择得分最高的页面，如果得分相同则选择基础得分更高的 + if (total_score > best_score) or \ + (total_score == best_score and base_score > best_base_score): + best_score = total_score + best_base_score = base_score + representative_html = page['original_data'] - return representative_path + return representative_html diff --git a/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py b/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py index 64524660..3d656811 100644 --- a/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py +++ b/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py @@ -51,7 +51,7 @@ def test_tag_simplifier3(self): pre_data_result = HtmlTagSimplifierParser({}).parse(pre_data) simplifier_raw_html = pre_data_result.get(PreDataJsonKey.TYPICAL_SIMPLIFIED_HTML, '') _item_id_count = simplifier_raw_html.count('_item_id') - self.assertEqual(_item_id_count, 5) + self.assertEqual(_item_id_count, 66) def test_tag_simplifier4(self): file_path = base_dir / 'assets/test_html_data/1.html' @@ -62,7 +62,7 @@ def test_tag_simplifier4(self): pre_data_result = HtmlTagSimplifierParser({}).parse(pre_data) simplifier_raw_html = pre_data_result.get(PreDataJsonKey.TYPICAL_SIMPLIFIED_HTML, '') _item_id_count = simplifier_raw_html.count('_item_id') - self.assertEqual(_item_id_count, 37) + self.assertEqual(_item_id_count, 51) if __name__ == '__main__': From c3fe3a07a465d70bfa1a6e8ca89833449c987a5a Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Wed, 4 Jun 2025 12:49:41 +0800 Subject: [PATCH 10/13] =?UTF-8?q?fix:=20=E6=A0=B9=E6=8D=AE=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E8=AF=84=E6=B5=8B=E8=B0=83=E6=95=B4=E7=B2=BE=E7=AE=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../simplify_html/simplify_html.py | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py index efb4ac31..adc350cd 100644 --- a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py +++ b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py @@ -18,7 +18,18 @@ # 需要删除的标签 tags_to_remove = { - 'head', 'header', 'footer', 'nav', 'aside', 'style', 'script', 'select', 'noscript', 'link', 'meta', 'iframe', 'frame' + 'head', + 'header', + 'footer', + 'nav', + 'aside', + 'style', + 'script', + 'noscript', + 'link', + 'meta', + 'iframe', + 'frame' } # 需要保留的特殊标签（即使它们是行内标签） @@ -31,7 +42,7 @@ # 需要删除的属性名模式（特定前缀/后缀） ATTR_SUFFIX_TO_REMOVE = { - '-nav', '_nav', + # '-nav', '_nav', # '-footer', '_footer', # 有特例，可能dl列表一组最后一项添加了自定义footer属性，先注释 # '-header', '_header', # 有特例，可能自定义的header中有标题，先注释 } @@ -544,7 +555,7 @@ def should_remove_element(element) -> bool: return True # 检查是否包含特定前缀/后缀 for pattern in ATTR_SUFFIX_TO_REMOVE: - if pattern in part: + if part.endswith(pattern): return True # 检查id属性 @@ -557,7 +568,7 @@ def should_remove_element(element) -> bool: return True # 检查是否包含特定前缀/后缀 for pattern in ATTR_SUFFIX_TO_REMOVE: - if pattern in part: + if part.endswith(pattern): return True # 检查style属性 @@ -647,9 +658,9 @@ def process_paragraphs(paragraphs: List[Dict[str, str]], uid_map: Dict[str, html content_type = para.get('content_type', 'block_element') # 公共处理步骤 - clean_attributes(root) + # clean_attributes(root) simplify_list(root) - remove_inline_tags(root) + # remove_inline_tags(root) # 跳过无意义内容 if not is_meaningful_content(root): From 965bf4cbf6314918c4db9347430a9a1b4c95f4e3 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Wed, 4 Jun 2025 15:17:12 +0800 Subject: [PATCH 11/13] =?UTF-8?q?fix:=20=E6=A0=B9=E6=8D=AE=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E8=AF=84=E6=B5=8B=E8=B0=83=E6=95=B4=E7=B2=BE=E7=AE=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../template_www.wdi.it_llm.json | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/llm_web_kit/main_html_parser/parser/assets/input_layout_batch_parser/template_www.wdi.it_llm.json b/tests/llm_web_kit/main_html_parser/parser/assets/input_layout_batch_parser/template_www.wdi.it_llm.json index f2bf5c1b..abb40deb 100644 --- a/tests/llm_web_kit/main_html_parser/parser/assets/input_layout_batch_parser/template_www.wdi.it_llm.json +++ b/tests/llm_web_kit/main_html_parser/parser/assets/input_layout_batch_parser/template_www.wdi.it_llm.json @@ -5,9 +5,9 @@ "item_id 4": "No", "item_id 5": "No", "item_id 6": "No", - "item_id 7": "Yes", + "item_id 7": "No", "item_id 8": "No", - "item_id 9": "No", + "item_id 9": "Yes", "item_id 10": "No", "item_id 11": "No", "item_id 12": "No", @@ -31,5 +31,9 @@ "item_id 30": "No", "item_id 31": "No", "item_id 32": "No", - "item_id 33": "No" - } \ No newline at end of file + "item_id 33": "No", + "item_id 34": "No", + "item_id 35": "No", + "item_id 36": "No", + "item_id 37": "No" +} \ No newline at end of file From 636653af775b3f942392b688d5c014219dc4b317 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Wed, 4 Jun 2025 15:39:11 +0800 Subject: [PATCH 12/13] =?UTF-8?q?fix:=20=E6=A0=B9=E6=8D=AE=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E8=AF=84=E6=B5=8B=E8=B0=83=E6=95=B4=E7=B2=BE=E7=AE=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main_html_parser/simplify_html/simplify_html.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py index adc350cd..44801c60 100644 --- a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py +++ b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py @@ -554,9 +554,9 @@ def should_remove_element(element) -> bool: if part in ATTR_PATTERNS_TO_REMOVE: return True # 检查是否包含特定前缀/后缀 - for pattern in ATTR_SUFFIX_TO_REMOVE: - if part.endswith(pattern): - return True + # for pattern in ATTR_SUFFIX_TO_REMOVE: + # if part.endswith(pattern): + # return True # 检查id属性 id_name = element.get('id', '') @@ -567,9 +567,9 @@ def should_remove_element(element) -> bool: if part in ATTR_PATTERNS_TO_REMOVE: return True # 检查是否包含特定前缀/后缀 - for pattern in ATTR_SUFFIX_TO_REMOVE: - if part.endswith(pattern): - return True + # for pattern in ATTR_SUFFIX_TO_REMOVE: + # if part.endswith(pattern): + # return True # 检查style属性 style_attr = element.get('style', '') From 2de6120d5e5d143aadd7b2618af671a65927ba20 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Mon, 9 Jun 2025 14:25:52 +0800 Subject: [PATCH 13/13] =?UTF-8?q?feat:=20=E7=B2=BE=E7=AE=80=E5=B1=9E?= =?UTF-8?q?=E6=80=A7=E5=8F=AA=E4=BF=9D=E7=95=99=E5=9B=BE=E7=89=87src?= =?UTF-8?q?=E5=92=8C=E5=85=83=E7=B4=A0class=E3=80=81id?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../simplify_html/simplify_html.py | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py index 44801c60..b47de995 100644 --- a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py +++ b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py @@ -417,14 +417,29 @@ def is_meaningful_content(element) -> bool: def clean_attributes(element): - """清理元素属性，只保留图片的有效src.""" + """清理元素属性，只保留图片的有效src以及所有元素的class和id.""" if element.tag == 'img': src = element.get('src', '').strip() - element.attrib.clear() + class_attr = element.get('class', '').strip() + id_attr = element.get('id', '').strip() + element.attrib.clear() # 先清除所有属性 if src: element.set('src', src) + if class_attr: + element.set('class', class_attr) + if id_attr: + element.set('id', id_attr) else: - element.attrib.clear() + # 对于其他元素，只保留class和id + class_attr = element.get('class', '').strip() + id_attr = element.get('id', '').strip() + element.attrib.clear() # 先清除所有属性 + if class_attr: + element.set('class', class_attr) + if id_attr: + element.set('id', id_attr) + + # 递归处理子元素 for child in element: clean_attributes(child) @@ -658,7 +673,7 @@ def process_paragraphs(paragraphs: List[Dict[str, str]], uid_map: Dict[str, html content_type = para.get('content_type', 'block_element') # 公共处理步骤 - # clean_attributes(root) + clean_attributes(root) simplify_list(root) # remove_inline_tags(root)