diff --git a/llm_web_kit/main_html_parser/parser/tag_mapping.py b/llm_web_kit/main_html_parser/parser/tag_mapping.py index 8878cb4b..d1f99090 100644 --- a/llm_web_kit/main_html_parser/parser/tag_mapping.py +++ b/llm_web_kit/main_html_parser/parser/tag_mapping.py @@ -178,9 +178,8 @@ def iter_process(elem): else: # 非正文节点直接删除 parent = elem.getparent() - if parent is None: - return - parent.remove(elem) + if parent is not None: + parent.remove(elem) for elem_child in elem: iter_process(elem_child) diff --git a/tests/llm_web_kit/main_html_parser/processor/test_tag_mapping.py b/tests/llm_web_kit/main_html_parser/processor/test_tag_mapping.py index fe90fed7..44b8b9c3 100644 --- a/tests/llm_web_kit/main_html_parser/processor/test_tag_mapping.py +++ b/tests/llm_web_kit/main_html_parser/processor/test_tag_mapping.py @@ -77,3 +77,20 @@ def test_parse_single(self): content_list = pre_data[PreDataJsonKey.HTML_TARGET_LIST] self.assertEqual(content_list, mock_dict['expected_content_list']) self.assertEqual(len(pre_data['typical_main_html']), 2269) + + def test_parse_single_empty(self): + data = [] + raw_html_path = base_dir.joinpath('assets/test_tag_mapping_web.jsonl') + with open(raw_html_path, 'r', encoding='utf-8') as f: + for line in f: + data.append(json.loads(line.strip())) # 解析每行 JSON + mock_dict = data[0] + pre_data = PreDataJson(mock_dict['pre_data']) + llm_response = pre_data[PreDataJsonKey.LLM_RESPONSE] + llm_response['item_id 5'] = 0 + llm_response['item_id 7'] = 0 + pre_data[PreDataJsonKey.LLM_RESPONSE] = llm_response + pre_data[PreDataJsonKey.TYPICAL_RAW_HTML] = pre_data[PreDataJsonKey.TYPICAL_RAW_TAG_HTML] + parser = MapItemToHtmlTagsParser({}) + pre_data = parser.parse_single(pre_data) + self.assertEqual(pre_data['typical_main_html'], '')