diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c3337c0..aa87dc57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- [CLI] DOM JSON 데이터에서 기사 제목(headline)만 추출하여 텍스트 파일로 저장하는 `tools/extract_headlines.py` 도구를 추가했습니다. +- [CLI] DOM JSON 데이터에서 띄어쓰기를 최소화하여 JSON 크기를 줄이는(Minify) `tools/minify_dom.py` 도구를 추가했습니다. + ### Changed - `/parse`를 언어 선택형 파서로 일반화: MinerU `-l japan`/`-m ocr` 하드코딩을 제거하고 optional form 필드 `language`(MinerU 3.4.4 공식 기본 `ch`, 공개 언어군/alias 검증)와 `mode`(`auto`/`ocr`/`txt`, 기본 `auto`)로 파라미터화. `mode=auto`는 born-digital PDF가 강제 OCR을 건너뛰도록 함. 기존 입력 `language=japan&mode=ocr`는 공식 규약대로 `ch`/`ocr`로 정규화됨. - OpenAPI 제목/설명, README, `ArticleNode.headline` 문서를 일반 문서용 (section heading) 표현으로 재구성하여 특정 언어/신문 가정을 소비자에게 노출하지 않도록 함. 응답 스키마 필드는 하위 호환을 위해 변경하지 않음. diff --git a/tests/test_tools_extract_headlines.py b/tests/test_tools_extract_headlines.py new file mode 100644 index 00000000..23481ac7 --- /dev/null +++ b/tests/test_tools_extract_headlines.py @@ -0,0 +1,122 @@ +import json +import pytest +from pathlib import Path +from tools.extract_headlines import extract_headlines, main + + +def test_extract_headlines_valid(tmp_path: Path): + input_path = tmp_path / "input.json" + output_path = tmp_path / "output.txt" + data = { + "document_id": "test", + "pages": [ + { + "page_number": 1, + "articles": [ + {"article_id": "a1", "headline": "Headline 1", "body_blocks": []}, + {"article_id": "a2", "headline": "Headline 2", "body_blocks": []}, + ], + } + ], + "quality": {"status": "success", "parser": "test", "warnings": []}, + } + input_path.write_text(json.dumps(data), encoding="utf-8") + extract_headlines(input_path, output_path) + result = output_path.read_text(encoding="utf-8") + assert result == "Headline 1\nHeadline 2\n" + + +def test_extract_headlines_file_not_found(tmp_path: Path): + with pytest.raises(FileNotFoundError): + extract_headlines(tmp_path / "nonexistent.json", tmp_path / "output.txt") + + +def test_extract_headlines_invalid_extension(tmp_path: Path): + invalid_file = tmp_path / "input.txt" + invalid_file.touch() + with pytest.raises(ValueError, match="must be a .json file"): + extract_headlines(invalid_file, tmp_path / "output.txt") + + +def test_extract_headlines_invalid_json(tmp_path: Path): + input_path = tmp_path / "input.json" + input_path.write_text("invalid json", encoding="utf-8") + with pytest.raises(ValueError, match="Invalid JSON file"): + extract_headlines(input_path, tmp_path / "output.txt") + + +def test_extract_headlines_invalid_schema(tmp_path: Path): + input_path = tmp_path / "input.json" + input_path.write_text('{"invalid": "schema"}', encoding="utf-8") + with pytest.raises(ValueError, match="does not match ParseResponse schema"): + extract_headlines(input_path, tmp_path / "output.txt") + + +def test_main_valid(tmp_path: Path, capsys: pytest.CaptureFixture[str]): + input_path = tmp_path / "input.json" + output_path = tmp_path / "output.txt" + data = { + "document_id": "test", + "pages": [], + "quality": {"status": "success", "parser": "test", "warnings": []}, + } + input_path.write_text(json.dumps(data), encoding="utf-8") + main([str(input_path), "-o", str(output_path)]) + assert f"Headlines successfully written to {output_path}" in capsys.readouterr().out + + +def test_main_error(tmp_path: Path, capsys: pytest.CaptureFixture[str]): + with pytest.raises(SystemExit): + main([str(tmp_path / "nonexistent.json"), "-o", str(tmp_path / "output.txt")]) + assert "Error extracting headlines" in capsys.readouterr().err + + +import sys # noqa: E402 +import importlib # noqa: E402 +import runpy # noqa: E402 + + +def test_extract_headlines_import_path(monkeypatch): + """Test sys.path injection for coverage.""" + import tools.extract_headlines + + target_path = str( + Path(tools.extract_headlines.__file__).resolve().parents[1] / "src" + ) + monkeypatch.setattr(sys, "path", [p for p in sys.path if p != target_path]) + importlib.reload(tools.extract_headlines) + assert target_path in sys.path + + +def test_extract_headlines_main_module(): + """Test if __name__ == '__main__' block coverage via run_module.""" + import warnings + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=RuntimeWarning) + try: + runpy.run_module("tools.extract_headlines", run_name="__main__") + except SystemExit: + pass + + +def test_extract_headlines_empty_headline(tmp_path: Path): + input_path = tmp_path / "input.json" + output_path = tmp_path / "output.txt" + data = { + "document_id": "test", + "pages": [ + { + "page_number": 1, + "articles": [ + {"article_id": "a1", "headline": "", "body_blocks": []}, + {"article_id": "a2", "headline": "Headline 2", "body_blocks": []}, + ], + } + ], + "quality": {"status": "success", "parser": "test", "warnings": []}, + } + input_path.write_text(json.dumps(data), encoding="utf-8") + extract_headlines(input_path, output_path) + result = output_path.read_text(encoding="utf-8") + assert result == "Headline 2\n" diff --git a/tests/test_tools_minify_dom.py b/tests/test_tools_minify_dom.py new file mode 100644 index 00000000..c9b09924 --- /dev/null +++ b/tests/test_tools_minify_dom.py @@ -0,0 +1,96 @@ +import json +import pytest +from pathlib import Path # noqa: E402 +from tools.minify_dom import minify_dom, main + + +def test_minify_dom_valid(tmp_path: Path): + input_path = tmp_path / "input.json" + output_path = tmp_path / "output.json" + data = { + "document_id": "test", + "pages": [], + "quality": {"status": "success", "parser": "test", "warnings": []}, + } + input_path.write_text(json.dumps(data, indent=4), encoding="utf-8") + minify_dom(input_path, output_path) + result = output_path.read_text(encoding="utf-8") + assert " " not in result + assert "\n" not in result + assert json.loads(result) == data + + +def test_minify_dom_file_not_found(tmp_path: Path): + with pytest.raises(FileNotFoundError): + minify_dom(tmp_path / "nonexistent.json", tmp_path / "output.json") + + +def test_minify_dom_invalid_extension(tmp_path: Path): + invalid_file = tmp_path / "input.txt" + invalid_file.touch() + with pytest.raises(ValueError, match="must be a .json file"): + minify_dom(invalid_file, tmp_path / "output.json") + + +def test_minify_dom_invalid_json(tmp_path: Path): + input_path = tmp_path / "input.json" + input_path.write_text("invalid json", encoding="utf-8") + with pytest.raises(ValueError, match="Invalid JSON file"): + minify_dom(input_path, tmp_path / "output.json") + + +def test_minify_dom_invalid_schema(tmp_path: Path): + input_path = tmp_path / "input.json" + input_path.write_text('{"invalid": "schema"}', encoding="utf-8") + with pytest.raises(ValueError, match="does not match ParseResponse schema"): + minify_dom(input_path, tmp_path / "output.json") + + +def test_main_valid(tmp_path: Path, capsys: pytest.CaptureFixture[str]): + input_path = tmp_path / "input.json" + output_path = tmp_path / "output.json" + data = { + "document_id": "test", + "pages": [], + "quality": {"status": "success", "parser": "test", "warnings": []}, + } + input_path.write_text(json.dumps(data), encoding="utf-8") + main([str(input_path), "-o", str(output_path)]) + assert ( + f"Minified JSON successfully written to {output_path}" + in capsys.readouterr().out + ) + + +def test_main_error(tmp_path: Path, capsys: pytest.CaptureFixture[str]): + with pytest.raises(SystemExit): + main([str(tmp_path / "nonexistent.json"), "-o", str(tmp_path / "output.json")]) + assert "Error minifying JSON file" in capsys.readouterr().err + + +import sys # noqa: E402 +import importlib # noqa: E402 +import runpy # noqa: E402 +from pathlib import Path # noqa: E402 + + +def test_minify_dom_import_path(monkeypatch): + """Test sys.path injection for coverage.""" + import tools.minify_dom + + target_path = str(Path(tools.minify_dom.__file__).resolve().parents[1] / "src") + monkeypatch.setattr(sys, "path", [p for p in sys.path if p != target_path]) + importlib.reload(tools.minify_dom) + assert target_path in sys.path + + +def test_minify_dom_main_module(): + """Test if __name__ == '__main__' block coverage via run_module.""" + import warnings + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=RuntimeWarning) + try: + runpy.run_module("tools.minify_dom", run_name="__main__") + except SystemExit: + pass diff --git a/tools/extract_headlines.py b/tools/extract_headlines.py new file mode 100644 index 00000000..4880b78b --- /dev/null +++ b/tools/extract_headlines.py @@ -0,0 +1,61 @@ +from __future__ import annotations +import argparse +import json +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[1] +_SRC_ROOT = _REPO_ROOT / "src" +if str(_SRC_ROOT) not in sys.path: + sys.path.insert(0, str(_SRC_ROOT)) +from pydantic import ValidationError # noqa: E402 +from newsdom_api.schemas import ParseResponse # noqa: E402 + + +def extract_headlines(json_path: Path, output_path: Path) -> None: + if not json_path.is_file(): + raise FileNotFoundError(f"File not found or is not a file: {json_path}") + if json_path.suffix.lower() != ".json": + raise ValueError(f"Input file must be a .json file: {json_path}") + try: + data = json.loads(json_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON file ({json_path}): {exc}") from exc + try: + ParseResponse.model_validate(data) + except ValidationError as exc: + raise ValueError( + f"File {json_path} does not match ParseResponse schema: {exc}" + ) from exc + headlines = [] + for page in data.get("pages", []): + for article in page.get("articles", []): + headline = article.get("headline") + if headline: + headlines.append(headline) + output_path.write_text("\n".join(headlines) + "\n", encoding="utf-8") + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser( + description="Extract headlines from a NewsDOM JSON file." + ) + parser.add_argument("input", type=Path, help="Path to the input JSON file.") + parser.add_argument( + "-o", + "--output", + type=Path, + required=True, + help="Path to write the headlines text file.", + ) + args = parser.parse_args(argv) + try: + extract_headlines(args.input, args.output) + print(f"Headlines successfully written to {args.output}") + except Exception as exc: + print(f"Error extracting headlines: {exc}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tools/minify_dom.py b/tools/minify_dom.py new file mode 100644 index 00000000..db75f758 --- /dev/null +++ b/tools/minify_dom.py @@ -0,0 +1,55 @@ +from __future__ import annotations +import argparse +import json +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[1] +_SRC_ROOT = _REPO_ROOT / "src" +if str(_SRC_ROOT) not in sys.path: + sys.path.insert(0, str(_SRC_ROOT)) +from pydantic import ValidationError # noqa: E402 +from newsdom_api.schemas import ParseResponse # noqa: E402 + + +def minify_dom(json_path: Path, output_path: Path) -> None: + if not json_path.is_file(): + raise FileNotFoundError(f"File not found or is not a file: {json_path}") + if json_path.suffix.lower() != ".json": + raise ValueError(f"Input file must be a .json file: {json_path}") + try: + data = json.loads(json_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON file ({json_path}): {exc}") from exc + try: + ParseResponse.model_validate(data) + except ValidationError as exc: + raise ValueError( + f"File {json_path} does not match ParseResponse schema: {exc}" + ) from exc + output_path.write_text( + json.dumps(data, ensure_ascii=False, separators=(",", ":")), encoding="utf-8" + ) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description="Minify a NewsDOM JSON file.") + parser.add_argument("input", type=Path, help="Path to the input JSON file.") + parser.add_argument( + "-o", + "--output", + type=Path, + required=True, + help="Path to write the minified JSON output file.", + ) + args = parser.parse_args(argv) + try: + minify_dom(args.input, args.output) + print(f"Minified JSON successfully written to {args.output}") + except Exception as exc: + print(f"Error minifying JSON file: {exc}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main()