feat: 기사 제목 추출 및 DOM JSON Minify 도구 추가

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- [CLI] DOM JSON 데이터에서 기사 제목(headline)만 추출하여 텍스트 파일로 저장하는 `tools/extract_headlines.py` 도구를 추가했습니다.
+- [CLI] DOM JSON 데이터에서 띄어쓰기를 최소화하여 JSON 크기를 줄이는(Minify) `tools/minify_dom.py` 도구를 추가했습니다.
+
 ### Changed
 - `/parse`를 언어 선택형 파서로 일반화: MinerU `-l japan`/`-m ocr` 하드코딩을 제거하고 optional form 필드 `language`(MinerU 3.4.4 공식 기본 `ch`, 공개 언어군/alias 검증)와 `mode`(`auto`/`ocr`/`txt`, 기본 `auto`)로 파라미터화. `mode=auto`는 born-digital PDF가 강제 OCR을 건너뛰도록 함. 기존 입력 `language=japan&mode=ocr`는 공식 규약대로 `ch`/`ocr`로 정규화됨.
 - OpenAPI 제목/설명, README, `ArticleNode.headline` 문서를 일반 문서용 (section heading) 표현으로 재구성하여 특정 언어/신문 가정을 소비자에게 노출하지 않도록 함. 응답 스키마 필드는 하위 호환을 위해 변경하지 않음.

tests/test_tools_extract_headlines.py

@@ -0,0 +1,122 @@
+import json
+import pytest
+from pathlib import Path
+from tools.extract_headlines import extract_headlines, main
+
+
+def test_extract_headlines_valid(tmp_path: Path):
+    input_path = tmp_path / "input.json"
+    output_path = tmp_path / "output.txt"
+    data = {
+        "document_id": "test",
+        "pages": [
+            {
+                "page_number": 1,
+                "articles": [
+                    {"article_id": "a1", "headline": "Headline 1", "body_blocks": []},
+                    {"article_id": "a2", "headline": "Headline 2", "body_blocks": []},
+                ],
+            }
+        ],
+        "quality": {"status": "success", "parser": "test", "warnings": []},
+    }
+    input_path.write_text(json.dumps(data), encoding="utf-8")
+    extract_headlines(input_path, output_path)
+    result = output_path.read_text(encoding="utf-8")
+    assert result == "Headline 1\nHeadline 2\n"
+
+
+def test_extract_headlines_file_not_found(tmp_path: Path):
+    with pytest.raises(FileNotFoundError):
+        extract_headlines(tmp_path / "nonexistent.json", tmp_path / "output.txt")
+
+
+def test_extract_headlines_invalid_extension(tmp_path: Path):
+    invalid_file = tmp_path / "input.txt"
+    invalid_file.touch()
+    with pytest.raises(ValueError, match="must be a .json file"):
+        extract_headlines(invalid_file, tmp_path / "output.txt")
+
+
+def test_extract_headlines_invalid_json(tmp_path: Path):
+    input_path = tmp_path / "input.json"
+    input_path.write_text("invalid json", encoding="utf-8")
+    with pytest.raises(ValueError, match="Invalid JSON file"):
+        extract_headlines(input_path, tmp_path / "output.txt")
+
+
+def test_extract_headlines_invalid_schema(tmp_path: Path):
+    input_path = tmp_path / "input.json"
+    input_path.write_text('{"invalid": "schema"}', encoding="utf-8")
+    with pytest.raises(ValueError, match="does not match ParseResponse schema"):
+        extract_headlines(input_path, tmp_path / "output.txt")
+
+
+def test_main_valid(tmp_path: Path, capsys: pytest.CaptureFixture[str]):
+    input_path = tmp_path / "input.json"
+    output_path = tmp_path / "output.txt"
+    data = {
+        "document_id": "test",
+        "pages": [],
+        "quality": {"status": "success", "parser": "test", "warnings": []},
+    }
+    input_path.write_text(json.dumps(data), encoding="utf-8")
+    main([str(input_path), "-o", str(output_path)])
+    assert f"Headlines successfully written to {output_path}" in capsys.readouterr().out
+
+
+def test_main_error(tmp_path: Path, capsys: pytest.CaptureFixture[str]):
+    with pytest.raises(SystemExit):
+        main([str(tmp_path / "nonexistent.json"), "-o", str(tmp_path / "output.txt")])
+    assert "Error extracting headlines" in capsys.readouterr().err
+
+
+import sys  # noqa: E402
+import importlib  # noqa: E402
+import runpy  # noqa: E402
+
+
+def test_extract_headlines_import_path(monkeypatch):
+    """Test sys.path injection for coverage."""
+    import tools.extract_headlines
+
+    target_path = str(
+        Path(tools.extract_headlines.__file__).resolve().parents[1] / "src"
+    )
+    monkeypatch.setattr(sys, "path", [p for p in sys.path if p != target_path])
+    importlib.reload(tools.extract_headlines)
+    assert target_path in sys.path
+
+
+def test_extract_headlines_main_module():
+    """Test if __name__ == '__main__' block coverage via run_module."""
+    import warnings
+
+    with warnings.catch_warnings():
+        warnings.filterwarnings("ignore", category=RuntimeWarning)
+        try:
+            runpy.run_module("tools.extract_headlines", run_name="__main__")
+        except SystemExit:
+            pass
+
+
+def test_extract_headlines_empty_headline(tmp_path: Path):
+    input_path = tmp_path / "input.json"
+    output_path = tmp_path / "output.txt"
+    data = {
+        "document_id": "test",
+        "pages": [
+            {
+                "page_number": 1,
+                "articles": [
+                    {"article_id": "a1", "headline": "", "body_blocks": []},
+                    {"article_id": "a2", "headline": "Headline 2", "body_blocks": []},
+                ],
+            }
+        ],
+        "quality": {"status": "success", "parser": "test", "warnings": []},
+    }
+    input_path.write_text(json.dumps(data), encoding="utf-8")
+    extract_headlines(input_path, output_path)
+    result = output_path.read_text(encoding="utf-8")
+    assert result == "Headline 2\n"

tests/test_tools_minify_dom.py

@@ -0,0 +1,96 @@
+import json
+import pytest
+from pathlib import Path  # noqa: E402
+from tools.minify_dom import minify_dom, main
+
+
+def test_minify_dom_valid(tmp_path: Path):
+    input_path = tmp_path / "input.json"
+    output_path = tmp_path / "output.json"
+    data = {
+        "document_id": "test",
+        "pages": [],
+        "quality": {"status": "success", "parser": "test", "warnings": []},
+    }
+    input_path.write_text(json.dumps(data, indent=4), encoding="utf-8")
+    minify_dom(input_path, output_path)
+    result = output_path.read_text(encoding="utf-8")
+    assert " " not in result
+    assert "\n" not in result
+    assert json.loads(result) == data
+
+
+def test_minify_dom_file_not_found(tmp_path: Path):
+    with pytest.raises(FileNotFoundError):
+        minify_dom(tmp_path / "nonexistent.json", tmp_path / "output.json")
+
+
+def test_minify_dom_invalid_extension(tmp_path: Path):
+    invalid_file = tmp_path / "input.txt"
+    invalid_file.touch()
+    with pytest.raises(ValueError, match="must be a .json file"):
+        minify_dom(invalid_file, tmp_path / "output.json")
+
+
+def test_minify_dom_invalid_json(tmp_path: Path):
+    input_path = tmp_path / "input.json"
+    input_path.write_text("invalid json", encoding="utf-8")
+    with pytest.raises(ValueError, match="Invalid JSON file"):
+        minify_dom(input_path, tmp_path / "output.json")
+
+
+def test_minify_dom_invalid_schema(tmp_path: Path):
+    input_path = tmp_path / "input.json"
+    input_path.write_text('{"invalid": "schema"}', encoding="utf-8")
+    with pytest.raises(ValueError, match="does not match ParseResponse schema"):
+        minify_dom(input_path, tmp_path / "output.json")
+
+
+def test_main_valid(tmp_path: Path, capsys: pytest.CaptureFixture[str]):
+    input_path = tmp_path / "input.json"
+    output_path = tmp_path / "output.json"
+    data = {
+        "document_id": "test",
+        "pages": [],
+        "quality": {"status": "success", "parser": "test", "warnings": []},
+    }
+    input_path.write_text(json.dumps(data), encoding="utf-8")
+    main([str(input_path), "-o", str(output_path)])
+    assert (
+        f"Minified JSON successfully written to {output_path}"
+        in capsys.readouterr().out
+    )
+
+
+def test_main_error(tmp_path: Path, capsys: pytest.CaptureFixture[str]):
+    with pytest.raises(SystemExit):
+        main([str(tmp_path / "nonexistent.json"), "-o", str(tmp_path / "output.json")])
+    assert "Error minifying JSON file" in capsys.readouterr().err
+
+
+import sys  # noqa: E402
+import importlib  # noqa: E402
+import runpy  # noqa: E402
+from pathlib import Path  # noqa: E402
+
+
+def test_minify_dom_import_path(monkeypatch):
+    """Test sys.path injection for coverage."""
+    import tools.minify_dom
+
+    target_path = str(Path(tools.minify_dom.__file__).resolve().parents[1] / "src")
+    monkeypatch.setattr(sys, "path", [p for p in sys.path if p != target_path])
+    importlib.reload(tools.minify_dom)
+    assert target_path in sys.path
+
+
+def test_minify_dom_main_module():
+    """Test if __name__ == '__main__' block coverage via run_module."""
+    import warnings
+
+    with warnings.catch_warnings():
+        warnings.filterwarnings("ignore", category=RuntimeWarning)
+        try:
+            runpy.run_module("tools.minify_dom", run_name="__main__")
+        except SystemExit:
+            pass

tools/extract_headlines.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+import argparse
+import json
+import sys
+from pathlib import Path
+
+_REPO_ROOT = Path(__file__).resolve().parents[1]
+_SRC_ROOT = _REPO_ROOT / "src"
+if str(_SRC_ROOT) not in sys.path:
+    sys.path.insert(0, str(_SRC_ROOT))
+from pydantic import ValidationError  # noqa: E402
+from newsdom_api.schemas import ParseResponse  # noqa: E402
+
+
+def extract_headlines(json_path: Path, output_path: Path) -> None:
+    if not json_path.is_file():
+        raise FileNotFoundError(f"File not found or is not a file: {json_path}")
+    if json_path.suffix.lower() != ".json":
+        raise ValueError(f"Input file must be a .json file: {json_path}")
+    try:
+        data = json.loads(json_path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"Invalid JSON file ({json_path}): {exc}") from exc
+    try:
+        ParseResponse.model_validate(data)
+    except ValidationError as exc:
+        raise ValueError(
+            f"File {json_path} does not match ParseResponse schema: {exc}"
+        ) from exc
+    headlines = []
+    for page in data.get("pages", []):
+        for article in page.get("articles", []):
+            headline = article.get("headline")
+            if headline:
+                headlines.append(headline)
+    output_path.write_text("\n".join(headlines) + "\n", encoding="utf-8")
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(
+        description="Extract headlines from a NewsDOM JSON file."
+    )
+    parser.add_argument("input", type=Path, help="Path to the input JSON file.")
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=Path,
+        required=True,
+        help="Path to write the headlines text file.",
+    )
+    args = parser.parse_args(argv)
+    try:
+        extract_headlines(args.input, args.output)
+        print(f"Headlines successfully written to {args.output}")
+    except Exception as exc:
+        print(f"Error extracting headlines: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

tools/minify_dom.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+import argparse
+import json
+import sys
+from pathlib import Path
+
+_REPO_ROOT = Path(__file__).resolve().parents[1]
+_SRC_ROOT = _REPO_ROOT / "src"
+if str(_SRC_ROOT) not in sys.path:
+    sys.path.insert(0, str(_SRC_ROOT))
+from pydantic import ValidationError  # noqa: E402
+from newsdom_api.schemas import ParseResponse  # noqa: E402
+
+
+def minify_dom(json_path: Path, output_path: Path) -> None:
+    if not json_path.is_file():
+        raise FileNotFoundError(f"File not found or is not a file: {json_path}")
+    if json_path.suffix.lower() != ".json":
+        raise ValueError(f"Input file must be a .json file: {json_path}")
+    try:
+        data = json.loads(json_path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"Invalid JSON file ({json_path}): {exc}") from exc
+    try:
+        ParseResponse.model_validate(data)
+    except ValidationError as exc:
+        raise ValueError(
+            f"File {json_path} does not match ParseResponse schema: {exc}"
+        ) from exc
+    output_path.write_text(
+        json.dumps(data, ensure_ascii=False, separators=(",", ":")), encoding="utf-8"
+    )
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(description="Minify a NewsDOM JSON file.")
+    parser.add_argument("input", type=Path, help="Path to the input JSON file.")
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=Path,
+        required=True,
+        help="Path to write the minified JSON output file.",
+    )
+    args = parser.parse_args(argv)
+    try:
+        minify_dom(args.input, args.output)
+        print(f"Minified JSON successfully written to {args.output}")
+    except Exception as exc:
+        print(f"Error minifying JSON file: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()