From 0d2e826dd3505e283de8e61fe01ed5bb82ff6896 Mon Sep 17 00:00:00 2001 From: Jeuner <62662523+Jeuners@users.noreply.github.com> Date: Mon, 7 Sep 2026 18:26:02 +0200 Subject: [PATCH] feat: read full article text via a new read_article tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Löst die RSS-Beschränkung: Feed-Einträge liefern nur eine kurze Teaser-Beschreibung, nie den vollen Artikeltext. - astra/articles.py: lädt eine URL, extrahiert den reinen Fließtext via trafilatura (ohne Navigation/Werbung/Boilerplate). Live gegen einen echten RP-ONLINE-Artikel verifiziert (1682 Zeichen sauberer Text). - read_article-Tool in tools.py, gleiches tool_start/tool_result/ tool_error-Muster wie die anderen beiden Tools. Beschreibung nennt explizit den erwarteten Anwendungsfall (Nachfrage zu einer schon genannten Schlagzeile) — konsequent nach der zuletzt beschlossenen Regel, Tool-Verhalten nur in der FunctionSchema, nie im SYSTEM_PROMPT. Gemessen: mit drei Tools sinkt read_news weiter auf ca. 15-20% (vorher mit zwei Tools 20-50%). read_article selbst liegt bei einer konkreten Nachfrage zu einer schon im Kontext stehenden Schlagzeile bei ~65% (2/3 in einem Live-Lauf gegen echtes Ollama, echter Artikeltext erfolgreich extrahiert; der eine Fehlschlag gab ehrlich zu, den Volltext nicht zu kennen, statt zu erfinden). NICHT gepusht — auf Anweisung bis auf Weiteres nur lokal. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LVgSHNHdRx3UNTBodFmhRA --- README.md | 20 +++-- astra/articles.py | 36 ++++++++ astra/tools.py | 38 ++++++++- pyproject.toml | 1 + tests/test_articles.py | 72 ++++++++++++++++ tests/test_tools.py | 49 +++++++++++ uv.lock | 189 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 398 insertions(+), 7 deletions(-) create mode 100644 astra/articles.py create mode 100644 tests/test_articles.py diff --git a/README.md b/README.md index 53a4824..0679363 100644 --- a/README.md +++ b/README.md @@ -60,17 +60,25 @@ nur für die Dauer der Session, nichts wird auf Disk geschrieben. - Tool `read_news` in `astra/tools.py`: Astra ruft es auf, wenn nach aktuellen Nachrichten gefragt wird, und fasst die Schlagzeilen mündlich zusammen statt sie roh vorzulesen. +- **`astra/articles.py`** — lädt eine Artikel-URL und extrahiert den reinen + Fließtext (`trafilatura`), ohne Navigation/Werbung/Boilerplate. +- Tool `read_article` in `astra/tools.py`: Astra ruft es auf, wenn der + Nutzer zu einer schon genannten Schlagzeile mehr wissen will, und liest + den vollen Artikeltext statt nur der RSS-Kurzbeschreibung. Tool-Verhalten steht bewusst ausschließlich in der jeweiligen `FunctionSchema.description` (siehe `astra/tools.py`), nicht im `SYSTEM_PROMPT` — eine Quelle der Wahrheit pro Werkzeug statt duplizierter Regeln in einem wachsenden globalen Prompt. Gemessener Preis davon: mit -zwei gleichzeitig verfügbaren Tools (Bild + Nachrichten) ruft das lokale -9,7B-Modell `read_news` nur noch in ca. 20–50 % der Fälle tatsächlich auf -(vorher, mit Tool-Regeln zusätzlich im System-Prompt, ca. 60–75 %) und -erfindet sonst Schlagzeilen. Bildgenerierung bleibt bei ~100 % zuverlässig. -Bekannte Grenze eines kleinen lokalen Modells bei Tool-Konkurrenz, kein -Bug — die saubere Trennung war eine bewusste Architekturentscheidung. +drei gleichzeitig verfügbaren Tools (Bild + Nachrichten + Artikel) ruft +das lokale 9,7B-Modell `read_news` nur noch in ca. 15–20 % der Fälle +tatsächlich auf (vorher mit zwei Tools ca. 20–50 %, mit Tool-Regeln +zusätzlich im System-Prompt ca. 60–75 %) und erfindet sonst Schlagzeilen. +`read_article` ist bei einer konkreten Nachfrage zu einer schon genannten +Schlagzeile brauchbarer (~65 %, vermutlich weil der Kontext dort weniger +mehrdeutig ist). Bildgenerierung bleibt bei ~100 % zuverlässig. Bekannte +Grenze eines kleinen lokalen Modells bei Tool-Konkurrenz, kein Bug — die +saubere Trennung war eine bewusste Architekturentscheidung. ## Starten diff --git a/astra/articles.py b/astra/articles.py new file mode 100644 index 0000000..2f19793 --- /dev/null +++ b/astra/articles.py @@ -0,0 +1,36 @@ +"""Fetch a web page and extract its clean article text. Knows nothing about +the voice pipeline or where the URL came from. +""" + +import asyncio + +import httpx +import trafilatura + +USER_AGENT = "Mozilla/5.0 (compatible; AstraVoice/1.0; +https://github.com/Jeuners/astra-vision)" + + +class ArticleError(Exception): + """Raised when a page can't be fetched or no article text could be extracted.""" + + +async def read_article(url: str, *, max_chars: int = 4000) -> str: + """Fetch `url` and return its main article text, stripped of navigation, + ads, and other boilerplate. Extraction runs in a thread since trafilatura + is CPU-bound and synchronous. + """ + async with httpx.AsyncClient( + headers={"User-Agent": USER_AGENT}, timeout=15, follow_redirects=True + ) as client: + try: + response = await client.get(url) + response.raise_for_status() + except httpx.HTTPError as exc: + raise ArticleError(f"Seite nicht erreichbar: {exc}") from exc + + text = await asyncio.to_thread( + trafilatura.extract, response.text, include_comments=False, include_tables=False + ) + if not text or not text.strip(): + raise ArticleError("Auf der Seite wurde kein lesbarer Artikeltext gefunden.") + return text.strip()[:max_chars] diff --git a/astra/tools.py b/astra/tools.py index 9a3198b..f0e7963 100644 --- a/astra/tools.py +++ b/astra/tools.py @@ -8,6 +8,7 @@ from collections.abc import Callable from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema +from astra.articles import ArticleError, read_article from astra.comfyui import ComfyUIError, generate_image from astra.core import Settings from astra.feeds import TOPICS @@ -58,6 +59,39 @@ def build_tools(config: Settings, notify: Callable[[dict], None], media_store: d } ) + async def handle_read_article(params): + url = params.arguments.get("url", "") + notify({"type": "activity", "text": "Astra liest den Artikel …"}) + notify({"type": "tool_start", "text": f'Lade Artikel: "{url}"'}) + try: + text = await read_article(url) + except ArticleError as exc: + notify({"type": "activity", "text": "Artikel konnte nicht geladen werden."}) + notify({"type": "tool_error", "text": str(exc)}) + await params.result_callback({"error": str(exc)}) + return + notify({"type": "tool_result", "text": text[:800]}) + await params.result_callback({"status": "ok", "text": text}) + + read_article_schema = FunctionSchema( + name="read_article", + description=( + "Lade eine Artikel-URL (z.B. aus einem vorherigen read_news-Ergebnis) und lies " + "den vollständigen Artikeltext. Rufe dieses Werkzeug auf, wenn der Nutzer zu " + "einer bereits genannten Schlagzeile mehr Details wissen will — erfinde selbst " + "keine Details. Fasse den Text danach mündlich in eigenen Worten zusammen, " + "lies ihn nicht roh vor." + ), + properties={ + "url": { + "type": "string", + "description": "Die 'link'-URL des Artikels aus dem read_news-Ergebnis.", + }, + }, + required=["url"], + handler=handle_read_article, + ) + read_news_schema = FunctionSchema( name="read_news", description=( @@ -104,4 +138,6 @@ def build_tools(config: Settings, notify: Callable[[dict], None], media_store: d required=["prompt"], handler=handle_generate_image, ) - return ToolsSchema(standard_tools=[generate_image_schema, read_news_schema]) + return ToolsSchema( + standard_tools=[generate_image_schema, read_news_schema, read_article_schema] + ) diff --git a/pyproject.toml b/pyproject.toml index 4b250eb..88921d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "pypdf", "python-multipart", "feedparser", + "trafilatura", ] [dependency-groups] diff --git a/tests/test_articles.py b/tests/test_articles.py new file mode 100644 index 0000000..48c20d7 --- /dev/null +++ b/tests/test_articles.py @@ -0,0 +1,72 @@ +import httpx +import pytest + +from astra.articles import ArticleError, read_article + +SAMPLE_HTML = b""" +Testartikel + + +
+

Ein wichtiger Titel

+

Dies ist der erste Absatz des eigentlichen Artikeltexts, lang genug fuer +eine sinnvolle Extraktion durch trafilatura, mit mehreren Saetzen.

+

Und hier folgt ein zweiter Absatz mit weiterem Inhalt, damit die +Extraktion genug Text zum Erkennen des Hauptinhalts hat.

+
+ +""" + + +def _patched_client(monkeypatch, handler): + real_async_client = httpx.AsyncClient + + def factory(*args, **kwargs): + kwargs["transport"] = httpx.MockTransport(handler) + return real_async_client(**kwargs) + + monkeypatch.setattr(httpx, "AsyncClient", factory) + + +@pytest.mark.asyncio +async def test_read_article_extracts_main_text(monkeypatch): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=SAMPLE_HTML) + + _patched_client(monkeypatch, handler) + text = await read_article("https://example.com/article") + + assert "wichtiger Titel" in text or "erste Absatz" in text + assert "Navigation" not in text + assert "Footer-Boilerplate" not in text + + +@pytest.mark.asyncio +async def test_read_article_respects_max_chars(monkeypatch): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=SAMPLE_HTML) + + _patched_client(monkeypatch, handler) + text = await read_article("https://example.com/article", max_chars=20) + + assert len(text) <= 20 + + +@pytest.mark.asyncio +async def test_read_article_raises_when_page_unreachable(monkeypatch): + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("refused", request=request) + + _patched_client(monkeypatch, handler) + with pytest.raises(ArticleError): + await read_article("https://example.com/article") + + +@pytest.mark.asyncio +async def test_read_article_raises_when_no_text_extractable(monkeypatch): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=b"") + + _patched_client(monkeypatch, handler) + with pytest.raises(ArticleError): + await read_article("https://example.com/article") diff --git a/tests/test_tools.py b/tests/test_tools.py index 552bb09..348862c 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -1,6 +1,7 @@ import pytest import astra.tools as tools_module +from astra.articles import ArticleError from astra.comfyui import ComfyUIError from astra.core import Settings from astra.rss import Entry, RSSError @@ -124,3 +125,51 @@ async def test_read_news_tool_reports_rss_errors(monkeypatch): "tool_error", ] assert "error" in params.results[0] + + +@pytest.mark.asyncio +async def test_read_article_tool_reports_extracted_text(monkeypatch): + async def fake_read_article(url, **kwargs): + assert url == "https://example.com/story" + return "Der vollständige Artikeltext." + + monkeypatch.setattr(tools_module, "read_article", fake_read_article) + + notifications = [] + schema = _find( + tools_module.build_tools(Settings(), notifications.append, {}).standard_tools, + "read_article", + ) + + params = FakeFunctionCallParams({"url": "https://example.com/story"}) + await schema.handler(params) + + assert [n["type"] for n in notifications] == ["activity", "tool_start", "tool_result"] + assert notifications[2]["text"] == "Der vollständige Artikeltext." + assert params.results[0]["status"] == "ok" + assert params.results[0]["text"] == "Der vollständige Artikeltext." + + +@pytest.mark.asyncio +async def test_read_article_tool_reports_article_errors(monkeypatch): + async def failing_read_article(url, **kwargs): + raise ArticleError("Seite nicht erreichbar.") + + monkeypatch.setattr(tools_module, "read_article", failing_read_article) + + notifications = [] + schema = _find( + tools_module.build_tools(Settings(), notifications.append, {}).standard_tools, + "read_article", + ) + + params = FakeFunctionCallParams({"url": "https://example.com/story"}) + await schema.handler(params) + + assert [n["type"] for n in notifications] == [ + "activity", + "tool_start", + "activity", + "tool_error", + ] + assert "error" in params.results[0] diff --git a/uv.lock b/uv.lock index bad9670..99c3f8c 100644 --- a/uv.lock +++ b/uv.lock @@ -143,6 +143,7 @@ dependencies = [ { name = "pipecat-ai", extra = ["pocket-tts", "webrtc"] }, { name = "pypdf" }, { name = "python-multipart" }, + { name = "trafilatura" }, { name = "uvicorn" }, ] @@ -164,6 +165,7 @@ requires-dist = [ { name = "pipecat-ai", extras = ["silero", "webrtc", "pocket-tts"] }, { name = "pypdf" }, { name = "python-multipart" }, + { name = "trafilatura" }, { name = "uvicorn" }, ] @@ -203,6 +205,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/66/47/cd9ae0edf2206351c1251bb94b5ec58728e42c5f6ee16c03c412f3a1bb3e/av-17.1.0-cp311-abi3-win_arm64.whl", hash = "sha256:ee98534242a74da847af78624779ac5a3177dc7c69f956a4da9e6f0fdb37d7f6", size = 21174601, upload-time = "2026-06-07T05:52:28.077Z" }, ] +[[package]] +name = "babel" +version = "2.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554, upload-time = "2026-02-01T12:30:56.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, +] + [[package]] name = "beartype" version = "0.22.9" @@ -312,6 +323,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "courlan" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "babel" }, + { name = "tld" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/16/2a771612ee0b3acaa95ac21cc7e8a3319e815d6360f8ffc5987d1ce28499/courlan-1.4.0.tar.gz", hash = "sha256:fbbac7b7fcde2195ea08e707609503c81cf39c891e8d26cdb1fed4585782d63d", size = 208997, upload-time = "2026-06-01T17:30:17.306Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/38/ce65091ff20a16e06d17418c4353af5f56d3190821b1a06983c79ae79274/courlan-1.4.0-py3-none-any.whl", hash = "sha256:ad1dbdefd912ca7238d4607dc855df5df097f56bac175dd662c84eed3802f49e", size = 34193, upload-time = "2026-06-01T17:30:14.984Z" }, +] + [[package]] name = "coverage" version = "7.16.0" @@ -442,6 +467,21 @@ nvtx = [ { name = "nvidia-nvtx", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] +[[package]] +name = "dateparser" +version = "1.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "regex" }, + { name = "tzlocal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/5d/bd21ba1519b6b1e222b29878301d2e1fb928e890dc7d085fa4222ac5671b/dateparser-1.4.3.tar.gz", hash = "sha256:bab8c43a746266e68142f4926e69438ce551441aa88e54e78bb6410bf3ee7000", size = 362152, upload-time = "2026-09-03T10:07:54.545Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/5f/63f927b5ffd1cc2c030fc2bd362bf920890e8806cf42f701198de5b3f9c7/dateparser-1.4.3-py3-none-any.whl", hash = "sha256:cbce86c64e0cea5c54c84c015d34c903d46c51bfad50632bf2f05a6141d10c05", size = 322384, upload-time = "2026-09-03T10:07:52.913Z" }, +] + [[package]] name = "defusedxml" version = "0.7.1" @@ -637,6 +677,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, ] +[[package]] +name = "htmldate" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "charset-normalizer" }, + { name = "dateparser" }, + { name = "lxml" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/1f/e7cf83e23d7b68105de8b874a8b36ba23b450d6f71388583e4ca3ce475ca/htmldate-1.10.0.tar.gz", hash = "sha256:a38df10772ab5d7dbb11896e3f6a852a8491fb1b0965465bc174e23fc2baae58", size = 44455, upload-time = "2026-06-01T17:43:53.437Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/17/d3356233c826c641f940983d9479eab27faec59d49f4070bc58e80fcc021/htmldate-1.10.0-py3-none-any.whl", hash = "sha256:9211dae35ab94147c8ed9e5fc2c9287a5cf31d2394cb7857e7f5dd814eb2aad6", size = 31561, upload-time = "2026-06-01T17:43:51.797Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -762,6 +818,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/53/84099323c2ec4be98d935f63c033ac4151ee83836ca1050ede3b3aadf155/joblib-1.6.0-py3-none-any.whl", hash = "sha256:3dbbf9f6e4b592a2357b854608e980fe6390d131d7a82f011a377ef2ebef7aba", size = 306115, upload-time = "2026-08-31T09:39:02.298Z" }, ] +[[package]] +name = "justext" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml", extra = ["html-clean"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/f3/45890c1b314f0d04e19c1c83d534e611513150939a7cf039664d9ab1e649/justext-3.0.2.tar.gz", hash = "sha256:13496a450c44c4cd5b5a75a5efcd9996066d2a189794ea99a49949685a0beb05", size = 828521, upload-time = "2025-02-25T20:21:49.934Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/ac/52f4e86d1924a7fc05af3aeb34488570eccc39b4af90530dd6acecdf16b5/justext-3.0.2-py2.py3-none-any.whl", hash = "sha256:62b1c562b15c3c6265e121cc070874243a443bfd53060e869393f09d6b6cc9a7", size = 837940, upload-time = "2025-02-25T20:21:44.179Z" }, +] + [[package]] name = "llvmlite" version = "0.49.0" @@ -804,6 +872,49 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4c/d4/ca7ef29878c7210b116cac40d53adc89622a78cb194580154f63bb7dfb01/loudness-0.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:fac6b39d3b7c94efce9539221812dbfcdc2e237308561faea2abfc5de0fa1d95", size = 100020, upload-time = "2025-12-26T15:19:58.25Z" }, ] +[[package]] +name = "lxml" +version = "6.1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/23/ad/28ecd7cb894d172f3c9c80a075eeeb2017ac62e3632cee05a5f9493547eb/lxml-6.1.3.tar.gz", hash = "sha256:45222d94ddd511536f3b2f7d9deae3b2339b4ce0f075f1ca25703b07cad9dd21", size = 4211198, upload-time = "2026-09-02T14:48:02.287Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/1f/a180b57d9eeabaab77f9d5aa30356898ea749c4795596a8f66d1eb6bef2e/lxml-6.1.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0c0710ac085a157b593c38fbcacd950f15c4afa8e2057527185875ab302752bc", size = 8602094, upload-time = "2026-09-02T14:47:26.054Z" }, + { url = "https://files.pythonhosted.org/packages/a8/25/070c92013a1c029a602b03560d68772313d918268667fa993da7961759c9/lxml-6.1.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:623c8799c17128753c65699f1c3aa32402657393a9ad6db09ed8b98ddf76611d", size = 4638308, upload-time = "2026-09-02T14:47:29.587Z" }, + { url = "https://files.pythonhosted.org/packages/1e/1c/722e88883173097a1a375153e3c2447eba3060d0231522cf6596e99f4195/lxml-6.1.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f683dc6300317700025e41d89a43e0276692ded16113a3c43eab704d605c58e5", size = 4939696, upload-time = "2026-09-02T14:47:32.997Z" }, + { url = "https://files.pythonhosted.org/packages/db/36/aa413bc214dc4f785ad2b2ddd8cc99aae7062d49ab155e91e6011af00daf/lxml-6.1.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:379f8a75cf6eb7eef0af074b55f49ab73b868388a98de14646abcdfa4564bb11", size = 5105247, upload-time = "2026-09-02T14:47:36.734Z" }, + { url = "https://files.pythonhosted.org/packages/a3/a0/a1f7f1313795bfec67b77f01ef3b1128d49f2d7f66a8413fa55d47f4e25f/lxml-6.1.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b37772102d44bb6628186accca3a121b1fa3a6b3d97518a8c29a5229ca4c0d0a", size = 5011915, upload-time = "2026-09-02T14:47:39.846Z" }, + { url = "https://files.pythonhosted.org/packages/b9/78/840e7e3f1d0cc7a5cfac5d8505b97e25b6427fd774ac4bae672aaebfb4b5/lxml-6.1.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ddcf547bea2aee967d6a77779376a45e77e610e8465147a1f3d7e20d539d6e32", size = 5638175, upload-time = "2026-09-02T14:47:43.644Z" }, + { url = "https://files.pythonhosted.org/packages/0a/20/e022dbc6b4753a9bc9fc5fb28a27163430c1731b9913997f6544c1b2518c/lxml-6.1.3-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:909f4e927bb051f7740d6367285fc60cdcfdaf0258c2dba4ff5ba7eadadc250c", size = 5244675, upload-time = "2026-09-02T14:47:47.635Z" }, + { url = "https://files.pythonhosted.org/packages/99/83/82cde81d2b5eb38d1539fdfdf318abdd014a7e604f4df01c9cd3deb18f2a/lxml-6.1.3-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:a5c18810318303ce9afb3f95e2ddb54834f96fa699a8600433fd5a93dcf44c56", size = 5358205, upload-time = "2026-09-02T14:47:50.306Z" }, + { url = "https://files.pythonhosted.org/packages/d2/a1/f3b057371c8cb29f2a9c9c44ea320592446e40b74a4b0af68c3d8e65bc73/lxml-6.1.3-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:3e42265103fb385d8642a78672edf376c6f7e1d3598a7a4f9cb1278f2f6b5f6f", size = 4704495, upload-time = "2026-09-02T14:47:53.251Z" }, + { url = "https://files.pythonhosted.org/packages/1a/a4/230eb28be5d412152ffc3c679b51fe1aeede5a53f3a8eb6e9748f2f4754f/lxml-6.1.3-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:21402998e4b78e7cce237d2788841aaa21ac9a4d1574d04dc2d12ee41ae807b5", size = 5255117, upload-time = "2026-09-02T14:47:55.963Z" }, + { url = "https://files.pythonhosted.org/packages/a3/18/1969f56763af24ce42ea156007b0b2d73fddea552e283b2010416394f0f4/lxml-6.1.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:38fc4e4e4e084e0bd491949482527d406788045c546d4f8789e93fc527b91385", size = 5054424, upload-time = "2026-09-02T14:47:58.131Z" }, + { url = "https://files.pythonhosted.org/packages/f4/d4/2a90acc1f6fabaa3a8db9340437822bd8d041b205d626a4b3e8621aaa390/lxml-6.1.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5609efdb0d3c95499c00046bc53648b3482ec2175b5503d6e611b3f0555dc71d", size = 4785572, upload-time = "2026-09-02T14:48:01.029Z" }, + { url = "https://files.pythonhosted.org/packages/a5/1e/b90e845b1dcd0f2f3f26b98283d857f25909223aacd265eee032c34ab8b1/lxml-6.1.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:97ce49699d87ebf8aad631b55d65b33219a4f1bfefbbf5bff19dc9af160aeaf9", size = 5656516, upload-time = "2026-09-02T14:48:03.419Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ab/0a1b802c57f3fba5c4efd77d5c6b78adaa8f7b681f0c90456b140fe8bf6c/lxml-6.1.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:48542c9acba9ff9450bd18d871d2c2c8787fdb283572b623d206f1b927cd7d9e", size = 5245982, upload-time = "2026-09-02T14:48:06.109Z" }, + { url = "https://files.pythonhosted.org/packages/da/ee/2c016fbceb3778137459292538d9dfa7e3ad9070fe409c15254ddd90d2cc/lxml-6.1.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c55e71a9b1db1f107efb60da49c093689b74c5c31a708e5379e2fd9439d4fbb5", size = 5267340, upload-time = "2026-09-02T14:48:08.374Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b1/736d18fd6f0835761923b7bac1f0c27d60c1200384e9093f05d8c5100525/lxml-6.1.3-cp312-cp312-win32.whl", hash = "sha256:b3ff39654f0ce6ebd4db154211136dbe7e8157bcc3bed2344c87f32c7c6ecb6c", size = 3602606, upload-time = "2026-09-02T14:48:10.384Z" }, + { url = "https://files.pythonhosted.org/packages/3a/5b/6ed903e4e6278a020c8a6f0dbbe78030d041840a6b4a64ea441a1e414077/lxml-6.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:3e9a00d1c2c30936f7add097c41afc5da6556c580909104aafd382cac92a855c", size = 4005999, upload-time = "2026-09-02T14:48:12.51Z" }, + { url = "https://files.pythonhosted.org/packages/e4/1b/7bcebb7b6332cb3ae85e9c13b139adb6f23f75c71d84041c56a5005d9a29/lxml-6.1.3-cp312-cp312-win_arm64.whl", hash = "sha256:1aeca87830c4fe649dcf93fe2b059525b71c72587f21be4ae4af7103082a79fa", size = 3666631, upload-time = "2026-09-02T14:48:14.567Z" }, +] + +[package.optional-dependencies] +html-clean = [ + { name = "lxml-html-clean" }, +] + +[[package]] +name = "lxml-html-clean" +version = "0.4.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0a/63/195dfdde380a84df309e3bccf4384b034b745dba43426886f7ae623b4fba/lxml_html_clean-0.4.5.tar.gz", hash = "sha256:e2a4c7d5beedd17cd7b484d848a0571e54baa239a4f9df5546e3acba7f990560", size = 24142, upload-time = "2026-05-20T12:17:53.574Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/bd/6e2b76a6c5dee10397db9c929f0c5066766ec1036046f0335b7ca7ca08b8/lxml_html_clean-0.4.5-py3-none-any.whl", hash = "sha256:c76fcadd1e5bfb9b8bafc2200d51e4e78eb0dad67f56881c21dfb6484c7e7746", size = 14573, upload-time = "2026-05-20T12:17:52.215Z" }, +] + [[package]] name = "markdown" version = "3.10.3" @@ -1556,6 +1667,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + [[package]] name = "python-multipart" version = "0.0.32" @@ -1565,6 +1688,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, ] +[[package]] +name = "pytz" +version = "2026.3.post1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fb/48/fb042503b6ca6cd271261dc559fd6432f7d8c713153e9ec5c591af4dfc1c/pytz-2026.3.post1.tar.gz", hash = "sha256:2211d3fcf9a797d3405cac96ac7f61d80e6a644f72a3309607282fe8a2010c5d", size = 319745, upload-time = "2026-07-25T15:12:07.385Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/7b/39c34ca613b0b198cb866466651b26b045e2009864c5183c979a3b83f383/pytz-2026.3.post1-py2.py3-none-any.whl", hash = "sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815", size = 508283, upload-time = "2026-07-25T15:12:05.782Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1751,6 +1883,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + [[package]] name = "sniffio" version = "1.3.1" @@ -1817,6 +1958,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, ] +[[package]] +name = "tld" +version = "0.13.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/5d/76b4383ac4e5b5e254e50c09807b3e13820bed6d6c11cd540264988d6802/tld-0.13.2.tar.gz", hash = "sha256:d983fa92b9d717400742fca844e29d5e18271079c7bcfabf66d01b39b4a14345", size = 467175, upload-time = "2026-03-06T23:50:34.498Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/90/39a85a4b63c84213e78b3c17d22e1bf45328acf8ebb33ef93be30d0a3911/tld-0.13.2-py2.py3-none-any.whl", hash = "sha256:9b8fdbdb880e7ba65b216a4937f2c94c49a7226723783d5838fc958ac76f4e0c", size = 296743, upload-time = "2026-03-06T23:50:32.465Z" }, +] + [[package]] name = "tokenizers" version = "0.23.2" @@ -1883,6 +2033,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953", size = 80184, upload-time = "2026-07-27T11:33:13.167Z" }, ] +[[package]] +name = "trafilatura" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "courlan" }, + { name = "htmldate" }, + { name = "justext" }, + { name = "lxml" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/96/737133a93e73e967f9c888e6cfb1f2c31b2083d27263edb19fd65a9aca02/trafilatura-2.2.0.tar.gz", hash = "sha256:8c2cabb84066465228d03183fb698ce0b1245b81c58140b8ae0de57fddf3aae7", size = 314748, upload-time = "2026-07-31T16:06:49.444Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/01/af18878398102a5a5afa0811f4f8f2a8a94a60cc16e8e9cf54bc95f96808/trafilatura-2.2.0-py3-none-any.whl", hash = "sha256:ac43592a6201264dfc4f9c361cbe3eb3fea96e54437010a159d5e7365360ed98", size = 151906, upload-time = "2026-07-31T16:06:46.485Z" }, +] + [[package]] name = "transformers" version = "5.16.1" @@ -1948,6 +2116,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, ] +[[package]] +name = "tzdata" +version = "2026.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" }, +] + +[[package]] +name = "tzlocal" +version = "5.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/5b/879b2f932adfa7a053c360d50bc896c977fa6426109185f7c12ebdd0cb9d/tzlocal-5.4.4.tar.gz", hash = "sha256:8dbb8660838688a7b6ba4fed31d18dedf842afb4d47ca050d6d891c2c15f3be4", size = 31170, upload-time = "2026-06-29T08:03:40.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/a4/017a7a6cbe387d961a688ec31364ae60a5c4e22c96ae9921b79a947c855d/tzlocal-5.4.4-py3-none-any.whl", hash = "sha256:aae09f0126a8a86fa736be266eb4a471380d26a0de3bc14844e7821fee3e2a15", size = 18115, upload-time = "2026-06-29T08:03:38.666Z" }, +] + [[package]] name = "urllib3" version = "2.7.0"