feat: add llms-full.txt checking alongside llms.txt

Probe /llms-full.txt and /.well-known/llms-full.txt in addition to
the existing llms.txt paths. Add llms_full_found and llms_full_url
fields to LlmsTxtReport. Score 10 points if either file is present.
Update detail string to list which files were found.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: hanselhansel

src/aeo_cli/core/checks/llms_txt.py

@@ -9,23 +9,50 @@
 from aeo_cli.core.models import LlmsTxtReport
 
 
-async def check_llms_txt(url: str, client: httpx.AsyncClient) -> LlmsTxtReport:
-    """Probe /llms.txt and /.well-known/llms.txt."""
-    parsed = urlparse(url)
-    base = f"{parsed.scheme}://{parsed.netloc}"
-    paths = ["/llms.txt", "/.well-known/llms.txt"]
-
+async def _probe_file(
+    base: str, paths: list[str], client: httpx.AsyncClient
+) -> str | None:
+    """Probe a list of URL paths, returning the first that has non-empty content."""
     for path in paths:
         probe_url = base + path
         try:
             resp = await client.get(probe_url, follow_redirects=True)
             if resp.status_code == 200 and len(resp.text.strip()) > 0:
-                return LlmsTxtReport(
-                    found=True,
-                    url=probe_url,
-                    detail=f"Found at {probe_url}",
-                )
+                return probe_url
         except httpx.HTTPError:
             continue
+    return None
+
+
+async def check_llms_txt(url: str, client: httpx.AsyncClient) -> LlmsTxtReport:
+    """Probe /llms.txt, /.well-known/llms.txt, /llms-full.txt, /.well-known/llms-full.txt."""
+    parsed = urlparse(url)
+    base = f"{parsed.scheme}://{parsed.netloc}"
 
-    return LlmsTxtReport(found=False, detail="llms.txt not found")
+    llms_url = await _probe_file(base, ["/llms.txt", "/.well-known/llms.txt"], client)
+    full_url = await _probe_file(
+        base, ["/llms-full.txt", "/.well-known/llms-full.txt"], client
+    )
+
+    found = llms_url is not None
+    full_found = full_url is not None
+
+    # Build detail string
+    parts: list[str] = []
+    if found:
+        parts.append(f"llms.txt at {llms_url}")
+    if full_found:
+        parts.append(f"llms-full.txt at {full_url}")
+
+    if parts:
+        detail = "Found: " + ", ".join(parts)
+    else:
+        detail = "llms.txt not found"
+
+    return LlmsTxtReport(
+        found=found,
+        url=llms_url,
+        llms_full_found=full_found,
+        llms_full_url=full_url,
+        detail=detail,
+    )

src/aeo_cli/core/models.py

@@ -76,6 +76,12 @@ class LlmsTxtReport(BaseModel):
 
     found: bool = Field(description="Whether llms.txt was found")
     url: str | None = Field(default=None, description="URL where llms.txt was found")
+    llms_full_found: bool = Field(
+        default=False, description="Whether llms-full.txt was found"
+    )
+    llms_full_url: str | None = Field(
+        default=None, description="URL where llms-full.txt was found"
+    )
     score: float = Field(default=0, description="llms.txt pillar score (0-10)")
     detail: str = Field(default="", description="Summary of llms.txt findings")
 
@@ -108,6 +114,11 @@ class ContentReport(BaseModel):
     has_headings: bool = Field(default=False, description="Whether headings were found")
     has_lists: bool = Field(default=False, description="Whether lists (ul/ol) were found")
     has_code_blocks: bool = Field(default=False, description="Whether code blocks were found")
+    chunk_count: int = Field(default=0, description="Number of content chunks split by headings")
+    avg_chunk_words: int = Field(default=0, description="Average word count per chunk")
+    chunks_in_sweet_spot: int = Field(
+        default=0, description="Chunks with 50-150 words (citation sweet spot)"
+    )
     score: float = Field(default=0, description="Content pillar score (0-40)")
     detail: str = Field(default="", description="Summary of content density findings")
 

src/aeo_cli/core/scoring.py

@@ -57,8 +57,10 @@ def compute_scores(
     else:
         robots.score = 0
 
-    # llms.txt: max LLMS_TXT_MAX
-    llms_txt.score = LLMS_TXT_MAX if llms_txt.found else 0
+    # llms.txt: max LLMS_TXT_MAX — either llms.txt or llms-full.txt qualifies
+    llms_txt.score = (
+        LLMS_TXT_MAX if (llms_txt.found or llms_txt.llms_full_found) else 0
+    )
 
     # Schema: max SCHEMA_MAX — reward high-value types more
     if schema_org.blocks_found > 0:

tests/test_llms_txt_edge_cases.py

@@ -91,3 +91,196 @@ async def mock_get(url, **kwargs):
 
     assert report.found is True
     assert "well-known" in report.url
+
+
+# ── llms-full.txt Tests ──────────────────────────────────────────────────────
+
+
+@pytest.mark.asyncio
+async def test_llms_full_txt_found_at_root():
+    """llms-full.txt at /llms-full.txt should be detected."""
+    async def mock_get(url, **kwargs):
+        resp = AsyncMock()
+        if "llms-full.txt" in url and "well-known" not in url:
+            resp.status_code = 200
+            resp.text = "# Full LLMs content"
+        elif "/llms.txt" in url and "full" not in url:
+            resp.status_code = 200
+            resp.text = "# LLMs.txt"
+        else:
+            resp.status_code = 404
+            resp.text = ""
+        return resp
+
+    mock_client = AsyncMock(spec=httpx.AsyncClient)
+    mock_client.get = AsyncMock(side_effect=mock_get)
+
+    report = await check_llms_txt("https://example.com", mock_client)
+
+    assert report.found is True
+    assert report.llms_full_found is True
+    assert report.llms_full_url == "https://example.com/llms-full.txt"
+
+
+@pytest.mark.asyncio
+async def test_llms_full_txt_found_at_well_known():
+    """llms-full.txt at /.well-known/llms-full.txt should be detected."""
+    async def mock_get(url, **kwargs):
+        resp = AsyncMock()
+        if "well-known/llms-full.txt" in url:
+            resp.status_code = 200
+            resp.text = "# Full LLMs content"
+        elif "/llms.txt" in url and "full" not in url:
+            resp.status_code = 200
+            resp.text = "# LLMs.txt"
+        else:
+            resp.status_code = 404
+            resp.text = ""
+        return resp
+
+    mock_client = AsyncMock(spec=httpx.AsyncClient)
+    mock_client.get = AsyncMock(side_effect=mock_get)
+
+    report = await check_llms_txt("https://example.com", mock_client)
+
+    assert report.found is True
+    assert report.llms_full_found is True
+    assert report.llms_full_url == "https://example.com/.well-known/llms-full.txt"
+
+
+@pytest.mark.asyncio
+async def test_only_llms_full_txt_found():
+    """Only llms-full.txt present (no llms.txt) should still score."""
+    async def mock_get(url, **kwargs):
+        resp = AsyncMock()
+        if "llms-full.txt" in url and "well-known" not in url:
+            resp.status_code = 200
+            resp.text = "# Full LLMs content"
+        else:
+            resp.status_code = 404
+            resp.text = ""
+        return resp
+
+    mock_client = AsyncMock(spec=httpx.AsyncClient)
+    mock_client.get = AsyncMock(side_effect=mock_get)
+
+    report = await check_llms_txt("https://example.com", mock_client)
+
+    assert report.found is False  # llms.txt not found
+    assert report.llms_full_found is True
+    assert report.llms_full_url == "https://example.com/llms-full.txt"
+    assert "llms-full.txt" in report.detail
+
+
+@pytest.mark.asyncio
+async def test_both_llms_and_llms_full_found():
+    """Both llms.txt and llms-full.txt present should be reported."""
+    async def mock_get(url, **kwargs):
+        resp = AsyncMock()
+        resp.status_code = 200
+        resp.text = "# Content"
+        return resp
+
+    mock_client = AsyncMock(spec=httpx.AsyncClient)
+    mock_client.get = AsyncMock(side_effect=mock_get)
+
+    report = await check_llms_txt("https://example.com", mock_client)
+
+    assert report.found is True
+    assert report.llms_full_found is True
+    assert "llms.txt" in report.detail
+    assert "llms-full.txt" in report.detail
+
+
+@pytest.mark.asyncio
+async def test_llms_full_txt_not_found():
+    """When only llms.txt exists, llms_full_found should be False."""
+    async def mock_get(url, **kwargs):
+        resp = AsyncMock()
+        if "/llms.txt" in url and "full" not in url and "well-known" not in url:
+            resp.status_code = 200
+            resp.text = "# LLMs.txt"
+        else:
+            resp.status_code = 404
+            resp.text = ""
+        return resp
+
+    mock_client = AsyncMock(spec=httpx.AsyncClient)
+    mock_client.get = AsyncMock(side_effect=mock_get)
+
+    report = await check_llms_txt("https://example.com", mock_client)
+
+    assert report.found is True
+    assert report.llms_full_found is False
+    assert report.llms_full_url is None
+
+
+@pytest.mark.asyncio
+async def test_llms_full_txt_empty_not_counted():
+    """An empty llms-full.txt should not count as found."""
+    async def mock_get(url, **kwargs):
+        resp = AsyncMock()
+        if "llms-full.txt" in url:
+            resp.status_code = 200
+            resp.text = "   \n  "
+        elif "/llms.txt" in url:
+            resp.status_code = 200
+            resp.text = "# LLMs.txt"
+        else:
+            resp.status_code = 404
+            resp.text = ""
+        return resp
+
+    mock_client = AsyncMock(spec=httpx.AsyncClient)
+    mock_client.get = AsyncMock(side_effect=mock_get)
+
+    report = await check_llms_txt("https://example.com", mock_client)
+
+    assert report.found is True
+    assert report.llms_full_found is False
+
+
+@pytest.mark.asyncio
+async def test_llms_full_txt_http_error():
+    """HTTP error on llms-full.txt should not crash; llms.txt still found."""
+    call_count = 0
+
+    async def mock_get(url, **kwargs):
+        nonlocal call_count
+        call_count += 1
+        if "llms-full.txt" in url:
+            raise httpx.ConnectError("Connection refused")
+        resp = AsyncMock()
+        if "/llms.txt" in url and "full" not in url and "well-known" not in url:
+            resp.status_code = 200
+            resp.text = "# LLMs.txt"
+        else:
+            resp.status_code = 404
+            resp.text = ""
+        return resp
+
+    mock_client = AsyncMock(spec=httpx.AsyncClient)
+    mock_client.get = AsyncMock(side_effect=mock_get)
+
+    report = await check_llms_txt("https://example.com", mock_client)
+
+    assert report.found is True
+    assert report.llms_full_found is False
+
+
+@pytest.mark.asyncio
+async def test_neither_llms_found():
+    """Neither llms.txt nor llms-full.txt found should report not found."""
+    mock_response = AsyncMock()
+    mock_response.status_code = 404
+    mock_response.text = ""
+
+    mock_client = AsyncMock(spec=httpx.AsyncClient)
+    mock_client.get = AsyncMock(return_value=mock_response)
+
+    report = await check_llms_txt("https://example.com", mock_client)
+
+    assert report.found is False
+    assert report.llms_full_found is False
+    assert report.url is None
+    assert report.llms_full_url is None

tests/test_scoring_integration.py

@@ -114,3 +114,45 @@ def test_overall_is_sum_of_pillars():
     r, lt, s, c, overall = compute_scores(robots, llms_txt, schema_org, content)
 
     assert overall == r.score + lt.score + s.score + c.score
+
+
+def test_llms_full_only_scores_10():
+    """Only llms-full.txt found (no llms.txt) should still score 10."""
+    llms_txt = LlmsTxtReport(
+        found=False,
+        llms_full_found=True,
+        llms_full_url="https://example.com/llms-full.txt",
+    )
+
+    _, lt, _, _, _ = compute_scores(
+        RobotsReport(found=False), llms_txt, SchemaReport(), ContentReport()
+    )
+
+    assert lt.score == 10
+
+
+def test_both_llms_files_scores_10():
+    """Both llms.txt and llms-full.txt found should still score max 10."""
+    llms_txt = LlmsTxtReport(
+        found=True,
+        url="https://example.com/llms.txt",
+        llms_full_found=True,
+        llms_full_url="https://example.com/llms-full.txt",
+    )
+
+    _, lt, _, _, _ = compute_scores(
+        RobotsReport(found=False), llms_txt, SchemaReport(), ContentReport()
+    )
+
+    assert lt.score == 10
+
+
+def test_neither_llms_scores_0():
+    """Neither llms.txt nor llms-full.txt should score 0."""
+    llms_txt = LlmsTxtReport(found=False, llms_full_found=False)
+
+    _, lt, _, _, _ = compute_scores(
+        RobotsReport(found=False), llms_txt, SchemaReport(), ContentReport()
+    )
+
+    assert lt.score == 0