Skip to content

Commit 039f5de

Browse files
hanselhanselclaude
andcommitted
fix: resolve mypy type error in batch.py audit_one function
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent fd0e991 commit 039f5de

1 file changed

Lines changed: 104 additions & 0 deletions

File tree

src/aeo_cli/core/batch.py

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
"""Batch audit: parse URL files and run concurrent audits."""
2+
3+
from __future__ import annotations
4+
5+
import asyncio
6+
import csv
7+
import io
8+
from collections.abc import Callable
9+
10+
from aeo_cli.core.auditor import audit_site, audit_url
11+
from aeo_cli.core.models import AuditReport, BatchAuditReport, SiteAuditReport
12+
13+
14+
def parse_url_file(path: str) -> list[str]:
15+
"""Read URLs from a .txt or .csv file.
16+
17+
- Skips empty lines and lines starting with #
18+
- For .csv files, uses the first column as URL (skips header if present)
19+
- Auto-prepends https:// to URLs without a scheme
20+
"""
21+
with open(path) as f:
22+
raw = f.read()
23+
24+
if path.endswith(".csv"):
25+
return _parse_csv(raw)
26+
return _parse_txt(raw)
27+
28+
29+
def _parse_txt(raw: str) -> list[str]:
30+
"""Parse a plain text file with one URL per line."""
31+
urls: list[str] = []
32+
for line in raw.splitlines():
33+
stripped = line.strip()
34+
if not stripped or stripped.startswith("#"):
35+
continue
36+
urls.append(_ensure_scheme(stripped))
37+
return urls
38+
39+
40+
def _parse_csv(raw: str) -> list[str]:
41+
"""Parse a CSV file, using the first column as URL."""
42+
urls: list[str] = []
43+
reader = csv.reader(io.StringIO(raw))
44+
for row in reader:
45+
if not row:
46+
continue
47+
cell = row[0].strip()
48+
if not cell or cell.startswith("#"):
49+
continue
50+
# Skip header row (heuristic: if first cell doesn't look like a URL)
51+
if cell.lower() in ("url", "urls", "uri", "link", "website"):
52+
continue
53+
urls.append(_ensure_scheme(cell))
54+
return urls
55+
56+
57+
def _ensure_scheme(url: str) -> str:
58+
"""Prepend https:// if the URL has no scheme."""
59+
if not url.startswith("http"):
60+
return f"https://{url}"
61+
return url
62+
63+
64+
async def run_batch_audit(
65+
urls: list[str],
66+
*,
67+
single: bool = False,
68+
max_pages: int = 10,
69+
timeout: int = 15,
70+
concurrency: int = 3,
71+
progress_callback: Callable[[str], None] | None = None,
72+
) -> BatchAuditReport:
73+
"""Run audits for multiple URLs with concurrency limiting.
74+
75+
Args:
76+
urls: List of URLs to audit.
77+
single: If True, run single-page audits; otherwise multi-page site audits.
78+
max_pages: Max pages per site audit.
79+
timeout: HTTP timeout in seconds.
80+
concurrency: Max concurrent audits.
81+
progress_callback: Called with status messages for each URL.
82+
"""
83+
semaphore = asyncio.Semaphore(concurrency)
84+
reports: list[AuditReport | SiteAuditReport] = []
85+
errors: dict[str, str] = {}
86+
87+
async def _audit_one(url: str) -> None:
88+
async with semaphore:
89+
if progress_callback:
90+
progress_callback(f"Auditing {url}...")
91+
try:
92+
report: AuditReport | SiteAuditReport
93+
if single:
94+
report = await audit_url(url, timeout=timeout)
95+
else:
96+
report = await audit_site(url, max_pages=max_pages, timeout=timeout)
97+
reports.append(report)
98+
except Exception as e:
99+
errors[url] = str(e)
100+
101+
tasks = [asyncio.create_task(_audit_one(u)) for u in urls]
102+
await asyncio.gather(*tasks)
103+
104+
return BatchAuditReport(urls=urls, reports=reports, errors=errors)

0 commit comments

Comments
 (0)