Generate Proxy Lists #2012
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generate Proxy Lists | |
| on: | |
| schedule: | |
| # Run every 3 hours. GitHub's schedule jitter can delay starts enough that a | |
| # 2-hour cadence still overlaps a healthy 80-90 minute generation run when | |
| # queued generation is serialized below. | |
| - cron: "0 */3 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| revalidate_limit: | |
| description: "Maximum oldest proxies to re-validate before fetch (0 = full sweep)" | |
| required: false | |
| default: "2000" | |
| # Ensure only one instance runs at a time to prevent database and network contention. | |
| # Queue scheduled/manual runs rather than overlapping validation-heavy jobs. | |
| concurrency: | |
| group: proxy-generation | |
| cancel-in-progress: false | |
| jobs: | |
| generate-proxy-lists: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| # Fetch with token that has write access for pushing | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.11" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v8.2.0 | |
| with: | |
| version: "0.8.22" | |
| - name: Install dependencies | |
| run: | | |
| uv venv --clear --python 3.11 | |
| uv pip install -e . | |
| - name: Install Playwright browsers | |
| env: | |
| NODE_OPTIONS: --disable-warning=DEP0169 | |
| run: | | |
| # Required for JS-rendered sources | |
| uv run python -m playwright install chromium | |
| - name: Download GeoLite2 database | |
| run: | | |
| # Download GeoLite2-Country database for IP geolocation | |
| if curl -fsSL --retry 5 --retry-delay 5 --retry-all-errors https://github.com/P3TERX/GeoLite.mmdb/releases/latest/download/GeoLite2-Country.mmdb -o GeoLite2-Country.mmdb; then | |
| echo "Downloaded GeoLite2-Country.mmdb ($(stat --format=%s GeoLite2-Country.mmdb) bytes)" | |
| else | |
| rm -f GeoLite2-Country.mmdb | |
| echo "GeoLite2 download failed after retries; continuing without geolocation database" | |
| fi | |
| - name: Show database stats before | |
| run: | | |
| if [ -f proxywhirl.db ]; then | |
| sqlite3 proxywhirl.db \ | |
| "SELECT 'proxy identities: ' || COUNT(*) FROM proxy_identities;" \ | |
| || echo "Stats unavailable (fresh database?)" | |
| else | |
| echo "Stats unavailable (fresh database?)" | |
| fi | |
| - name: Re-validate existing proxies | |
| timeout-minutes: 60 | |
| env: | |
| REVALIDATE_LIMIT: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.revalidate_limit || '2000' }} | |
| run: | | |
| # Re-validate the oldest proxies first so scheduled runs stay within | |
| # budget while still making progress across the full database. | |
| # Pass 0 on manual dispatch to force a full sweep. | |
| # Failed proxies are pruned by the current fetch command. | |
| uv run proxywhirl fetch --revalidate --revalidate-limit "${REVALIDATE_LIMIT}" --timeout 10 --concurrency 2000 --prune-failed --no-export | |
| - name: Fetch new proxies from sources | |
| timeout-minutes: 180 | |
| run: | | |
| # Fetch and validate NEW proxies from all sources, save to database. | |
| # No export yet - export happens after database compaction. | |
| # Use 10s timeout for more coverage; fastest proxies sort to top of exports | |
| uv run proxywhirl fetch --timeout 10 --concurrency 2000 --no-export | |
| - name: Compact database | |
| run: | | |
| # Drop orphaned validation records and VACUUM before export | |
| sqlite3 proxywhirl.db <<'SQL' | |
| DELETE FROM validation_results WHERE proxy_url NOT IN (SELECT url FROM proxy_identities); | |
| VACUUM; | |
| SQL | |
| DB_SIZE=$(stat --format=%s proxywhirl.db) | |
| DB_MB=$((DB_SIZE / 1048576)) | |
| echo "Database size after compact: ${DB_MB} MB" | |
| - name: Export proxy lists | |
| run: | | |
| # Export all files for web dashboard after cleanup: | |
| # - metadata.json, http.txt, socks4.txt, socks5.txt, all.txt, proxies.json | |
| # - proxies-rich.json, stats.json | |
| uv run proxywhirl export | |
| - name: Show database stats after | |
| run: | | |
| sqlite3 proxywhirl.db \ | |
| "SELECT 'proxy identities: ' || COUNT(*) FROM proxy_identities;" | |
| - name: Sync proxy lists to web directory | |
| run: | | |
| # Copy proxy lists to web/public for Next.js deployment | |
| cp -r docs/proxy-lists/* web/public/proxy-lists/ | |
| - name: Set up pnpm | |
| uses: pnpm/action-setup@v6.0.8 | |
| with: | |
| version: 10 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: "24" | |
| cache: pnpm | |
| cache-dependency-path: web/pnpm-lock.yaml | |
| - name: Generate docs references | |
| env: | |
| NODE_OPTIONS: --disable-warning=DEP0169 | |
| run: | | |
| pnpm --dir web install --frozen-lockfile | |
| pnpm --dir web run docs:generate | |
| - name: Commit and push if changes | |
| run: | | |
| git config --local user.email "github-actions[bot]@users.noreply.github.com" | |
| git config --local user.name "github-actions[bot]" | |
| # Preserve this run's generated outputs before integrating remote changes. | |
| GENERATED_SNAPSHOT="$(mktemp -d)" | |
| cp -a docs/proxy-lists "${GENERATED_SNAPSHOT}/docs-proxy-lists" | |
| cp -a web/public/proxy-lists "${GENERATED_SNAPSHOT}/web-proxy-lists" | |
| cp -a web/content/docs/generated "${GENERATED_SNAPSHOT}/web-generated-docs" | |
| cp -a web/content/generated "${GENERATED_SNAPSHOT}/web-generated-data" | |
| # Rebase-prone generated files are last-writer-wins. Re-anchor on the | |
| # latest branch head first, then restore this run's generated outputs. | |
| git fetch origin main | |
| git reset --hard origin/main | |
| rm -rf docs/proxy-lists web/public/proxy-lists web/content/docs/generated web/content/generated | |
| mkdir -p docs web/public web/content/docs web/content | |
| cp -a "${GENERATED_SNAPSHOT}/docs-proxy-lists" docs/proxy-lists | |
| cp -a "${GENERATED_SNAPSHOT}/web-proxy-lists" web/public/proxy-lists | |
| cp -a "${GENERATED_SNAPSHOT}/web-generated-docs" web/content/docs/generated | |
| cp -a "${GENERATED_SNAPSHOT}/web-generated-data" web/content/generated | |
| # Add generated proxy list files and source-grounded docs data. | |
| git add docs/proxy-lists/ web/public/proxy-lists/ web/content/docs/generated/ web/content/generated/ | |
| # Only commit if there are staged changes. | |
| if git diff --staged --quiet; then | |
| echo "No changes to commit" | |
| else | |
| git commit -m "chore: Update proxy lists - $(date -u '+%Y-%m-%d %H:%M:%S UTC')" | |
| git push | |
| fi | |
| - name: Upload proxy lists as artifacts | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: proxy-lists-${{ github.run_number }} | |
| path: docs/proxy-lists/ | |
| retention-days: 7 | |
| - name: Upload database as artifact | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: proxywhirl-db-${{ github.run_number }} | |
| path: proxywhirl.db | |
| retention-days: 30 |