feature(metrics): SSH tunnel observability and Grafana dashboard

Add per-build attribution for SSH-tunneled client traffic and a Grafana
dashboard to observe it, proving traffic flows through the tunnel and
surfacing proxy vitals.

Client (argus/client/session.py):
- Emit X-Argus-Build-Id (JOB_NAME#BUILD_NUMBER, ARGUS_BUILD_ID override)
  and X-Argus-Build-Url on tunneled requests only.

Backend (argus_backend.py):
- Add http_request_tunnel_build_total{build_id, build_url} counter;
  build_url is 1:1 with build_id (no extra series), carried for linking.

Dashboard (scripts/grafana/argus-overview.json):
- Make datasource portable via a ${datasource} template variable.
- Add "SSH Tunneling" row (tunneled-vs-direct proof, % via tunnel, header
  anomalies, SSH auth-attempt rate, registrations, by-endpoint, by-UA,
  by-build with a clickable Jenkins data link).
- Add "SSH Proxy Vitals" row (bandwidth/CPU/mem/disk/connections) gated on
  a proxy_job variable; requires node_exporter on the proxy host.

Tests: cover build-id composition, build-url, and omission outside CI.

Author: CodeLieutenant

argus/client/session.py

@@ -27,6 +27,45 @@ def _resolve_use_tunnel(use_tunnel: bool | None) -> bool:
     return os.environ.get("ARGUS_USE_TUNNEL", "").strip().lower() in ("1", "true", "yes", "on")
 
 
+# Upper bound on the build-id header value to keep the backend's per-build
+# Prometheus label cardinality bounded even if a client sends junk.
+_MAX_BUILD_ID_LEN = 256
+
+
+def _resolve_build_id() -> str | None:
+    """Resolve a human-identifiable build id for tunnel attribution.
+
+    Order of preference:
+
+    1. ``ARGUS_BUILD_ID`` — explicit override, used verbatim.
+    2. Jenkins ``JOB_NAME`` (full folder path) combined with ``BUILD_NUMBER``
+       when available, formatted as ``job/path#42`` for easy identification.
+
+    Returns ``None`` outside CI so the ``X-Argus-Build-Id`` header is simply
+    omitted.
+    """
+    override = os.environ.get("ARGUS_BUILD_ID", "").strip()
+    if override:
+        return override[:_MAX_BUILD_ID_LEN]
+
+    job_name = os.environ.get("JOB_NAME", "").strip()
+    if not job_name:
+        return None
+    build_number = os.environ.get("BUILD_NUMBER", "").strip()
+    build_id = f"{job_name}#{build_number}" if build_number else job_name
+    return build_id[:_MAX_BUILD_ID_LEN]
+
+
+def _resolve_build_url() -> str | None:
+    """Jenkins ``BUILD_URL`` (or ``ARGUS_BUILD_URL`` override) for the run.
+
+    Carried as ``X-Argus-Build-Url`` so the Grafana ``build_id`` series can link
+    straight back to the originating build. ``None`` when not running in CI.
+    """
+    value = (os.environ.get("ARGUS_BUILD_URL") or os.environ.get("BUILD_URL") or "").strip()
+    return value[:_MAX_BUILD_ID_LEN * 2] if value else None
+
+
 def _resolve_monitor_interval() -> float:
     raw = os.environ.get("ARGUS_TUNNEL_MONITOR_INTERVAL")
     if raw is None:
@@ -82,6 +121,8 @@ def __init__(self, auth_token: str, original_base_url: str, max_retries: int = 3
 
         self._auth_token = auth_token
         self._original_base_url = original_base_url
+        self._build_id = _resolve_build_id()
+        self._build_url = _resolve_build_url()
 
         self._tunnel: SSHTunnel | None = None
         self._tunnel_config: TunnelConfig | None = None
@@ -252,6 +293,10 @@ def _tunnel_headers(self) -> dict[str, str]:
         }
         if self._tunnel_config.key_id:
             headers["X-Forwarded-Key-ID"] = self._tunnel_config.key_id
+        if self._build_id:
+            headers["X-Argus-Build-Id"] = self._build_id
+        if self._build_url:
+            headers["X-Argus-Build-Url"] = self._build_url
         return headers
 
     def request(self, method: str, url: str, *args, **kwargs) -> requests.Response:

argus/client/tests/test_tunnel.py

@@ -388,6 +388,67 @@ def test_tunneled_session_close_unregisters_atexit():
     callback(ref)
 
 
+def _prime_tunnel_state(session):
+    """Force a session into the 'tunnel active' state so _tunnel_headers() emits."""
+    from types import SimpleNamespace
+
+    session._tunnel_port = 12345
+    session._tunnel_established_at = "2026-06-16T00:00:00+00:00"
+    session._tunnel_config = SimpleNamespace(proxy_host="proxy.example.com", key_id="key-uuid")
+
+
+def test_tunnel_headers_explicit_build_id_used_verbatim(monkeypatch):
+    monkeypatch.setenv("ARGUS_BUILD_ID", "custom-id#7")
+    session = TunneledSession(auth_token="token", original_base_url="https://argus.scylladb.com")
+    try:
+        _prime_tunnel_state(session)
+        headers = session._tunnel_headers()
+        assert headers["X-Argus-Build-Id"] == "custom-id#7"
+        assert headers["X-SSH-Tunnel-Origin"] == "proxy.example.com"
+    finally:
+        session.close()
+
+
+def test_tunnel_headers_compose_job_name_and_build_number(monkeypatch):
+    monkeypatch.delenv("ARGUS_BUILD_ID", raising=False)
+    monkeypatch.setenv("JOB_NAME", "scylla-master/longevity/longevity-100gb")
+    monkeypatch.setenv("BUILD_NUMBER", "42")
+    monkeypatch.setenv("BUILD_URL", "https://jenkins.scylladb.com/job/scylla-master/job/longevity/42/")
+    session = TunneledSession(auth_token="token", original_base_url="https://argus.scylladb.com")
+    try:
+        _prime_tunnel_state(session)
+        headers = session._tunnel_headers()
+        assert headers["X-Argus-Build-Id"] == "scylla-master/longevity/longevity-100gb#42"
+        assert headers["X-Argus-Build-Url"] == "https://jenkins.scylladb.com/job/scylla-master/job/longevity/42/"
+    finally:
+        session.close()
+
+
+def test_tunnel_headers_build_id_job_name_without_build_number(monkeypatch):
+    monkeypatch.delenv("ARGUS_BUILD_ID", raising=False)
+    monkeypatch.delenv("BUILD_NUMBER", raising=False)
+    monkeypatch.setenv("JOB_NAME", "jenkins-job-name")
+    session = TunneledSession(auth_token="token", original_base_url="https://argus.scylladb.com")
+    try:
+        _prime_tunnel_state(session)
+        assert session._tunnel_headers()["X-Argus-Build-Id"] == "jenkins-job-name"
+    finally:
+        session.close()
+
+
+def test_tunnel_headers_omit_build_id_and_url_when_unset(monkeypatch):
+    for var in ("ARGUS_BUILD_ID", "JOB_NAME", "BUILD_NUMBER", "ARGUS_BUILD_URL", "BUILD_URL"):
+        monkeypatch.delenv(var, raising=False)
+    session = TunneledSession(auth_token="token", original_base_url="https://argus.scylladb.com")
+    try:
+        _prime_tunnel_state(session)
+        headers = session._tunnel_headers()
+        assert "X-Argus-Build-Id" not in headers
+        assert "X-Argus-Build-Url" not in headers
+    finally:
+        session.close()
+
+
 def test_argus_client_works_as_context_manager(requests_mock, monkeypatch):
     requests_mock.get(
         "https://argus.scylladb.com/api/v1/client/testrun/test-type/test-id/get",

argus_backend.py

@@ -79,6 +79,21 @@ def register_metrics():
             },
         )
     )
+    METRICS.register_default(
+        METRICS.counter(
+            "http_request_tunnel_build_total",
+            "Tunneled requests by Jenkins build/job id (X-Argus-Build-Id)",
+            labels={
+                # One series per Jenkins build (job/path#42). Requests without the
+                # header (non-tunnel or pre-attribution clients) fall into the
+                # "unknown" bucket and are filtered out in dashboards.
+                "build_id": lambda: request.headers.get("X-Argus-Build-Id") or "unknown",
+                # 1:1 with build_id (no extra series) — carried so Grafana can
+                # link the build_id straight back to the Jenkins build.
+                "build_url": lambda: request.headers.get("X-Argus-Build-Url") or "",
+            },
+        )
+    )
 
 
 def start_server(config=None) -> Flask: