Skip to content
CI

Add files via upload (#16) #88

Sign in to view logs

Add files via upload (#16)

Add files via upload (#16) #88

Workflow file for this run

.github/workflows/ci.yml at 400e857
name: CI
on:
push:
pull_request:
workflow_dispatch:
jobs:
quality:
name: Lint, format, and type checks on Python ${{ matrix.python-version }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12"]
steps:
- name: Check out repository
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- name: Install quality tools
run: |
python -m pip install --upgrade pip
pip install -r requirements-dev.txt
- name: Ruff (lint)
run: ruff check underwriting app tests
- name: Black (format check)
run: black --check underwriting app tests
- name: Mypy (type check)
run: mypy
test-and-smoke:
name: Test and smoke workflow on Python ${{ matrix.python-version }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12"]
steps:
- name: Check out repository
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
pip install 'coverage[toml]==7.14.3'
- name: Check source files compile
run: |
python -m compileall underwriting app tests
- name: Run unit tests with coverage
run: |
coverage run -m unittest discover -s tests -v
coverage report
- name: Run pipeline smoke workflow
run: |
python -m underwriting.pipeline \
--input data/raw/loanapproval.csv \
--out-dir ci_outputs \
--figures-dir ci_figures \
--target-coverage 0.70
- name: Verify expected outputs
run: |
test -f ci_outputs/metrics_overall.json
test -f ci_outputs/baseline_metrics.json
test -f ci_outputs/evaluation_summary.json
test -f ci_outputs/policy_variants.json
test -f ci_outputs/slice_report.csv
test -f ci_outputs/slice_report.json
test -f ci_outputs/slice_summary.json
test -f ci_outputs/coverage_curve.csv
test -f ci_outputs/abstention_policy.json
test -f ci_outputs/test_predictions.csv
test -f ci_outputs/model.joblib
test -f ci_outputs/policy_card.md
test -f ci_outputs/data_quality.json
test -f ci_figures/confusion_matrix.png
test -f ci_figures/reliability_diagram.png
test -f ci_figures/probability_histograms.png
test -f ci_figures/precision_recall_curve.png
test -f ci_figures/coverage_vs_performance.png
test -f ci_figures/slice_review_rates.png
test -f ci_figures/slice_error_rates.png
- name: Validate smoke output contracts
run: |
python - <<'PY'
import json
from pathlib import Path
import pandas as pd
metrics = json.loads(Path("ci_outputs/metrics_overall.json").read_text())
policy = json.loads(Path("ci_outputs/abstention_policy.json").read_text())
baselines = json.loads(Path("ci_outputs/baseline_metrics.json").read_text())
variants = json.loads(Path("ci_outputs/policy_variants.json").read_text())
summary = json.loads(Path("ci_outputs/evaluation_summary.json").read_text())
slice_summary = json.loads(Path("ci_outputs/slice_summary.json").read_text())
predictions = pd.read_csv("ci_outputs/test_predictions.csv")
coverage = pd.read_csv("ci_outputs/coverage_curve.csv")
slice_report = pd.read_csv("ci_outputs/slice_report.csv")
required_metrics = {"accuracy", "f1", "brier", "roc_auc", "average_precision", "ece"}
missing_metrics = required_metrics - set(metrics)
if missing_metrics:
raise AssertionError(f"Missing metrics: {missing_metrics}")
if not 0 <= metrics["brier"] <= 1:
raise AssertionError("Brier score should be between 0 and 1")
if not 0 <= metrics["ece"] <= 1:
raise AssertionError("ECE should be between 0 and 1")
if not 0 <= metrics["average_precision"] <= 1:
raise AssertionError("Average precision should be between 0 and 1")
required_policy = {
"recommended_threshold",
"expected_coverage",
"expected_accuracy_auto",
"expected_f1_auto",
}
missing_policy = required_policy - set(policy)
if missing_policy:
raise AssertionError(f"Missing policy keys: {missing_policy}")
required_prediction_cols = {"p_approve", "p_reject", "confidence", "auto_decide"}
missing_cols = required_prediction_cols - set(predictions.columns)
if missing_cols:
raise AssertionError(f"Missing prediction columns: {missing_cols}")
if not predictions["p_approve"].between(0, 1).all():
raise AssertionError("p_approve must be between 0 and 1")
if not coverage["coverage"].between(0, 1).all():
raise AssertionError("Coverage values must be between 0 and 1")
required_baselines = {"majority_class", "empirical_prior", "stratified_random"}
if required_baselines - set(baselines):
raise AssertionError("Missing baseline metrics")
required_variants = {"target_coverage", "quality_first", "high_coverage", "balanced", "conservative_review"}
if required_variants - set(variants):
raise AssertionError("Missing policy variants")
for key in ["model_metrics", "baseline_metrics", "recommended_policy", "policy_variants", "slice_summary", "slice_artifacts"]:
if key not in summary:
raise AssertionError(f"Missing evaluation summary key: {key}")
required_slice_cols = {"slice_feature", "slice_value", "n", "review_rate", "error_rate", "ece"}
missing_slice_cols = required_slice_cols - set(slice_report.columns)
if missing_slice_cols:
raise AssertionError(f"Missing slice report columns: {missing_slice_cols}")
if slice_report.empty:
raise AssertionError("Slice report should not be empty")
if slice_summary.get("n_slices", 0) <= 0:
raise AssertionError("Slice summary should report at least one slice")
print("Smoke output contracts OK")
PY
- name: Upload generated artifacts
uses: actions/upload-artifact@v4
with:
name: underwriting-decision-safety-artifacts-python-${{ matrix.python-version }}
path: |
ci_outputs/
ci_figures/