.gitlab-ci.yml

# =============================================================================
# GitLab CI/CD Pipeline for Sentiment Analysis MLOps Project
# =============================================================================
# Pipeline stages:
#   1. test     - Run unit tests and linting
#   2. build    - Build Docker images
#   3. push     - Push images to registry
#   4. deploy   - Deploy to staging/production (optional)
#   5. smoke    - Smoke test with minimal training (CT - Continuous Training)
# =============================================================================

stages:
  - test
  - build
  - push
  - deploy
  - smoke

# =============================================================================
# Global Variables
# =============================================================================
variables:
  # Docker settings
  DOCKER_DRIVER: overlay2
  DOCKER_TLS_CERTDIR: "/certs"

  # Registry settings (GitLab Container Registry)
  REGISTRY: $CI_REGISTRY
  IMAGE_API: $CI_REGISTRY_IMAGE/sentiment-api
  IMAGE_RETRAIN: $CI_REGISTRY_IMAGE/retrain-service
  IMAGE_FRONTEND: $CI_REGISTRY_IMAGE/sentiment-frontend

  # Python settings
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
  PYTHONUNBUFFERED: "1"

  # MLflow settings (for smoke tests)
  MLFLOW_TRACKING_URI: "http://mlflow:5000"

# =============================================================================
# Cache Configuration
# =============================================================================
cache:
  key: ${CI_COMMIT_REF_SLUG}
  paths:
    - .cache/pip
    - venv/

# =============================================================================
# Stage 1: TEST - Unit Tests & Linting
# =============================================================================

# Linting job
lint:
  stage: test
  image: python:3.10-slim
  before_script:
    - pip install --upgrade pip
    - pip install flake8 black isort
  script:
    - echo "Running linting checks..."
    # Flake8 - PEP8 compliance
    - flake8 src/ --max-line-length=120 --ignore=E501,W503,E203 --exclude=__pycache__
    # Black - Code formatting check
    - black --check --line-length=120 src/ || true
    # isort - Import sorting check
    - isort --check-only src/ || true
  allow_failure: true
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH == "master"

# Unit tests job
test:
  stage: test
  image: python:3.10-slim
  before_script:
    - apt-get update && apt-get install -y gcc g++ --no-install-recommends
    - pip install --upgrade pip
    - pip install -r requirements.txt
    - pip install pytest pytest-cov pytest-html
  script:
    - echo "Running unit tests..."
    - python -m pytest tests/ -v --cov=src --cov-report=xml --cov-report=html --junitxml=report.xml
  coverage: '/TOTAL.*\s+(\d+%)/'
  artifacts:
    when: always
    paths:
      - htmlcov/
      - report.xml
      - coverage.xml
    reports:
      junit: report.xml
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
    expire_in: 1 week
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH == "master"

# Security scan (optional)
security_scan:
  stage: test
  image: python:3.10-slim
  before_script:
    - pip install safety bandit
  script:
    - echo "Running security checks..."
    # Check dependencies for known vulnerabilities
    - safety check -r requirements.txt || true
    # Static security analysis
    - bandit -r src/ -ll || true
  allow_failure: true
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH == "master"

# =============================================================================
# Stage 2: BUILD - Build Docker Images
# =============================================================================

build_api:
  stage: build
  image: docker:24.0.5
  services:
    - docker:24.0.5-dind
  before_script:
    - docker info
  script:
    - echo "Building Sentiment API image..."
    - docker build -t $IMAGE_API:$CI_COMMIT_SHA -t $IMAGE_API:latest -f Dockerfile.serve .
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH == "master"
    - if: $CI_COMMIT_TAG

build_retrain:
  stage: build
  image: docker:24.0.5
  services:
    - docker:24.0.5-dind
  before_script:
    - docker info
  script:
    - echo "Building Retrain Service image..."
    - docker build -t $IMAGE_RETRAIN:$CI_COMMIT_SHA -t $IMAGE_RETRAIN:latest -f Dockerfile.train .
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH == "master"
    - if: $CI_COMMIT_TAG

build_frontend:
  stage: build
  image: docker:24.0.5
  services:
    - docker:24.0.5-dind
  before_script:
    - docker info
  script:
    - echo "Building Frontend image..."
    - docker build -t $IMAGE_FRONTEND:$CI_COMMIT_SHA -t $IMAGE_FRONTEND:latest -f frontend/Dockerfile.frontend .
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH == "master"
    - if: $CI_COMMIT_TAG

# =============================================================================
# Stage 3: PUSH - Push Images to Registry
# =============================================================================

push_api:
  stage: push
  image: docker:24.0.5
  services:
    - docker:24.0.5-dind
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
  script:
    - echo "Pushing Sentiment API image to registry..."
    - docker build -t $IMAGE_API:$CI_COMMIT_SHA -t $IMAGE_API:latest -f Dockerfile.serve .
    - docker push $IMAGE_API:$CI_COMMIT_SHA
    - docker push $IMAGE_API:latest
    # Tag with version if this is a tagged release
    - |
      if [ -n "$CI_COMMIT_TAG" ]; then
        docker tag $IMAGE_API:$CI_COMMIT_SHA $IMAGE_API:$CI_COMMIT_TAG
        docker push $IMAGE_API:$CI_COMMIT_TAG
      fi
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH == "master"
    - if: $CI_COMMIT_TAG

push_retrain:
  stage: push
  image: docker:24.0.5
  services:
    - docker:24.0.5-dind
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
  script:
    - echo "Pushing Retrain Service image to registry..."
    - docker build -t $IMAGE_RETRAIN:$CI_COMMIT_SHA -t $IMAGE_RETRAIN:latest -f Dockerfile.train .
    - docker push $IMAGE_RETRAIN:$CI_COMMIT_SHA
    - docker push $IMAGE_RETRAIN:latest
    # Tag with version if this is a tagged release
    - |
      if [ -n "$CI_COMMIT_TAG" ]; then
        docker tag $IMAGE_RETRAIN:$CI_COMMIT_SHA $IMAGE_RETRAIN:$CI_COMMIT_TAG
        docker push $IMAGE_RETRAIN:$CI_COMMIT_TAG
      fi
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH == "master"
    - if: $CI_COMMIT_TAG

push_frontend:
  stage: push
  image: docker:24.0.5
  services:
    - docker:24.0.5-dind
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
  script:
    - echo "Pushing Frontend image to registry..."
    - docker build -t $IMAGE_FRONTEND:$CI_COMMIT_SHA -t $IMAGE_FRONTEND:latest -f frontend/Dockerfile.frontend .
    - docker push $IMAGE_FRONTEND:$CI_COMMIT_SHA
    - docker push $IMAGE_FRONTEND:latest
    # Tag with version if this is a tagged release
    - |
      if [ -n "$CI_COMMIT_TAG" ]; then
        docker tag $IMAGE_FRONTEND:$CI_COMMIT_SHA $IMAGE_FRONTEND:$CI_COMMIT_TAG
        docker push $IMAGE_FRONTEND:$CI_COMMIT_TAG
      fi
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH == "master"
    - if: $CI_COMMIT_TAG

# =============================================================================
# Stage 4: DEPLOY (Optional)
# =============================================================================

deploy_staging:
  stage: deploy
  image: alpine:latest
  before_script:
    - apk add --no-cache curl openssh-client
  script:
    - echo "Deploying to staging environment..."
    # Add your deployment commands here
    # Example: SSH to server and pull new images
    # - ssh $STAGING_USER@$STAGING_HOST "cd /app && docker-compose pull && docker-compose up -d"
  environment:
    name: staging
    url: https://staging.example.com
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      when: manual
    - if: $CI_COMMIT_BRANCH == "master"
      when: manual

deploy_production:
  stage: deploy
  image: alpine:latest
  before_script:
    - apk add --no-cache curl openssh-client
  script:
    - echo "Deploying to production environment..."
    # Add your deployment commands here
  environment:
    name: production
    url: https://api.example.com
  rules:
    - if: $CI_COMMIT_TAG
      when: manual
  needs:
    - push_api
    - push_retrain

# =============================================================================
# Stage 5: SMOKE TEST - Continuous Training (CT)
# =============================================================================
# Scheduled job that runs a minimal training run to verify the pipeline works
# Run with: epochs=1 or data subset

smoke_test_training:
  stage: smoke
  image: python:3.10-slim
  before_script:
    - apt-get update && apt-get install -y gcc g++ --no-install-recommends
    - pip install --upgrade pip
    - pip install -r requirements.txt
  script:
    - echo "============================================"
    - echo "  SMOKE TEST - Minimal Training Run"
    - echo "============================================"
    - |
      python -c "
      import pandas as pd
      import numpy as np
      from sklearn.feature_extraction.text import TfidfVectorizer
      from sklearn.model_selection import train_test_split
      from lightgbm import LGBMClassifier
      from sklearn.metrics import f1_score, accuracy_score
      import sys

      print('Loading sample data for smoke test...')

      # Load a small subset of data (first 1000 rows)
      try:
          df = pd.read_csv('data/Reddit_Data.csv', nrows=1000)
      except:
          # Create synthetic data if real data not available
          print('Using synthetic data for smoke test')
          np.random.seed(42)
          texts = ['positive text ' * 5] * 400 + ['negative text ' * 5] * 400 + ['neutral text ' * 5] * 200
          labels = [1] * 400 + [-1] * 400 + [0] * 200
          df = pd.DataFrame({'clean_comment': texts, 'category': labels})

      # Preprocess
      df = df.dropna(subset=['clean_comment', 'category'])
      X = df['clean_comment'].astype(str)
      y = df['category']

      # Simple train/test split
      X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

      # Vectorize (minimal features for speed)
      vectorizer = TfidfVectorizer(max_features=1000, ngram_range=(1, 2))
      X_train_vec = vectorizer.fit_transform(X_train)
      X_test_vec = vectorizer.transform(X_test)

      # Train minimal model (few iterations)
      print('Training smoke test model (n_estimators=10)...')
      model = LGBMClassifier(n_estimators=10, max_depth=3, verbose=-1, random_state=42)
      model.fit(X_train_vec, y_train)

      # Evaluate
      y_pred = model.predict(X_test_vec)
      f1 = f1_score(y_test, y_pred, average='weighted')
      acc = accuracy_score(y_test, y_pred)

      print(f'Smoke Test Results:')
      print(f'  - F1 Score: {f1:.4f}')
      print(f'  - Accuracy: {acc:.4f}')

      # Verify minimum performance threshold
      if f1 < 0.3:
          print('ERROR: Model performance below minimum threshold!')
          sys.exit(1)

      print('Smoke test PASSED!')
      "
  artifacts:
    when: always
    expire_in: 1 day
  rules:
    # Run on schedule (daily/weekly)
    - if: $CI_PIPELINE_SOURCE == "schedule"
    # Manual trigger
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      when: manual
    - if: $CI_COMMIT_BRANCH == "master"
      when: manual

# =============================================================================
# CT - Full Training Pipeline (Scheduled)
# =============================================================================
# This job runs the full training pipeline on a schedule (e.g., weekly)
# to ensure the model stays fresh with new data patterns

ct_full_training:
  stage: smoke
  image: python:3.10-slim
  timeout: 2 hours
  before_script:
    - apt-get update && apt-get install -y gcc g++ --no-install-recommends
    - pip install --upgrade pip
    - pip install -r requirements.txt
  script:
    - echo "============================================"
    - echo "  CONTINUOUS TRAINING - Full Pipeline"
    - echo "============================================"
    - |
      # Run the full training pipeline with Optuna HPT
      python src/training/optuna_sentiment.py \
        --n-trials 5 \
        --model-type lightgbm \
        --max-features 5000 \
        --ngram-min 1 \
        --ngram-max 2 \
        --exp-name "ct_scheduled_$(date +%Y%m%d)"
  artifacts:
    paths:
      - models/
      - mlruns/
    expire_in: 1 week
  rules:
    # Only run on schedule (weekly CT)
    - if: $CI_PIPELINE_SOURCE == "schedule"
      when: always
  allow_failure: true