table.py

#!/usr/bin/env python3

# Copyright 2026 Fondazione Bruno Kessler
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#   https://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import pandas as pd
import numpy as np
import json

class TableGenerator:
    def __init__(self):
        self.benchmarks = {}
        self.metrics = ['Precision', 'Accuracy', 'F1 Score', 'Specificity']
        self.deepeval_data = {}  # Store raw deepeval data by benchmark and model
        self.load_data()
        
    def clean_name(self, name):
        """Clean up folder names by replacing weird characters with colon and removing ':latest' suffix"""
        # Replace the weird character with colon (assuming it's a dash or underscore)
        # We're looking for patterns like "name-latest" or "name_latest" that should become "name"
        if name.endswith("latest"):
            name = name[:-7]  # Remove "-latest"
        
        # Remove excessive colons that appear in the output
        name = name.replace(":", "")
        
        # Format multiagent prefix as "multi-agent_"
        if name.lower().startswith("multiagent"):
            name = "multi-agent_" + name[len("multiagent"):]
        
        return name
    
    def load_data(self):
        """Load all benchmark results data from the benchmarks directory"""
        benchmark_folders = []
        for root, dirs, files in os.walk('benchmarks'):
            for dir_name in dirs:
                if 'results' not in dir_name and 'results' not in root:
                    benchmark_folders.append(os.path.join(root, dir_name))
        
        for benchmark_folder in benchmark_folders:
            raw_benchmark_name = os.path.basename(benchmark_folder)
            benchmark_name = self.clean_name(raw_benchmark_name)
            results_folders = []
            
            # Find all results* folders
            for item in os.listdir(benchmark_folder):
                item_path = os.path.join(benchmark_folder, item)
                if os.path.isdir(item_path) and item.startswith('results'):
                    # Skip folders with "mistral" in the name
                    if "mistral" not in item.lower():
                        results_folders.append(item_path)
            
            if results_folders:
                self.benchmarks[benchmark_name] = {}
                # Initialize deepeval data for this benchmark
                if benchmark_name not in self.deepeval_data:
                    self.deepeval_data[benchmark_name] = {}
                
                # Load data from each results folder
                for results_folder in results_folders:
                    raw_model_name = os.path.basename(results_folder).replace('results', '')
                    if not raw_model_name:
                        model_name = 'default'
                    else:
                        model_name = self.clean_name(raw_model_name)
                    
                    csv_path = os.path.join(results_folder, 'metrics_results.csv')
                    if os.path.exists(csv_path):
                        df = pd.read_csv(csv_path)
                        # Filter to keep only the confusion matrix values
                        conf_matrix_metrics = ['True Positives', 'True Negatives', 'False Positives', 'False Negatives']
                        conf_matrix_df = df[df['Metric'].isin(conf_matrix_metrics)]
                        
                        if not conf_matrix_df.empty:
                            self.benchmarks[benchmark_name][model_name] = conf_matrix_df

    def calculate_metric(self, df, metric_name):
        """Calculate specified metric using confusion matrix values"""
        # Extract confusion matrix values
        try:
            tp = float(df[df['Metric'] == 'True Positives']['Value'].values[0])
            tn = float(df[df['Metric'] == 'True Negatives']['Value'].values[0])
            fp = float(df[df['Metric'] == 'False Positives']['Value'].values[0])
            fn = float(df[df['Metric'] == 'False Negatives']['Value'].values[0])
        except (IndexError, ValueError):
            # If any required value is missing
            return None
        
        # Calculate the requested metric
        if metric_name == 'Precision':
            denominator = tp + fp
            return tp / denominator if denominator > 0 else 0
        
        elif metric_name == 'Recall':
            denominator = tp + fn
            return tp / denominator if denominator > 0 else 0
        
        elif metric_name == 'Specificity':
            denominator = tn + fp
            return tn / denominator if denominator > 0 else 0
        
        elif metric_name == 'Accuracy':
            denominator = tp + tn + fp + fn
            return (tp + tn) / denominator if denominator > 0 else 0
        
        elif metric_name == 'F1 Score':
            precision = tp / (tp + fp) if (tp + fp) > 0 else 0
            recall = tp / (tp + fn) if (tp + fn) > 0 else 0
            denominator = precision + recall
            return 2 * (precision * recall) / denominator if denominator > 0 else 0
        
        return None  # Unknown metric
    
    def calculate_overall_metrics(self):
        """Calculate overall metrics for all models across all benchmarks"""
        # This will store {model_name: {metric: value}}
        overall_metrics = {}
        
        # Original logic for confusion matrix metrics
        confusion_matrix_sums = {}  # Model -> {TP, TN, FP, FN}
        
        for benchmark, models_data in self.benchmarks.items():
            for model, df in models_data.items():
                if model not in confusion_matrix_sums:
                    confusion_matrix_sums[model] = {
                        'True Positives': 0,
                        'True Negatives': 0, 
                        'False Positives': 0,
                        'False Negatives': 0
                    }
                
                # Extract and sum confusion matrix values
                try:
                    tp = float(df[df['Metric'] == 'True Positives']['Value'].values[0])
                    tn = float(df[df['Metric'] == 'True Negatives']['Value'].values[0])
                    fp = float(df[df['Metric'] == 'False Positives']['Value'].values[0])
                    fn = float(df[df['Metric'] == 'False Negatives']['Value'].values[0])
                    
                    confusion_matrix_sums[model]['True Positives'] += tp
                    confusion_matrix_sums[model]['True Negatives'] += tn
                    confusion_matrix_sums[model]['False Positives'] += fp
                    confusion_matrix_sums[model]['False Negatives'] += fn
                except (IndexError, ValueError):
                    # Skip this model if values are missing
                    continue
        
        # Create a pseudo-dataframe for each model and calculate metrics
        for model, cm_values in confusion_matrix_sums.items():
            # Create a dataframe-like structure with the summed values
            pseudo_df = pd.DataFrame({
                'Metric': list(cm_values.keys()),
                'Value': list(cm_values.values())
            })
            
            # Calculate each metric
            overall_metrics[model] = {}
            for metric in self.metrics:
                metric_value = self.calculate_metric(pseudo_df, metric)
                if metric_value is not None:
                    overall_metrics[model][metric] = metric_value
        
        return overall_metrics
    
    def pair_models(self, metrics_dict):
        """Group models with their multiagent versions"""
        # Create mapping between base models and their multiagent versions
        model_dict = {}
        for model in metrics_dict.keys():
            # Clean the model name first to remove stray colons
            clean_model = model.replace(":", "")
            
            # Handle both old 'multiagent' and new 'multi-agent_' patterns
            base_name = clean_model.lower()
            if base_name.startswith('multi-agent_'):
                base_name = base_name[len('multi-agent_'):]
            elif 'multiagent' in base_name:
                base_name = base_name.replace('multiagent', '')
                
            # Remove any remaining special characters and normalize
            base_name = ''.join(c for c in base_name if c.isalnum() or c in ['-', '_'])
            base_name = base_name.strip('-_')
            
            # Create entry for this base model if it doesn't exist
            if base_name not in model_dict:
                model_dict[base_name] = {'base': None, 'multiagent': None}
            
            # Assign to either base or multiagent slot
            if 'multiagent' in clean_model.lower() or clean_model.lower().startswith('multi-agent_'):
                model_dict[base_name]['multiagent'] = model
            else:
                model_dict[base_name]['base'] = model
        
        # Sort base names alphabetically
        sorted_base_names = sorted(model_dict.keys())
        
        # Create paired sorted list
        paired_models = []
        for base_name in sorted_base_names:
            # Add base model first if it exists
            if model_dict[base_name]['base']:
                paired_models.append(model_dict[base_name]['base'])
            
            # Add multiagent version if it exists
            if model_dict[base_name]['multiagent']:
                paired_models.append(model_dict[base_name]['multiagent'])
        
        # Check if pairing worked - if not, fall back to alphabetical
        if len(paired_models) < len(metrics_dict):
            print("Warning: Some models could not be paired. Using alphabetical ordering.")
            return sorted(metrics_dict.keys())
        
        return paired_models
    
    def generate_latex_table(self):
        """Generate a LaTeX table with the metrics"""
        overall_metrics = self.calculate_overall_metrics()
        paired_models = self.pair_models(overall_metrics)
        
        # Start LaTeX table
        latex_output = [
            "\\begin{table}[htbp]",
            "\\centering",
            "\\begin{tabular}{l|rrrr}", 
            "\\hline",
            "Model & Precision & Accuracy & F1 Score & Specificity \\\\",
            "\\hline"
        ]
        
        # Add rows for each model
        for model in paired_models:
            if model in overall_metrics:
                metrics = overall_metrics[model]
                
                # Clean model name for display
                display_name = model.replace(":", "")
                row = f"{display_name} & "
                
                # Add each metric with formatting
                for i, metric in enumerate(self.metrics):
                    if metric in metrics:
                        value = f"{metrics[metric]:.3f}"
                    else:
                        value = "N/A"
                    
                    row += value
                    if i < len(self.metrics) - 1:
                        row += " & "
                
                latex_output.append(row + " \\\\")
        
        # Close the table
        latex_output.extend([
            "\\hline",
            "\\end{tabular}",
            "\\caption{Overall Performance Metrics by Model}",
            "\\label{tab:overall_metrics}",
            "\\end{table}"
        ])
        
        return "\n".join(latex_output)

def main():
    generator = TableGenerator()
    latex_table = generator.generate_latex_table()
    print(latex_table)
    
    # Also save to file
    with open('metrics_table.tex', 'w') as f:
        f.write(latex_table)
    print(f"LaTeX table saved to 'metrics_table.tex'")

if __name__ == "__main__":
    main()