run_eval.sh

#!/bin/bash
# =============================================================================
# run_eval.sh — Media Profiling Benchmark
#
# Runs: models × modes × samples
# Output: results/ directory with per-combo JSONL + summary table
# =============================================================================

set -e

# --- Configuration -----------------------------------------------------------
# Models to evaluate (space-separated).
# Edit this list or pass via CLI override.
MODELS=(
    "gpt-5.2-2025-12-11"
    "gpt-5-mini-2025-08-07"
    "gpt-5-nano-2025-08-07"
    "gpt-5-2025-08-07"
    "gpt-4.1-2025-04-14"
)

# Modes to evaluate
MODES="llm articles search system"

# Samples: use --limit 5 for smoke test, --limit 50 for full run, 0 for all
LIMIT=${1:-5}  # Default: 5 samples (smoke test). Pass 50 for full run.

DATASET="evaluation_dataset.json"
# -----------------------------------------------------------------------------

echo "============================================"
echo " Media Profiling Benchmark"
echo "============================================"
echo " Models: ${MODELS[*]}"
echo " Modes:  ${MODES}"
echo " Limit:  ${LIMIT} samples"
echo " Dataset: ${DATASET}"
echo "============================================"

# Step 0: Ensure dataset exists
if [ ! -f "$DATASET" ]; then
    echo ""
    echo "Dataset not found. Building from evaluation_dataset_full.json..."
    python3 dataset_builder.py
fi

# Clean old results (optional — comment out to keep previous runs)
# rm -rf results/

# Step 1: Run the full grid
python3 run_benchmark.py \
    --input "$DATASET" \
    --limit "$LIMIT" \
    --models ${MODELS[@]} \
    --modes $MODES

echo ""
echo "============================================"
echo " Evaluation Complete!"
echo " Results in: results/"
echo "   - Per-outlet: results/<model>_<mode>.jsonl"
echo "   - Summary:    results/summary_table.tsv"
echo "   - Summary:    results/summary_table.json"
echo "============================================"