Add single-outlet evaluation script and sample hybrid output

Adds eval_single_output.py for computing ROUGE-L, METEOR, FACTScore
precision/recall, and error rate on a single pre-computed output against
gold labels. Includes Newfoundland Independent hybrid output for testing.

https://claude.ai/code/session_01FzSqaEMj4tQogQY87L8fzA

eval_single_output.py

@@ -0,0 +1,148 @@
+"""
+Evaluate a single pre-computed output against gold labels.
+
+Usage:
+  python eval_single_output.py --input evaluation_dataset.json --output hybrid_output.json --outlet "Newfoundland Independent"
+
+Where hybrid_output.json contains the raw_output from the hybrid run.
+"""
+
+import json
+import argparse
+import logging
+from metrics import MetricsCalculator
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+
+def evaluate_output(gold_item: dict, generated: dict) -> dict:
+    """Compute all metrics for a single outlet given gold and generated data."""
+    metrics_calc = MetricsCalculator()
+
+    # Construct text blocks (same logic as run_benchmark.py evaluate_single)
+    gen_text = " ".join(filter(None, [
+        generated.get('bias_category_description'),
+        generated.get('overall_summary'),
+        generated.get('analysis'),
+        generated.get('history'),
+        generated.get('ownership'),
+    ])).strip()
+
+    gold_text_full = "\n\n".join(filter(None, [
+        gold_item.get('bias_category_description', ''),
+        gold_item.get('overall_summary', ''),
+        gold_item.get('history', ''),
+        gold_item.get('analysis', ''),
+        gold_item.get('ownership', ''),
+    ]))
+
+    gold_summary = " ".join(filter(None, [
+        gold_item.get('bias_category_description', ''),
+        gold_item.get('overall_summary', ''),
+        gold_item.get('analysis', ''),
+        gold_item.get('history', ''),
+        gold_item.get('ownership', ''),
+    ]))
+
+    # A. Text Overlap
+    logger.info("Computing ROUGE-L and METEOR...")
+    text_m = metrics_calc.calculate_text_metrics([gold_summary], [gen_text])
+
+    # B. FactScore Precision: Generated -> Gold
+    logger.info("Computing FACTScore Precision...")
+    fs_precision = metrics_calc.check_fact_recall(gold_text_full, gen_text)
+
+    # C. Fact Recall: Gold -> Generated
+    logger.info("Computing FACTScore Recall...")
+    fs_recall = metrics_calc.check_gold_fact_recall(gold_text_full, gen_text)
+
+    # D. Fact Check Detection
+    logger.info("Computing Fact Check Detection...")
+    gold_fc = gold_item.get('failed_fact_checks', [])
+    gen_fc_list = generated.get('failed_fact_checks', [])
+    fc_m = MetricsCalculator.evaluate_fact_checks(gold_fc, gen_text, gen_fc_list)
+
+    result = {
+        "name": gold_item['name'],
+        "source_url": gold_item.get('source_url', ''),
+        "gold_bias": gold_item.get('bias_score', 0),
+        "pred_bias": generated.get('bias_score'),
+        "gold_factuality": gold_item.get('factual_score', 0),
+        "pred_factuality": generated.get('factual_score'),
+        "rougeL": text_m['rougeL'],
+        "meteor": text_m['meteor'],
+        "factscore_precision": fs_precision.get('factscore', 0.0),
+        "error_rate": fs_precision.get('error_rate', 0.0),
+        "factscore_recall": fs_recall.get('gold_recall', 0.0),
+        "fact_check_hit": fc_m,
+    }
+
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Evaluate a single pre-computed output")
+    parser.add_argument("--input", type=str, default="evaluation_dataset.json",
+                        help="Path to evaluation dataset JSON")
+    parser.add_argument("--output", type=str, required=True,
+                        help="Path to JSON file with the generated output (raw_output)")
+    parser.add_argument("--outlet", type=str, required=True,
+                        help="Name of the outlet to evaluate")
+    args = parser.parse_args()
+
+    # Load gold dataset
+    with open(args.input) as f:
+        dataset = json.load(f)
+
+    # Find gold item
+    gold_item = None
+    for item in dataset:
+        if item['name'].lower() == args.outlet.lower():
+            gold_item = item
+            break
+
+    if not gold_item:
+        logger.error(f"Outlet '{args.outlet}' not found in {args.input}")
+        return
+
+    # Load generated output
+    with open(args.output) as f:
+        generated = json.load(f)
+
+    # If the file has a 'raw_output' key (from benchmark results), use that
+    if 'raw_output' in generated:
+        generated = generated['raw_output']
+    # If wrapped in 'output' key (from hybrid runner results)
+    if 'output' in generated:
+        generated = generated['output']
+
+    logger.info(f"Evaluating: {gold_item['name']}")
+    logger.info(f"  Gold bias: {gold_item.get('bias_score')}, Pred bias: {generated.get('bias_score')}")
+    logger.info(f"  Gold factuality: {gold_item.get('factual_score')}, Pred factuality: {generated.get('factual_score')}")
+
+    result = evaluate_output(gold_item, generated)
+
+    # Print results
+    print("\n" + "=" * 60)
+    print(f"EVALUATION RESULTS: {result['name']}")
+    print("=" * 60)
+    print(f"  Bias:       gold={result['gold_bias']}, pred={result['pred_bias']}")
+    print(f"  Factuality: gold={result['gold_factuality']}, pred={result['pred_factuality']}")
+    print(f"  ROUGE-L:           {result['rougeL']:.4f}")
+    print(f"  METEOR:            {result['meteor']:.4f}")
+    print(f"  FACTScore Prec:    {result['factscore_precision']:.4f}")
+    print(f"  Error Rate:        {result['error_rate']:.4f}")
+    print(f"  FACTScore Recall:  {result['factscore_recall']:.4f}")
+    print(f"  Fact Check Hit:    {result['fact_check_hit']}")
+    print("=" * 60)
+
+    # Save full result
+    out_path = args.output.replace('.json', '_evaluated.json')
+    with open(out_path, 'w') as f:
+        json.dump(result, f, indent=2, default=str)
+    print(f"\nFull result saved to: {out_path}")
+
+
+if __name__ == "__main__":
+    main()

newfoundland_hybrid_output.json

@@ -0,0 +1,23 @@
+{
+  "output": {
+    "mbfc_url": null,
+    "name": "Newfoundland Independent",
+    "source_url": "https://theindependent.ca/",
+    "bias_rating": "LEFT-CENTER",
+    "bias_score": -4.5,
+    "factual_reporting": "MOSTLY FACTUAL",
+    "factual_score": 2.9,
+    "credibility_rating": "MEDIUM CREDIBILITY",
+    "country": "Canada",
+    "country_freedom_rating": "Free",
+    "media_type": "Website",
+    "traffic_popularity": "Unknown",
+    "bias_category_description": "Left-Center \u2014 leans progressive and advocacy-focused, emphasizing community voices, labour, anti-racism and climate coverage while maintaining journalistic reporting elements.",
+    "overall_summary": "Overall, we rate Newfoundland Independent as Left-Center based on its sustained progressive editorial posture, frequent emphasis on critics and community advocates, and use of emotive framing. We also rate them Mostly Factual based on generally accurate reporting, named primary local sources, no found failed fact-checks, but moderate sourcing/linking and limited site transparency.",
+    "history": "Founded in 2011; Founder: Unknown; Original name: N/A. The Independent emerged from a predecessor St. John\u2019s print newspaper (2003\u20132008), re-launched online in early 2011 as a not-for-profit-minded media collective under editors including Justin Brake and Rhea Rollmann, gained recognition for investigative/community reporting (notably Muskrat Falls coverage and related press-freedom litigation), and has received regional journalism awards. Current headquarters: Unknown from supplied evidence.",
+    "ownership": "Owner/parent company: Unknown. Funding model: Unknown (site references 'reader-supported' in a newsletter snippet but no formal funding disclosures were provided in the evidence). Headquarters location: Unknown based on the supplied materials.",
+    "analysis": "The outlet's coverage and opinion pieces show a consistent left-of-center editorial direction: articles foreground anti-racism, labour and community voices, climate science and skepticism of fossil-fuel narratives, and advocacy for expanded social supports (examples: pieces on homelessness and NDP leader comments, Black History Month coverage, fisheries/climate reporting). Straight news items include government statements and official quotes but selection and framing disproportionately center critics, advocates and policy deficits rather than equivalent conservative advocacy (pipeline examples: Article 1 on PC budget engagement frames critics' views; Article 3 highlights NDP leader criticism; Article 5 frames government treatment of Black History Month as 'disappointing'). Sourcing quality is moderate \u2014 many articles name primary local sources (government departments, ministers, community leaders) but links to original releases/reports are rarely provided and some attributions are vague ('critics say', 'they say'), which reduces transparency and verifiability. The pipeline found no discrete failed fact-checks across searches of major fact-check databases (PolitiFact, Reuters Fact Check, AP Fact Check, FactCheck.org, Snopes, FullFact, LeadStories). Pseudoscience detection returned no promotion of fringe science and reporting aligns with mainstream scientific consensus where applicable. Overall: clear progressive editorial viewpoint with generally factual reporting, moderate sourcing practices, and limited site-level transparency.",
+    "failed_fact_checks": [],
+    "last_updated": "2026-03-18"
+  }
+}