{
  "benchmark_id": "cautomates-agent-utility-index-001",
  "benchmark_name": "CAutomates Agent Utility Index: Benchmark #001",
  "published_at": "2026-08-20T13:36:00.000Z",
  "updated_at": "2026-08-20T20:28:00.000Z",
  "primary_metric": "reference_normalized_cost_per_correct_field",
  "decision": "CONTINUE_INDEX",
  "scope_note": "Results apply only to the tested Benchmark #001 page sets, extraction tasks, scoring methodology, runtime conditions, and pricing assumptions. The 35-URL provider index and 45-URL routing holdout are separate datasets and should not be treated as one sample.",
  "intended_use_cases": [
    "AI web-research agents",
    "structured web extraction",
    "product and pricing comparison agents",
    "business-research workflows",
    "evidence-sensitive agent workflows",
    "retrieval-provider selection",
    "multi-provider routing research"
  ],
  "provider_index_35_url": {
    "dataset_size_urls": 35,
    "providers": [
      {
        "provider": "Firecrawl",
        "retrieval_success_rate": 0.962,
        "difficult_page_recovery_rate": 0.97,
        "difficult_page_rate_method": "attempt_level_33_attempts",
        "evaluator_evidence_coverage_rate": 0.912,
        "correct_fields": 51,
        "unsupported_claim_rate": 0.0865,
        "tokens_per_correct_field": 12716,
        "retrieval_median_latency_ms_approx": 1132,
        "reference_normalized_cost_per_correct_field_usd": 0.003644,
        "rank_by_primary_metric": 1
      },
      {
        "provider": "Bright Data MCP",
        "calls": 105,
        "successful_calls": 99,
        "empty_content_results": 6,
        "errors": 0,
        "retrieval_success_rate": 0.943,
        "difficult_page_recovery_rate": 0.909,
        "difficult_page_rate_method": "url_level_majority_11_urls_three_runs_each",
        "correct_fields": 57,
        "unsupported_claims": 0,
        "claims_total": 93,
        "unsupported_claim_rate": 0,
        "evaluator_evidence_coverage_rate": 0.9412,
        "evaluator_input_tokens": 443469,
        "evaluator_output_tokens": 93818,
        "tokens_per_correct_field": 7780,
        "retrieval_median_latency_ms": 4179,
        "retrieval_p95_latency_ms": 11746,
        "median_total_latency_ms": 13006,
        "p95_total_latency_ms": 21262,
        "actual_free_tier_workflow_cost_usd": 0.059701,
        "actual_cost_per_correct_field_usd": 0.001047,
        "reference_normalized_cost_usd": 0.217201,
        "reference_normalized_cost_per_correct_field_usd": 0.003811,
        "rank_by_primary_metric": 2
      },
      {
        "provider": "Tavily",
        "retrieval_success_rate": 0.743,
        "difficult_page_recovery_rate": 0.273,
        "difficult_page_rate_method": "attempt_level_33_attempts",
        "evaluator_evidence_coverage_rate": 0.912,
        "correct_fields": 36,
        "unsupported_claim_rate": 0,
        "reference_normalized_cost_per_correct_field_usd": 0.004551,
        "rank_by_primary_metric": 3
      },
      {
        "provider": "Direct fetch",
        "retrieval_success_rate": 0.686,
        "difficult_page_recovery_rate": 0,
        "difficult_page_rate_method": "attempt_level_33_attempts",
        "evaluator_evidence_coverage_rate": 0.824,
        "correct_fields": 11,
        "unsupported_claim_rate": 0,
        "reference_normalized_cost_per_correct_field_usd": 0.008151,
        "rank_by_primary_metric": 4
      },
      {
        "provider": "ScrapingBee",
        "retrieval_success_rate": 0.724,
        "difficult_page_recovery_rate": 0.182,
        "difficult_page_rate_method": "attempt_level_33_attempts",
        "evaluator_evidence_coverage_rate": 0.814,
        "correct_fields": 13,
        "unsupported_claim_rate": 0,
        "reference_normalized_cost_per_correct_field_usd": 0.009856,
        "rank_by_primary_metric": 5
      }
    ],
    "difficult_page_overlap_firecrawl_bright_data": {
      "both_recovered": 10,
      "firecrawl_only": 1,
      "bright_data_only": 0,
      "both_failed": 0,
      "firecrawl_only_test_id": "pr05"
    },
    "bright_data_unique_value": {
      "unique_fields": 1,
      "unique_urls": 1,
      "test_id": "bl08",
      "reference_cost_per_unique_correct_field_usd": 0.217201
    },
    "pairwise_retrieval_overlap": [
      { "pair": "Firecrawl vs Bright Data", "both_succeed": 33, "first_only": 2, "second_only": 0, "both_fail": 0 },
      { "pair": "Tavily vs Bright Data", "both_succeed": 25, "first_only": 1, "second_only": 8, "both_fail": 1 },
      { "pair": "Firecrawl vs Tavily", "both_succeed": 26, "first_only": 9, "second_only": 0, "both_fail": 0 }
    ],
    "updated_oracle": {
      "correct_fields": 21,
      "reference_normalized_cost_per_correct_field_usd": 0.00228,
      "advantage_vs_best_single_provider_percent": 37.43,
      "caveat": "Many direct-fetch URL wins are zero-cost ties on URLs where every provider scored zero correct fields; practical economic value is concentrated in successful provider wins."
    }
  },
  "routing_holdout_45_url": {
    "dataset_size_urls": 45,
    "categories": 7,
    "overlap_with_35_url_index": 0,
    "ground_truth_subset_urls": 21,
    "collection_passes": 1,
    "actual_total_phase_spend_usd_approx": 0.51,
    "sample_size_caveat": "Several correct-field totals are in the single digits, so holdout findings are directional evidence rather than statistically settled universal rankings.",
    "strategies": [
      {
        "strategy": "ALWAYS_DIRECT",
        "correct_fields": 1,
        "total_cost_usd": 0.0218,
        "cost_per_correct_field_usd": 0.0218,
        "median_total_latency_ms": 5566,
        "p95_total_latency_ms": 8845,
        "evaluator_evidence_coverage_rate": 0.895,
        "unsupported_claim_rate": 0
      },
      {
        "strategy": "DIRECT_THEN_FIRECRAWL",
        "correct_fields": 1,
        "total_cost_usd": 0.0257,
        "cost_per_correct_field_usd": 0.02568,
        "median_total_latency_ms": 6593,
        "p95_total_latency_ms": 15934,
        "evaluator_evidence_coverage_rate": 0.895,
        "unsupported_claim_rate": 0
      },
      {
        "strategy": "ROUTER_V1_NO_TAVILY",
        "correct_fields": 1,
        "total_cost_usd": 0.0257,
        "cost_per_correct_field_usd": 0.02568,
        "median_total_latency_ms": null,
        "p95_total_latency_ms": null,
        "evaluator_evidence_coverage_rate": 0.895,
        "unsupported_claim_rate": 0
      },
      {
        "strategy": "ALWAYS_FIRECRAWL",
        "correct_fields": 5,
        "total_cost_usd": 0.0399,
        "cost_per_correct_field_usd": 0.00798,
        "median_total_latency_ms": 8163,
        "p95_total_latency_ms": 17959,
        "evaluator_evidence_coverage_rate": 0.947,
        "unsupported_claim_rate": 0.143
      },
      {
        "strategy": "ROUTER_V1",
        "correct_fields": 5,
        "total_cost_usd": 0.0708,
        "cost_per_correct_field_usd": 0.01416,
        "median_total_latency_ms": 6593,
        "p95_total_latency_ms": 18036,
        "evaluator_evidence_coverage_rate": 0.895,
        "unsupported_claim_rate": 0
      },
      {
        "strategy": "ALWAYS_TAVILY",
        "correct_fields": 10,
        "total_cost_usd": 0.1837,
        "cost_per_correct_field_usd": 0.01837,
        "median_total_latency_ms": 8975,
        "p95_total_latency_ms": 13492,
        "evaluator_evidence_coverage_rate": 1,
        "unsupported_claim_rate": 0
      },
      {
        "strategy": "ORACLE",
        "correct_fields": 11,
        "total_cost_usd": 0.0631,
        "cost_per_correct_field_usd": 0.00574,
        "median_total_latency_ms": null,
        "p95_total_latency_ms": null,
        "evaluator_evidence_coverage_rate": null,
        "unsupported_claim_rate": 0
      }
    ],
    "router_v1_vs_always_firecrawl": {
      "router_cost_increase_percent": 77.5,
      "router_correct_fields": 5,
      "firecrawl_correct_fields": 5
    },
    "router_v1_vs_always_direct": {
      "correct_field_count_increase_percent": 400,
      "router_correct_fields": 5,
      "direct_correct_fields": 1
    },
    "tavily_removal": {
      "router_with_tavily_correct_fields": 5,
      "router_without_tavily_correct_fields": 1,
      "quality_loss_percent": 80,
      "router_with_tavily_total_cost_usd": 0.0708,
      "router_without_tavily_total_cost_usd": 0.0257,
      "router_with_tavily_cost_per_correct_field_usd": 0.01416,
      "router_without_tavily_cost_per_correct_field_usd": 0.02568
    },
    "oracle_vs_always_firecrawl": {
      "oracle_correct_fields": 11,
      "firecrawl_correct_fields": 5,
      "oracle_cost_per_correct_field_usd": 0.00574,
      "firecrawl_cost_per_correct_field_usd": 0.00798,
      "oracle_cost_advantage_percent": 28
    },
    "router_regret": {
      "wrong_provider_choices_on_ground_truth_subset": 11,
      "ground_truth_subset_urls": 21,
      "regret_rate": 0.524,
      "avoidable_cost_usd": 0.0261,
      "avoidable_correct_field_loss": 6
    }
  },
  "routing_opportunity_assessment": {
    "economic_headroom": "STRONG",
    "provider_complementarity": "MODERATE",
    "pre_purchase_predictability": "WEAK",
    "recommendation": "CONTINUE_INDEX",
    "router_mvp_supported": false
  },
  "canonical_article": "https://chrissaautomates.com/post/ai-web-retrieval-cost-per-correct-field-benchmark-001"
}