Few-shot Benchmark Report

Run ID: 20260507T192344Z | Generated UTC: 2026-05-07T19:23:56.733695+00:00

Configuration

{
  "models": [
    "sonnet-4-6"
  ],
  "fewshot_strategies": [
    "raw_only"
  ],
  "chat_count": 20,
  "runs_per_chat": 1,
  "max_workers": 15,
  "raw_fewshot_count": 10,
  "raw_fewshot_labels_filter": [
    "[chats] multiple_product_multiple_shipment_complex.json",
    "[chats] single_product_multiple_shipment_complex.json",
    "[chats] single_product_single_shipment_complex.json",
    "[updates] update_change_quantity.json",
    "[updates] update_change_unit_price.json"
  ]
}

Model + Strategy Summary

Model	Strategy	Runs	Success rate	Avg attempts	Avg elapsed (s)	Avg mismatch/expected run	Field match rate
sonnet-4-6	raw_only	20	1.0000	1.0000	6.5484	3.4500	0.8988

Per-chat Breakdown

Chat	Model	Strategy	Runs	Success rate	Avg elapsed (s)	Mismatch counts
multiple_product_multiple_shipment_medium.json	sonnet-4-6	raw_only	2	1.0000	8.1882	[7, 7]
multiple_product_multiple_shipment_simple.json	sonnet-4-6	raw_only	2	1.0000	6.5055	[7, 1]
real_world_msgs_test_v1.json	sonnet-4-6	raw_only	2	1.0000	5.8689	[5, 5]
real_world_msgs_test_v2.json	sonnet-4-6	raw_only	2	1.0000	8.9747	[5, 3]
real_world_msgs_test_v3.json	sonnet-4-6	raw_only	2	1.0000	6.4257	[4, 4]
single_product_multiple_shipment_complex.json	sonnet-4-6	raw_only	2	1.0000	7.4196	[1, 0]
single_product_multiple_shipment_medium.json	sonnet-4-6	raw_only	2	1.0000	5.7999	[1, 1]
single_product_multiple_shipment_simple.json	sonnet-4-6	raw_only	2	1.0000	5.8030	[0, 0]
single_product_single_shipment_complex.json	sonnet-4-6	raw_only	2	1.0000	5.6489	[6, 6]
single_product_single_shipment_medium.json	sonnet-4-6	raw_only	2	1.0000	4.8492	[3, 3]

Top Mismatches (up to 100 runs)

Chat	Model	Strategy	Run	Mismatch count	Sample mismatches
multiple_product_multiple_shipment_simple.json	sonnet-4-6	raw_only	1	7	[ { "path": "data[0].items[0].unit_price", "expected": 25.0, "actual": 250.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/BAG", "actual": "" }, { "path": "data[0].items[0].total", "expected": 250.0, "actual": 2500.0 }, { "path": "data[0].items[1].description", "expected": "Assam tea", "actual": "Assam Tea" }, { "path": "data[0].items[1].unit_price", "expected": 12.0, "actual": 240.0 } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-6	raw_only	1	7	[ { "path": "data", "expected_len": 1, "actual_len": 2 }, { "path": "data[0].items", "expected_len": 3, "actual_len": 1 }, { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": null, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "", "actual": "USD/BAG" } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-6	raw_only	1	7	[ { "path": "data", "expected_len": 1, "actual_len": 2 }, { "path": "data[0].items", "expected_len": 3, "actual_len": 1 }, { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": null, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "", "actual": "USD/BAG" } ]
single_product_single_shipment_complex.json	sonnet-4-6	raw_only	1	6	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].shipment_date", "expected": "2025-11-28", "actual": "2026-11-28" }, { "path": "data[0].items[0].shipping_address", "expected": "352 Indiana Jones St.", "actual": "352 Indiana Jones St" }, { "path": "data[0].do_date", "expected": "2025-11-28", "actual": "2026-11-28" } ]
single_product_single_shipment_complex.json	sonnet-4-6	raw_only	1	6	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].shipment_date", "expected": "2025-11-28", "actual": "2026-11-28" }, { "path": "data[0].items[0].shipping_address", "expected": "352 Indiana Jones St.", "actual": "352 Indiana Jones St" }, { "path": "data[0].do_date", "expected": "2025-11-28", "actual": "2026-11-28" } ]
real_world_msgs_test_v2.json	sonnet-4-6	raw_only	1	5	[ { "path": "data[0].vendor_name", "expected": "AG Lipids Pte Ltd", "actual": "Van Beethoven" }, { "path": "data[1].items[0].unit_price", "expected": 4.2, "actual": 4200.0 }, { "path": "data[1].items[0].pricing_unit", "expected": "USD/KG", "actual": "USD/MT" }, { "path": "data[1].items[0].loading", "expected": "", "actual": "18MT/40'FCL" }, { "path": "data[1].vendor_name", "expected": "AG Lipids Pte Ltd", "actual": "Van Beethoven" } ]
real_world_msgs_test_v1.json	sonnet-4-6	raw_only	1	5	[ { "path": "data[0].items[0].description", "expected": "soy lecithin powder", "actual": "Soy Lecithin Powder" }, { "path": "data[0].items[0].unit_price", "expected": 4.1, "actual": 4100.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/KG", "actual": "USD/MT" }, { "path": "data[0].items[0].loading", "expected": "12MT/20'FCL", "actual": "2x 12MT/20'FCL" }, { "path": "data[0].shipping_method", "expected": "", "actual": "by sea" } ]
real_world_msgs_test_v1.json	sonnet-4-6	raw_only	1	5	[ { "path": "data[0].items[0].description", "expected": "soy lecithin powder", "actual": "Soy Lecithin Powder" }, { "path": "data[0].items[0].unit_price", "expected": 4.1, "actual": 4100.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/KG", "actual": "USD/MT" }, { "path": "data[0].items[0].loading", "expected": "12MT/20'FCL", "actual": "2x 12MT/20'FCL" }, { "path": "data[0].shipping_method", "expected": "", "actual": "by sea" } ]
real_world_msgs_test_v3.json	sonnet-4-6	raw_only	1	4	[ { "path": "data[0].items[0].description", "expected": "lecithin fat powder", "actual": "Lecithin Fat Powder" }, { "path": "data[0].items[0].total", "expected": 96000.0, "actual": null }, { "path": "data[0].items[1].description", "expected": "lecithin fat powder", "actual": "Lecithin Fat Powder" }, { "path": "data[0].items[1].total", "expected": 144000.0, "actual": null } ]
real_world_msgs_test_v3.json	sonnet-4-6	raw_only	1	4	[ { "path": "data[0].items[0].description", "expected": "lecithin fat powder", "actual": "Lecithin Fat Powder" }, { "path": "data[0].items[0].total", "expected": 96000.0, "actual": null }, { "path": "data[0].items[1].description", "expected": "lecithin fat powder", "actual": "Lecithin Fat Powder" }, { "path": "data[0].items[1].total", "expected": 144000.0, "actual": null } ]
single_product_single_shipment_medium.json	sonnet-4-6	raw_only	1	3	[ { "path": "data[0].items[0].shipment_date", "expected": "2026-05-28", "actual": "2026-05-31" }, { "path": "data[0].do_date", "expected": "2026-05-28", "actual": "2026-05-31" }, { "path": "data[0].payment_date", "expected": "Net 30 from delivery", "actual": "" } ]
single_product_single_shipment_medium.json	sonnet-4-6	raw_only	1	3	[ { "path": "data[0].items[0].shipment_date", "expected": "2026-05-28", "actual": "2026-05-31" }, { "path": "data[0].do_date", "expected": "2026-05-28", "actual": "2026-05-31" }, { "path": "data[0].payment_date", "expected": "Net 30 from delivery", "actual": "" } ]
real_world_msgs_test_v2.json	sonnet-4-6	raw_only	1	3	[ { "path": "data[1].items[0].unit_price", "expected": 4.2, "actual": 4200.0 }, { "path": "data[1].items[0].pricing_unit", "expected": "USD/KG", "actual": "USD/MT" }, { "path": "data[1].items[0].loading", "expected": "", "actual": "18MT/40'FCL" } ]
single_product_multiple_shipment_medium.json	sonnet-4-6	raw_only	1	1	[ { "path": "data[0].do_date", "expected": "2026-05-31", "actual": "" } ]
single_product_multiple_shipment_medium.json	sonnet-4-6	raw_only	1	1	[ { "path": "data[0].do_date", "expected": "2026-05-31", "actual": "" } ]
single_product_multiple_shipment_complex.json	sonnet-4-6	raw_only	1	1	[ { "path": "data[0].do_date", "expected": "2026-03-10", "actual": "" } ]
multiple_product_multiple_shipment_simple.json	sonnet-4-6	raw_only	1	1	[ { "path": "data[0].items[1].description", "expected": "Assam tea", "actual": "Assam Tea" } ]