Few-shot Benchmark Report

Run ID: 20260505T180307Z | Generated UTC: 2026-05-05T18:07:21.150598+00:00

Configuration

{
  "models": [
    "opus-4-5",
    "opus-4-6",
    "sonnet-4-5",
    "sonnet-4-6"
  ],
  "fewshot_strategies": [
    "db_only",
    "db_plus_raw",
    "raw_only",
    "zero_shot"
  ],
  "chat_count": 12,
  "runs_per_chat": 2,
  "max_workers": 15,
  "raw_fewshot_count": 26,
  "raw_fewshot_labels_filter": []
}

Model + Strategy Summary

Model	Strategy	Runs	Success rate	Avg attempts	Avg elapsed (s)	Avg mismatch/expected run	Field match rate
opus-4-5	db_only	24	1.0000	1.0000	9.0073	15.4583	0.5726
opus-4-5	db_plus_raw	24	1.0000	1.0000	12.7979	15.0833	0.5829
opus-4-5	raw_only	24	1.0000	1.0000	21.7177	14.2083	0.6071
opus-4-5	zero_shot	24	1.0000	1.0000	6.5281	12.9167	0.5216
opus-4-6	db_only	24	1.0000	1.0000	8.4391	16.4167	0.4903
opus-4-6	db_plus_raw	24	1.0000	1.0000	19.8156	15.2500	0.5783
opus-4-6	raw_only	24	1.0000	1.0000	7.2128	15.5000	0.5714
opus-4-6	zero_shot	24	1.0000	1.0000	6.0742	9.7917	0.5165
sonnet-4-5	db_only	24	1.0000	1.0000	8.6587	15.0000	0.5663
sonnet-4-5	db_plus_raw	24	1.0000	1.0000	9.0419	15.7500	0.5645
sonnet-4-5	raw_only	24	1.0000	1.0000	7.8363	15.8750	0.5611
sonnet-4-5	zero_shot	24	1.0000	1.0000	6.7775	11.9583	0.5176
sonnet-4-6	db_only	24	1.0000	1.0000	7.7667	16.0417	0.5565
sonnet-4-6	db_plus_raw	24	1.0000	1.0000	9.0471	14.9583	0.5864
sonnet-4-6	raw_only	24	1.0000	1.0000	7.1928	15.2917	0.5772
sonnet-4-6	zero_shot	24	1.0000	1.0000	6.7891	13.0417	0.5437

Per-chat Breakdown

Chat	Model	Strategy	Runs	Success rate	Avg elapsed (s)	Mismatch counts
multiple_product_multiple_shipment_complex.json	opus-4-5	db_only	2	1.0000	13.4062	[41, 41]
multiple_product_multiple_shipment_complex.json	opus-4-5	db_plus_raw	2	1.0000	11.9343	[43, 41]
multiple_product_multiple_shipment_complex.json	opus-4-5	raw_only	2	1.0000	24.5134	[40, 40]
multiple_product_multiple_shipment_complex.json	opus-4-5	zero_shot	2	1.0000	11.8529	[41, 41]
multiple_product_multiple_shipment_complex.json	opus-4-6	db_only	2	1.0000	13.1892	[47, 47]
multiple_product_multiple_shipment_complex.json	opus-4-6	db_plus_raw	2	1.0000	13.3903	[38, 38]
multiple_product_multiple_shipment_complex.json	opus-4-6	raw_only	2	1.0000	12.0736	[40, 40]
multiple_product_multiple_shipment_complex.json	opus-4-6	zero_shot	2	1.0000	9.3648	[10, 10]
multiple_product_multiple_shipment_complex.json	sonnet-4-5	db_only	2	1.0000	12.6261	[38, 40]
multiple_product_multiple_shipment_complex.json	sonnet-4-5	db_plus_raw	2	1.0000	14.4224	[41, 41]
multiple_product_multiple_shipment_complex.json	sonnet-4-5	raw_only	2	1.0000	12.8820	[38, 41]
multiple_product_multiple_shipment_complex.json	sonnet-4-5	zero_shot	2	1.0000	11.1218	[11, 30]
multiple_product_multiple_shipment_complex.json	sonnet-4-6	db_only	2	1.0000	10.3423	[38, 38]
multiple_product_multiple_shipment_complex.json	sonnet-4-6	db_plus_raw	2	1.0000	12.0421	[38, 33]
multiple_product_multiple_shipment_complex.json	sonnet-4-6	raw_only	2	1.0000	11.2341	[37, 37]
multiple_product_multiple_shipment_complex.json	sonnet-4-6	zero_shot	2	1.0000	10.5679	[31, 31]
multiple_product_multiple_shipment_medium.json	opus-4-5	db_only	2	1.0000	10.4988	[24, 24]
multiple_product_multiple_shipment_medium.json	opus-4-5	db_plus_raw	2	1.0000	9.7952	[22, 22]
multiple_product_multiple_shipment_medium.json	opus-4-5	raw_only	2	1.0000	23.0609	[19, 22]
multiple_product_multiple_shipment_medium.json	opus-4-5	zero_shot	2	1.0000	7.9537	[24, 24]
multiple_product_multiple_shipment_medium.json	opus-4-6	db_only	2	1.0000	10.2524	[24, 24]
multiple_product_multiple_shipment_medium.json	opus-4-6	db_plus_raw	2	1.0000	9.9708	[24, 24]
multiple_product_multiple_shipment_medium.json	opus-4-6	raw_only	2	1.0000	8.2871	[22, 22]
multiple_product_multiple_shipment_medium.json	opus-4-6	zero_shot	2	1.0000	6.7422	[11, 11]
multiple_product_multiple_shipment_medium.json	sonnet-4-5	db_only	2	1.0000	9.5209	[24, 24]
multiple_product_multiple_shipment_medium.json	sonnet-4-5	db_plus_raw	2	1.0000	11.0477	[20, 24]
multiple_product_multiple_shipment_medium.json	sonnet-4-5	raw_only	2	1.0000	8.5159	[22, 22]
multiple_product_multiple_shipment_medium.json	sonnet-4-5	zero_shot	2	1.0000	7.8221	[24, 26]
multiple_product_multiple_shipment_medium.json	sonnet-4-6	db_only	2	1.0000	8.5846	[24, 24]
multiple_product_multiple_shipment_medium.json	sonnet-4-6	db_plus_raw	2	1.0000	10.5435	[24, 24]
multiple_product_multiple_shipment_medium.json	sonnet-4-6	raw_only	2	1.0000	7.7728	[22, 22]
multiple_product_multiple_shipment_medium.json	sonnet-4-6	zero_shot	2	1.0000	7.6468	[25, 22]
multiple_product_multiple_shipment_simple.json	opus-4-5	db_only	2	1.0000	9.5594	[16, 16]
multiple_product_multiple_shipment_simple.json	opus-4-5	db_plus_raw	2	1.0000	8.3421	[16, 16]
multiple_product_multiple_shipment_simple.json	opus-4-5	raw_only	2	1.0000	20.3248	[14, 14]
multiple_product_multiple_shipment_simple.json	opus-4-5	zero_shot	2	1.0000	5.9148	[12, 12]
multiple_product_multiple_shipment_simple.json	opus-4-6	db_only	2	1.0000	8.0840	[12, 27]
multiple_product_multiple_shipment_simple.json	opus-4-6	db_plus_raw	2	1.0000	18.7685	[16, 18]
multiple_product_multiple_shipment_simple.json	opus-4-6	raw_only	2	1.0000	7.0049	[18, 18]
multiple_product_multiple_shipment_simple.json	opus-4-6	zero_shot	2	1.0000	5.8224	[13, 13]
multiple_product_multiple_shipment_simple.json	sonnet-4-5	db_only	2	1.0000	7.2254	[10, 10]
multiple_product_multiple_shipment_simple.json	sonnet-4-5	db_plus_raw	2	1.0000	9.0899	[16, 16]
multiple_product_multiple_shipment_simple.json	sonnet-4-5	raw_only	2	1.0000	8.1257	[18, 16]
multiple_product_multiple_shipment_simple.json	sonnet-4-5	zero_shot	2	1.0000	6.1588	[13, 13]
multiple_product_multiple_shipment_simple.json	sonnet-4-6	db_only	2	1.0000	6.9738	[16, 16]
multiple_product_multiple_shipment_simple.json	sonnet-4-6	db_plus_raw	2	1.0000	8.4032	[16, 16]
multiple_product_multiple_shipment_simple.json	sonnet-4-6	raw_only	2	1.0000	7.0220	[17, 16]
multiple_product_multiple_shipment_simple.json	sonnet-4-6	zero_shot	2	1.0000	5.8578	[12, 12]
real_world_msgs_test_v1.json	opus-4-5	db_only	2	1.0000	9.1291	[11, 8]
real_world_msgs_test_v1.json	opus-4-5	db_plus_raw	2	1.0000	7.1828	[8, 8]
real_world_msgs_test_v1.json	opus-4-5	raw_only	2	1.0000	17.3862	[8, 8]
real_world_msgs_test_v1.json	opus-4-5	zero_shot	2	1.0000	5.0774	[8, 8]
real_world_msgs_test_v1.json	opus-4-6	db_only	2	1.0000	6.8878	[8, 8]
real_world_msgs_test_v1.json	opus-4-6	db_plus_raw	2	1.0000	29.3460	[8, 8]
real_world_msgs_test_v1.json	opus-4-6	raw_only	2	1.0000	5.9718	[8, 8]
real_world_msgs_test_v1.json	opus-4-6	zero_shot	2	1.0000	5.2084	[10, 10]
real_world_msgs_test_v1.json	sonnet-4-5	db_only	2	1.0000	8.6640	[11, 11]
real_world_msgs_test_v1.json	sonnet-4-5	db_plus_raw	2	1.0000	8.7967	[11, 11]
real_world_msgs_test_v1.json	sonnet-4-5	raw_only	2	1.0000	7.3437	[12, 12]
real_world_msgs_test_v1.json	sonnet-4-5	zero_shot	2	1.0000	6.2823	[11, 11]
real_world_msgs_test_v1.json	sonnet-4-6	db_only	2	1.0000	6.9921	[8, 8]
real_world_msgs_test_v1.json	sonnet-4-6	db_plus_raw	2	1.0000	6.7997	[8, 8]
real_world_msgs_test_v1.json	sonnet-4-6	raw_only	2	1.0000	5.3427	[9, 9]
real_world_msgs_test_v1.json	sonnet-4-6	zero_shot	2	1.0000	5.1170	[10, 10]
real_world_msgs_test_v2.json	opus-4-5	db_only	2	1.0000	8.6968	[13, 13]
real_world_msgs_test_v2.json	opus-4-5	db_plus_raw	2	1.0000	7.6117	[13, 13]
real_world_msgs_test_v2.json	opus-4-5	raw_only	2	1.0000	29.3183	[13, 13]
real_world_msgs_test_v2.json	opus-4-5	zero_shot	2	1.0000	7.2797	[13, 13]
real_world_msgs_test_v2.json	opus-4-6	db_only	2	1.0000	8.0128	[13, 13]
real_world_msgs_test_v2.json	opus-4-6	db_plus_raw	2	1.0000	26.2668	[13, 13]
real_world_msgs_test_v2.json	opus-4-6	raw_only	2	1.0000	7.4627	[13, 13]
real_world_msgs_test_v2.json	opus-4-6	zero_shot	2	1.0000	6.4021	[13, 14]
real_world_msgs_test_v2.json	sonnet-4-5	db_only	2	1.0000	8.3810	[13, 13]
real_world_msgs_test_v2.json	sonnet-4-5	db_plus_raw	2	1.0000	9.2966	[13, 13]
real_world_msgs_test_v2.json	sonnet-4-5	raw_only	2	1.0000	7.5710	[17, 17]
real_world_msgs_test_v2.json	sonnet-4-5	zero_shot	2	1.0000	6.7523	[13, 13]
real_world_msgs_test_v2.json	sonnet-4-6	db_only	2	1.0000	6.9755	[13, 13]
real_world_msgs_test_v2.json	sonnet-4-6	db_plus_raw	2	1.0000	7.6709	[13, 13]
real_world_msgs_test_v2.json	sonnet-4-6	raw_only	2	1.0000	7.2782	[16, 16]
real_world_msgs_test_v2.json	sonnet-4-6	zero_shot	2	1.0000	8.1861	[14, 14]
real_world_msgs_test_v3.json	opus-4-5	db_only	2	1.0000	8.9480	[13, 13]
real_world_msgs_test_v3.json	opus-4-5	db_plus_raw	2	1.0000	7.3866	[13, 13]
real_world_msgs_test_v3.json	opus-4-5	raw_only	2	1.0000	24.0110	[13, 13]
real_world_msgs_test_v3.json	opus-4-5	zero_shot	2	1.0000	5.9911	[9, 9]
real_world_msgs_test_v3.json	opus-4-6	db_only	2	1.0000	8.1748	[13, 13]
real_world_msgs_test_v3.json	opus-4-6	db_plus_raw	2	1.0000	29.0950	[13, 13]
real_world_msgs_test_v3.json	opus-4-6	raw_only	2	1.0000	7.0847	[13, 13]
real_world_msgs_test_v3.json	opus-4-6	zero_shot	2	1.0000	6.0732	[9, 9]
real_world_msgs_test_v3.json	sonnet-4-5	db_only	2	1.0000	8.4088	[13, 13]
real_world_msgs_test_v3.json	sonnet-4-5	db_plus_raw	2	1.0000	8.3666	[17, 17]
real_world_msgs_test_v3.json	sonnet-4-5	raw_only	2	1.0000	7.5119	[17, 17]
real_world_msgs_test_v3.json	sonnet-4-5	zero_shot	2	1.0000	7.1396	[9, 9]
real_world_msgs_test_v3.json	sonnet-4-6	db_only	2	1.0000	6.7129	[13, 13]
real_world_msgs_test_v3.json	sonnet-4-6	db_plus_raw	2	1.0000	8.7110	[13, 13]
real_world_msgs_test_v3.json	sonnet-4-6	raw_only	2	1.0000	6.9232	[13, 13]
real_world_msgs_test_v3.json	sonnet-4-6	zero_shot	2	1.0000	5.7824	[11, 11]
single_product_multiple_shipment_complex.json	opus-4-5	db_only	2	1.0000	10.6272	[22, 22]
single_product_multiple_shipment_complex.json	opus-4-5	db_plus_raw	2	1.0000	9.1463	[22, 22]
single_product_multiple_shipment_complex.json	opus-4-5	raw_only	2	1.0000	27.7767	[19, 19]
single_product_multiple_shipment_complex.json	opus-4-5	zero_shot	2	1.0000	7.0074	[11, 11]
single_product_multiple_shipment_complex.json	opus-4-6	db_only	2	1.0000	8.8402	[12, 12]
single_product_multiple_shipment_complex.json	opus-4-6	db_plus_raw	2	1.0000	28.5284	[22, 22]
single_product_multiple_shipment_complex.json	opus-4-6	raw_only	2	1.0000	8.8787	[25, 25]
single_product_multiple_shipment_complex.json	opus-4-6	zero_shot	2	1.0000	7.1064	[11, 11]
single_product_multiple_shipment_complex.json	sonnet-4-5	db_only	2	1.0000	10.6799	[22, 22]
single_product_multiple_shipment_complex.json	sonnet-4-5	db_plus_raw	2	1.0000	10.7180	[21, 21]
single_product_multiple_shipment_complex.json	sonnet-4-5	raw_only	2	1.0000	10.0674	[19, 19]
single_product_multiple_shipment_complex.json	sonnet-4-5	zero_shot	2	1.0000	7.6867	[11, 11]
single_product_multiple_shipment_complex.json	sonnet-4-6	db_only	2	1.0000	10.7506	[22, 22]
single_product_multiple_shipment_complex.json	sonnet-4-6	db_plus_raw	2	1.0000	11.4762	[22, 22]
single_product_multiple_shipment_complex.json	sonnet-4-6	raw_only	2	1.0000	11.2134	[22, 22]
single_product_multiple_shipment_complex.json	sonnet-4-6	zero_shot	2	1.0000	8.2816	[11, 11]
single_product_multiple_shipment_medium.json	opus-4-5	db_only	2	1.0000	9.2031	[17, 17]
single_product_multiple_shipment_medium.json	opus-4-5	db_plus_raw	2	1.0000	8.3981	[17, 17]
single_product_multiple_shipment_medium.json	opus-4-5	raw_only	2	1.0000	20.5861	[17, 17]
single_product_multiple_shipment_medium.json	opus-4-5	zero_shot	2	1.0000	6.0220	[12, 12]
single_product_multiple_shipment_medium.json	opus-4-6	db_only	2	1.0000	8.7698	[19, 19]
single_product_multiple_shipment_medium.json	opus-4-6	db_plus_raw	2	1.0000	27.0612	[17, 19]
single_product_multiple_shipment_medium.json	opus-4-6	raw_only	2	1.0000	7.5534	[17, 17]
single_product_multiple_shipment_medium.json	opus-4-6	zero_shot	2	1.0000	6.0306	[12, 12]
single_product_multiple_shipment_medium.json	sonnet-4-5	db_only	2	1.0000	9.0231	[17, 17]
single_product_multiple_shipment_medium.json	sonnet-4-5	db_plus_raw	2	1.0000	8.1665	[17, 17]
single_product_multiple_shipment_medium.json	sonnet-4-5	raw_only	2	1.0000	7.7417	[17, 17]
single_product_multiple_shipment_medium.json	sonnet-4-5	zero_shot	2	1.0000	6.4371	[12, 12]
single_product_multiple_shipment_medium.json	sonnet-4-6	db_only	2	1.0000	7.6027	[17, 17]
single_product_multiple_shipment_medium.json	sonnet-4-6	db_plus_raw	2	1.0000	13.3270	[17, 17]
single_product_multiple_shipment_medium.json	sonnet-4-6	raw_only	2	1.0000	7.0010	[17, 17]
single_product_multiple_shipment_medium.json	sonnet-4-6	zero_shot	2	1.0000	6.9483	[19, 19]
single_product_multiple_shipment_simple.json	opus-4-5	db_only	2	1.0000	7.2896	[14, 14]
single_product_multiple_shipment_simple.json	opus-4-5	db_plus_raw	2	1.0000	20.9075	[13, 12]
single_product_multiple_shipment_simple.json	opus-4-5	raw_only	2	1.0000	19.7298	[12, 12]
single_product_multiple_shipment_simple.json	opus-4-5	zero_shot	2	1.0000	6.0084	[10, 9]
single_product_multiple_shipment_simple.json	opus-4-6	db_only	2	1.0000	8.4985	[23, 23]
single_product_multiple_shipment_simple.json	opus-4-6	db_plus_raw	2	1.0000	23.0870	[14, 14]
single_product_multiple_shipment_simple.json	opus-4-6	raw_only	2	1.0000	6.6646	[14, 14]
single_product_multiple_shipment_simple.json	opus-4-6	zero_shot	2	1.0000	5.6699	[10, 10]
single_product_multiple_shipment_simple.json	sonnet-4-5	db_only	2	1.0000	8.9361	[14, 14]
single_product_multiple_shipment_simple.json	sonnet-4-5	db_plus_raw	2	1.0000	8.1629	[14, 14]
single_product_multiple_shipment_simple.json	sonnet-4-5	raw_only	2	1.0000	7.4617	[13, 13]
single_product_multiple_shipment_simple.json	sonnet-4-5	zero_shot	2	1.0000	6.6643	[10, 10]
single_product_multiple_shipment_simple.json	sonnet-4-6	db_only	2	1.0000	6.9893	[14, 14]
single_product_multiple_shipment_simple.json	sonnet-4-6	db_plus_raw	2	1.0000	9.0193	[14, 14]
single_product_multiple_shipment_simple.json	sonnet-4-6	raw_only	2	1.0000	6.5538	[14, 14]
single_product_multiple_shipment_simple.json	sonnet-4-6	zero_shot	2	1.0000	5.9300	[9, 9]
single_product_single_shipment_complex.json	opus-4-5	db_only	2	1.0000	6.6842	[4, 4]
single_product_single_shipment_complex.json	opus-4-5	db_plus_raw	2	1.0000	21.8049	[4, 4]
single_product_single_shipment_complex.json	opus-4-5	raw_only	2	1.0000	21.0643	[3, 3]
single_product_single_shipment_complex.json	opus-4-5	zero_shot	2	1.0000	4.8121	[3, 6]
single_product_single_shipment_complex.json	opus-4-6	db_only	2	1.0000	7.2630	[5, 5]
single_product_single_shipment_complex.json	opus-4-6	db_plus_raw	2	1.0000	18.6269	[4, 4]
single_product_single_shipment_complex.json	opus-4-6	raw_only	2	1.0000	5.1209	[4, 4]
single_product_single_shipment_complex.json	opus-4-6	zero_shot	2	1.0000	4.7135	[6, 6]
single_product_single_shipment_complex.json	sonnet-4-5	db_only	2	1.0000	7.8461	[5, 5]
single_product_single_shipment_complex.json	sonnet-4-5	db_plus_raw	2	1.0000	6.6891	[5, 5]
single_product_single_shipment_complex.json	sonnet-4-5	raw_only	2	1.0000	6.0804	[5, 5]
single_product_single_shipment_complex.json	sonnet-4-5	zero_shot	2	1.0000	5.3951	[5, 5]
single_product_single_shipment_complex.json	sonnet-4-6	db_only	2	1.0000	6.6700	[5, 5]
single_product_single_shipment_complex.json	sonnet-4-6	db_plus_raw	2	1.0000	7.2455	[5, 5]
single_product_single_shipment_complex.json	sonnet-4-6	raw_only	2	1.0000	5.8954	[5, 5]
single_product_single_shipment_complex.json	sonnet-4-6	zero_shot	2	1.0000	7.9609	[3, 5]
single_product_single_shipment_medium.json	opus-4-5	db_only	2	1.0000	6.8616	[5, 5]
single_product_single_shipment_medium.json	opus-4-5	db_plus_raw	2	1.0000	18.8684	[4, 5]
single_product_single_shipment_medium.json	opus-4-5	raw_only	2	1.0000	27.9534	[4, 4]
single_product_single_shipment_medium.json	opus-4-5	zero_shot	2	1.0000	5.2085	[4, 4]
single_product_single_shipment_medium.json	opus-4-6	db_only	2	1.0000	6.8168	[6, 7]
single_product_single_shipment_medium.json	opus-4-6	db_plus_raw	2	1.0000	7.2542	[5, 5]
single_product_single_shipment_medium.json	opus-4-6	raw_only	2	1.0000	5.4114	[5, 5]
single_product_single_shipment_medium.json	opus-4-6	zero_shot	2	1.0000	4.9310	[5, 5]
single_product_single_shipment_medium.json	sonnet-4-5	db_only	2	1.0000	6.6914	[5, 5]
single_product_single_shipment_medium.json	sonnet-4-5	db_plus_raw	2	1.0000	7.3883	[5, 5]
single_product_single_shipment_medium.json	sonnet-4-5	raw_only	2	1.0000	5.8366	[5, 5]
single_product_single_shipment_medium.json	sonnet-4-5	zero_shot	2	1.0000	5.0717	[6, 6]
single_product_single_shipment_medium.json	sonnet-4-6	db_only	2	1.0000	7.6573	[6, 5]
single_product_single_shipment_medium.json	sonnet-4-6	db_plus_raw	2	1.0000	7.0603	[5, 5]
single_product_single_shipment_medium.json	sonnet-4-6	raw_only	2	1.0000	5.0431	[5, 5]
single_product_single_shipment_medium.json	sonnet-4-6	zero_shot	2	1.0000	4.5857	[5, 5]
single_product_single_shipment_simple.json	opus-4-5	db_only	2	1.0000	7.1831	[7, 7]
single_product_single_shipment_simple.json	opus-4-5	db_plus_raw	2	1.0000	22.1966	[7, 7]
single_product_single_shipment_simple.json	opus-4-5	raw_only	2	1.0000	4.8875	[7, 7]
single_product_single_shipment_simple.json	opus-4-5	zero_shot	2	1.0000	5.2086	[7, 7]
single_product_single_shipment_simple.json	opus-4-6	db_only	2	1.0000	6.4802	[7, 7]
single_product_single_shipment_simple.json	opus-4-6	db_plus_raw	2	1.0000	6.3922	[7, 7]
single_product_single_shipment_simple.json	opus-4-6	raw_only	2	1.0000	5.0399	[7, 7]
single_product_single_shipment_simple.json	opus-4-6	zero_shot	2	1.0000	4.8259	[7, 7]
single_product_single_shipment_simple.json	sonnet-4-5	db_only	2	1.0000	5.9010	[7, 7]
single_product_single_shipment_simple.json	sonnet-4-5	db_plus_raw	2	1.0000	6.3576	[7, 7]
single_product_single_shipment_simple.json	sonnet-4-5	raw_only	2	1.0000	4.8972	[7, 7]
single_product_single_shipment_simple.json	sonnet-4-5	zero_shot	2	1.0000	4.7984	[8, 8]
single_product_single_shipment_simple.json	sonnet-4-6	db_only	2	1.0000	6.9497	[17, 17]
single_product_single_shipment_simple.json	sonnet-4-6	db_plus_raw	2	1.0000	6.2664	[7, 7]
single_product_single_shipment_simple.json	sonnet-4-6	raw_only	2	1.0000	5.0337	[7, 7]
single_product_single_shipment_simple.json	sonnet-4-6	zero_shot	2	1.0000	4.6042	[7, 7]

Top Mismatches (up to 100 runs)

Chat	Model	Strategy	Run	Mismatch count	Sample mismatches
multiple_product_multiple_shipment_complex.json	opus-4-6	db_only	1	47	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	opus-4-6	db_only	2	47	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	opus-4-5	db_plus_raw	2	43	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	sonnet-4-5	raw_only	2	41	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	sonnet-4-5	db_plus_raw	1	41	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	sonnet-4-5	db_plus_raw	2	41	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	opus-4-5	zero_shot	2	41	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	opus-4-5	zero_shot	1	41	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	opus-4-5	db_plus_raw	1	41	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	opus-4-5	db_only	1	41	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	opus-4-5	db_only	2	41	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	sonnet-4-5	db_only	1	40	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	opus-4-6	raw_only	1	40	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	opus-4-6	raw_only	2	40	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	opus-4-5	raw_only	2	40	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	opus-4-5	raw_only	1	40	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	sonnet-4-6	db_plus_raw	1	38	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	sonnet-4-6	db_only	1	38	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	sonnet-4-6	db_only	2	38	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	sonnet-4-5	raw_only	1	38	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	sonnet-4-5	db_only	2	38	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	opus-4-6	db_plus_raw	2	38	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	opus-4-6	db_plus_raw	1	38	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	sonnet-4-6	raw_only	1	37	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	sonnet-4-6	raw_only	2	37	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" } ]
multiple_product_multiple_shipment_complex.json	sonnet-4-6	db_plus_raw	2	33	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" } ]
multiple_product_multiple_shipment_complex.json	sonnet-4-6	zero_shot	1	31	[ { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from first delivery", "Net 30", "2026-03-30" ], "actual": "" } ]
multiple_product_multiple_shipment_complex.json	sonnet-4-6	zero_shot	2	31	[ { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].total", "expected": 276.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from first delivery", "Net 30", "2026-03-30" ], "actual": "" } ]
multiple_product_multiple_shipment_complex.json	sonnet-4-5	zero_shot	2	30	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.0, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" } ]
multiple_product_multiple_shipment_simple.json	opus-4-6	db_only	2	27	[ { "path": "data[0].items[0].description", "expected": "KNM Coffee", "actual": "lecithin fat powder" }, { "path": "data[0].items[0].quantity", "expected": 10.0, "actual": 8.0 }, { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "MT" }, { "path": "data[0].items[0].unit_price", "expected": 25.0, "actual": 12.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/KG" } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-5	zero_shot	2	26	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
single_product_multiple_shipment_complex.json	opus-4-6	raw_only	1	25	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
single_product_multiple_shipment_complex.json	opus-4-6	raw_only	2	25	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-6	zero_shot	1	25	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 300.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAGS" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 3600.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-6	db_plus_raw	1	24	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-6	db_plus_raw	2	24	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-6	db_only	2	24	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-6	db_only	1	24	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-5	zero_shot	1	24	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-5	db_plus_raw	2	24	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-5	db_only	1	24	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-5	db_only	2	24	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	opus-4-6	db_plus_raw	1	24	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	opus-4-6	db_plus_raw	2	24	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	opus-4-6	db_only	1	24	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	opus-4-6	db_only	2	24	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	opus-4-5	zero_shot	1	24	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": null }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	opus-4-5	zero_shot	2	24	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	opus-4-5	db_only	2	24	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	opus-4-5	db_only	1	24	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
single_product_multiple_shipment_simple.json	opus-4-6	db_only	1	23	[ { "path": "data[0].items[0].description", "expected": "KNM Coffee", "actual": "lecithin fat powder" }, { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "MT" }, { "path": "data[0].items[0].unit_price", "expected": 25.0, "actual": 12.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/KG" }, { "path": "data[0].items[0].total", "expected": 375.0, "actual": 96000.0 } ]
single_product_multiple_shipment_simple.json	opus-4-6	db_only	2	23	[ { "path": "data[0].items[0].description", "expected": "KNM Coffee", "actual": "lecithin fat powder" }, { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "MT" }, { "path": "data[0].items[0].unit_price", "expected": 25.0, "actual": 12.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/KG" }, { "path": "data[0].items[0].total", "expected": 375.0, "actual": 96000.0 } ]
single_product_multiple_shipment_complex.json	sonnet-4-6	raw_only	1	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
single_product_multiple_shipment_complex.json	sonnet-4-6	raw_only	2	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
single_product_multiple_shipment_complex.json	sonnet-4-6	db_plus_raw	1	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
single_product_multiple_shipment_complex.json	sonnet-4-6	db_plus_raw	2	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
single_product_multiple_shipment_complex.json	sonnet-4-6	db_only	1	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
single_product_multiple_shipment_complex.json	sonnet-4-6	db_only	2	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
single_product_multiple_shipment_complex.json	sonnet-4-5	db_only	2	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
single_product_multiple_shipment_complex.json	sonnet-4-5	db_only	1	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
single_product_multiple_shipment_complex.json	opus-4-6	db_plus_raw	2	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
single_product_multiple_shipment_complex.json	opus-4-6	db_plus_raw	1	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
single_product_multiple_shipment_complex.json	opus-4-5	db_plus_raw	2	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
single_product_multiple_shipment_complex.json	opus-4-5	db_plus_raw	1	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
single_product_multiple_shipment_complex.json	opus-4-5	db_only	1	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
single_product_multiple_shipment_complex.json	opus-4-5	db_only	2	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-6	zero_shot	2	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 300.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD" }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-6	raw_only	1	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-6	raw_only	2	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-5	raw_only	1	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-5	raw_only	2	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	opus-4-6	raw_only	1	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	opus-4-6	raw_only	2	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	opus-4-5	raw_only	2	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	opus-4-5	db_plus_raw	1	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
multiple_product_multiple_shipment_medium.json	opus-4-5	db_plus_raw	2	22	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].total", "expected": 285.0, "actual": 300.0 }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" } ]
single_product_multiple_shipment_complex.json	sonnet-4-5	db_plus_raw	1	21	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
single_product_multiple_shipment_complex.json	sonnet-4-5	db_plus_raw	2	21	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].payment_date", "expected": [ "Net 30 from last delivery", "Net 30", "2026-04-08" ], "actual": "" } ]
multiple_product_multiple_shipment_medium.json	sonnet-4-5	db_plus_raw	1	20	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" } ]
single_product_multiple_shipment_medium.json	sonnet-4-6	zero_shot	2	19	[ { "path": "data[0].items[0].description", "expected": "KNM Coffee", "actual": "KNM Coffee (medium roast)" }, { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "CIF Singapore", "actual": "CIF" }, { "path": "data[0].do_date", "expected": "2026-04-30", "actual": "2026-05-31" } ]
single_product_multiple_shipment_medium.json	sonnet-4-6	zero_shot	1	19	[ { "path": "data[0].items[0].description", "expected": "KNM Coffee", "actual": "KNM Coffee (medium roast)" }, { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "CIF Singapore", "actual": "CIF" }, { "path": "data[0].do_date", "expected": "2026-04-30", "actual": "2026-05-31" } ]
single_product_multiple_shipment_medium.json	opus-4-6	db_plus_raw	2	19	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "CIF Singapore", "actual": "CIF" }, { "path": "data[0].do_date", "expected": "2026-04-30", "actual": "2026-05-31" }, { "path": "data[0].po_ref_no", "expected": "PO-2025-11-180", "actual": "" } ]
single_product_multiple_shipment_medium.json	opus-4-6	db_only	1	19	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "CIF Singapore", "actual": "CIF" }, { "path": "data[0].do_date", "expected": "2026-04-30", "actual": "2026-05-31" }, { "path": "data[0].po_ref_no", "expected": "PO-2025-11-180", "actual": "" } ]
single_product_multiple_shipment_medium.json	opus-4-6	db_only	2	19	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "CIF Singapore", "actual": "CIF" }, { "path": "data[0].do_date", "expected": "2026-04-30", "actual": "2026-05-31" }, { "path": "data[0].po_ref_no", "expected": "PO-2025-11-180", "actual": "" } ]
single_product_multiple_shipment_complex.json	sonnet-4-5	raw_only	1	19	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].billing_address", "expected": [], "actual": "Leonardo da Vinci, da@vinci.com" } ]
single_product_multiple_shipment_complex.json	sonnet-4-5	raw_only	2	19	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].billing_address", "expected": [], "actual": "Leonardo da Vinci, da@vinci.com" } ]
single_product_multiple_shipment_complex.json	opus-4-5	raw_only	2	19	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].billing_address", "expected": [], "actual": "" } ]
single_product_multiple_shipment_complex.json	opus-4-5	raw_only	1	19	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "FOB Singapore", "actual": "FOB" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" }, { "path": "data[0].billing_address", "expected": [], "actual": "" } ]
multiple_product_multiple_shipment_medium.json	opus-4-5	raw_only	1	19	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].unit_price", "expected": 23.75, "actual": 25.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "EXW Singapore", "actual": "EXW" }, { "path": "data[0].vendor_name", "expected": "flamingos", "actual": "Van Beethoven" } ]
multiple_product_multiple_shipment_simple.json	sonnet-4-5	raw_only	1	18	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "CIF Singapore", "actual": "CIF" }, { "path": "data[0].do_date", "expected": "2026-04-30", "actual": "2026-05-31" }, { "path": "data[0].po_ref_no", "expected": "PO-2024-11-200", "actual": "PO-2025-11-200" } ]
multiple_product_multiple_shipment_simple.json	opus-4-6	raw_only	1	18	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "CIF Singapore", "actual": "CIF" }, { "path": "data[0].do_date", "expected": "2026-04-30", "actual": "2026-05-31" }, { "path": "data[0].po_ref_no", "expected": "PO-2024-11-200", "actual": "PO-2025-11-200" } ]
multiple_product_multiple_shipment_simple.json	opus-4-6	raw_only	2	18	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "CIF Singapore", "actual": "CIF" }, { "path": "data[0].do_date", "expected": "2026-04-30", "actual": "2026-05-31" }, { "path": "data[0].po_ref_no", "expected": "PO-2024-11-200", "actual": "PO-2025-11-200" } ]
multiple_product_multiple_shipment_simple.json	opus-4-6	db_plus_raw	2	18	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "CIF Singapore", "actual": "CIF" }, { "path": "data[0].do_date", "expected": "2026-04-30", "actual": "2026-05-31" }, { "path": "data[0].po_ref_no", "expected": "PO-2024-11-200", "actual": "PO-2025-11-200" } ]
single_product_single_shipment_simple.json	sonnet-4-6	db_only	2	17	[ { "path": "data[0].items[0].description", "expected": "KNM Coffee", "actual": "BP102" }, { "path": "data[0].items[0].quantity", "expected": 5.0, "actual": 23.0 }, { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "MT" }, { "path": "data[0].items[0].unit_price", "expected": 25.0, "actual": 1410.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/MT" } ]
single_product_single_shipment_simple.json	sonnet-4-6	db_only	1	17	[ { "path": "data[0].items[0].description", "expected": "KNM Coffee", "actual": "BP102" }, { "path": "data[0].items[0].quantity", "expected": 5.0, "actual": 23.0 }, { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "MT" }, { "path": "data[0].items[0].unit_price", "expected": 25.0, "actual": 1410.0 }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/MT" } ]
single_product_multiple_shipment_medium.json	sonnet-4-6	raw_only	1	17	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "CIF Singapore", "actual": "CIF" }, { "path": "data[0].do_date", "expected": "2026-04-30", "actual": "2026-05-31" }, { "path": "data[0].po_ref_no", "expected": "PO-2025-11-180", "actual": "" } ]
single_product_multiple_shipment_medium.json	sonnet-4-6	raw_only	2	17	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "CIF Singapore", "actual": "CIF" }, { "path": "data[0].do_date", "expected": "2026-04-30", "actual": "2026-05-31" }, { "path": "data[0].po_ref_no", "expected": "PO-2025-11-180", "actual": "" } ]
single_product_multiple_shipment_medium.json	sonnet-4-6	db_plus_raw	2	17	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "CIF Singapore", "actual": "CIF" }, { "path": "data[0].do_date", "expected": "2026-04-30", "actual": "2026-05-31" }, { "path": "data[0].po_ref_no", "expected": "PO-2025-11-180", "actual": "" } ]
single_product_multiple_shipment_medium.json	sonnet-4-6	db_plus_raw	1	17	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "CIF Singapore", "actual": "CIF" }, { "path": "data[0].do_date", "expected": "2026-04-30", "actual": "2026-05-31" }, { "path": "data[0].po_ref_no", "expected": "PO-2025-11-180", "actual": "" } ]
single_product_multiple_shipment_medium.json	sonnet-4-6	db_only	1	17	[ { "path": "data[0].items[0].quantity_unit", "expected": "bags", "actual": "BAGS" }, { "path": "data[0].items[0].pricing_unit", "expected": "USD/bag", "actual": "USD/BAG" }, { "path": "data[0].items[0].ship_term", "expected": "CIF Singapore", "actual": "CIF" }, { "path": "data[0].do_date", "expected": "2026-04-30", "actual": "2026-05-31" }, { "path": "data[0].po_ref_no", "expected": "PO-2025-11-180", "actual": "" } ]