| multiple_product_multiple_shipment_complex.json | opus-4-6 | db_only | 1 | 47 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.0,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 276.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
}
] |
| multiple_product_multiple_shipment_complex.json | opus-4-5 | db_plus_raw | 1 | 43 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.0,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 276.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
}
] |
| multiple_product_multiple_shipment_complex.json | opus-4-5 | zero_shot | 1 | 41 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.0,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 276.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
}
] |
| multiple_product_multiple_shipment_complex.json | opus-4-5 | db_only | 1 | 41 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.0,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 276.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
}
] |
| multiple_product_multiple_shipment_complex.json | sonnet-4-5 | db_only | 1 | 40 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.0,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 276.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
}
] |
| multiple_product_multiple_shipment_complex.json | opus-4-6 | raw_only | 1 | 40 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.0,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 276.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
}
] |
| multiple_product_multiple_shipment_complex.json | opus-4-5 | raw_only | 1 | 40 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.0,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 276.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
}
] |
| multiple_product_multiple_shipment_complex.json | sonnet-4-6 | db_only | 1 | 38 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.0,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 276.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
}
] |
| multiple_product_multiple_shipment_complex.json | sonnet-4-5 | raw_only | 1 | 38 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.0,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 276.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
}
] |
| multiple_product_multiple_shipment_complex.json | sonnet-4-5 | db_plus_raw | 1 | 38 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.0,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 276.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
}
] |
| multiple_product_multiple_shipment_complex.json | opus-4-6 | db_plus_raw | 1 | 38 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.0,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 276.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
}
] |
| multiple_product_multiple_shipment_complex.json | sonnet-4-6 | raw_only | 1 | 37 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.0,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 276.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
}
] |
| multiple_product_multiple_shipment_complex.json | sonnet-4-6 | zero_shot | 1 | 34 | [
{
"path": "data[0].items[0].unit_price",
"expected": 23.0,
"actual": 300.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD"
},
{
"path": "data[0].items[0].total",
"expected": 276.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
},
{
"path": "data[0].vendor_name",
"expected": "flamingos",
"actual": "Van Beethoven"
}
] |
| multiple_product_multiple_shipment_complex.json | sonnet-4-6 | db_plus_raw | 1 | 33 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.0,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
},
{
"path": "data[0].vendor_name",
"expected": "flamingos",
"actual": "Van Beethoven"
}
] |
| multiple_product_multiple_shipment_medium.json | sonnet-4-5 | zero_shot | 1 | 26 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": ""
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.75,
"actual": 300.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": ""
},
{
"path": "data[0].items[0].total",
"expected": 285.0,
"actual": 3600.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "EXW Singapore",
"actual": "EXW"
}
] |
| single_product_multiple_shipment_complex.json | opus-4-6 | raw_only | 1 | 25 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
},
{
"path": "data[0].vendor_name",
"expected": "flamingos",
"actual": "Van Beethoven"
},
{
"path": "data[0].payment_date",
"expected": [
"Net 30 from last delivery",
"Net 30",
"2026-04-08"
],
"actual": ""
}
] |
| multiple_product_multiple_shipment_medium.json | sonnet-4-6 | zero_shot | 1 | 25 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.75,
"actual": 300.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAGS"
},
{
"path": "data[0].items[0].total",
"expected": 285.0,
"actual": 3600.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "EXW Singapore",
"actual": "EXW"
}
] |
| multiple_product_multiple_shipment_medium.json | sonnet-4-6 | db_plus_raw | 1 | 24 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.75,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 285.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "EXW Singapore",
"actual": "EXW"
}
] |
| multiple_product_multiple_shipment_medium.json | sonnet-4-6 | db_only | 1 | 24 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.75,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 285.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "EXW Singapore",
"actual": "EXW"
}
] |
| multiple_product_multiple_shipment_medium.json | sonnet-4-5 | db_plus_raw | 1 | 24 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.75,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 285.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "EXW Singapore",
"actual": "EXW"
}
] |
| multiple_product_multiple_shipment_medium.json | sonnet-4-5 | db_only | 1 | 24 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.75,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 285.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "EXW Singapore",
"actual": "EXW"
}
] |
| multiple_product_multiple_shipment_medium.json | opus-4-6 | db_plus_raw | 1 | 24 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.75,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 285.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "EXW Singapore",
"actual": "EXW"
}
] |
| multiple_product_multiple_shipment_medium.json | opus-4-6 | db_only | 1 | 24 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.75,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 285.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "EXW Singapore",
"actual": "EXW"
}
] |
| multiple_product_multiple_shipment_medium.json | opus-4-5 | zero_shot | 1 | 24 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.75,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 285.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "EXW Singapore",
"actual": "EXW"
}
] |
| multiple_product_multiple_shipment_medium.json | opus-4-5 | db_only | 1 | 24 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.75,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 285.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "EXW Singapore",
"actual": "EXW"
}
] |
| single_product_multiple_shipment_simple.json | opus-4-6 | db_only | 1 | 23 | [
{
"path": "data[0].items[0].description",
"expected": "KNM Coffee",
"actual": "lecithin fat powder"
},
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "MT"
},
{
"path": "data[0].items[0].unit_price",
"expected": 25.0,
"actual": 12.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/KG"
},
{
"path": "data[0].items[0].total",
"expected": 375.0,
"actual": 96000.0
}
] |
| single_product_multiple_shipment_complex.json | sonnet-4-6 | raw_only | 1 | 22 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
},
{
"path": "data[0].vendor_name",
"expected": "flamingos",
"actual": "Van Beethoven"
},
{
"path": "data[0].payment_date",
"expected": [
"Net 30 from last delivery",
"Net 30",
"2026-04-08"
],
"actual": ""
}
] |
| single_product_multiple_shipment_complex.json | sonnet-4-6 | db_plus_raw | 1 | 22 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
},
{
"path": "data[0].vendor_name",
"expected": "flamingos",
"actual": "Van Beethoven"
},
{
"path": "data[0].payment_date",
"expected": [
"Net 30 from last delivery",
"Net 30",
"2026-04-08"
],
"actual": ""
}
] |
| single_product_multiple_shipment_complex.json | sonnet-4-6 | db_only | 1 | 22 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
},
{
"path": "data[0].vendor_name",
"expected": "flamingos",
"actual": "Van Beethoven"
},
{
"path": "data[0].payment_date",
"expected": [
"Net 30 from last delivery",
"Net 30",
"2026-04-08"
],
"actual": ""
}
] |
| single_product_multiple_shipment_complex.json | sonnet-4-5 | db_only | 1 | 22 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
},
{
"path": "data[0].vendor_name",
"expected": "flamingos",
"actual": "Van Beethoven"
},
{
"path": "data[0].payment_date",
"expected": [
"Net 30 from last delivery",
"Net 30",
"2026-04-08"
],
"actual": ""
}
] |
| single_product_multiple_shipment_complex.json | opus-4-6 | db_plus_raw | 1 | 22 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
},
{
"path": "data[0].vendor_name",
"expected": "flamingos",
"actual": "Van Beethoven"
},
{
"path": "data[0].payment_date",
"expected": [
"Net 30 from last delivery",
"Net 30",
"2026-04-08"
],
"actual": ""
}
] |
| single_product_multiple_shipment_complex.json | opus-4-5 | db_only | 1 | 22 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
},
{
"path": "data[0].vendor_name",
"expected": "flamingos",
"actual": "Van Beethoven"
},
{
"path": "data[0].payment_date",
"expected": [
"Net 30 from last delivery",
"Net 30",
"2026-04-08"
],
"actual": ""
}
] |
| multiple_product_multiple_shipment_medium.json | sonnet-4-5 | raw_only | 1 | 22 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.75,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 285.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "EXW Singapore",
"actual": "EXW"
}
] |
| multiple_product_multiple_shipment_medium.json | opus-4-6 | raw_only | 1 | 22 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.75,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 285.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "EXW Singapore",
"actual": "EXW"
}
] |
| multiple_product_multiple_shipment_medium.json | opus-4-5 | db_plus_raw | 1 | 22 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.75,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 285.0,
"actual": 300.0
},
{
"path": "data[0].items[0].ship_term",
"expected": "EXW Singapore",
"actual": "EXW"
}
] |
| single_product_multiple_shipment_complex.json | sonnet-4-5 | db_plus_raw | 1 | 21 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
},
{
"path": "data[0].vendor_name",
"expected": "flamingos",
"actual": "Van Beethoven"
},
{
"path": "data[0].payment_date",
"expected": [
"Net 30 from last delivery",
"Net 30",
"2026-04-08"
],
"actual": ""
}
] |
| single_product_multiple_shipment_medium.json | sonnet-4-6 | zero_shot | 1 | 19 | [
{
"path": "data[0].items[0].description",
"expected": "KNM Coffee",
"actual": "KNM Coffee (Medium Roast)"
},
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
}
] |
| single_product_multiple_shipment_medium.json | sonnet-4-5 | db_plus_raw | 1 | 19 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2025-11-180",
"actual": ""
}
] |
| single_product_multiple_shipment_medium.json | opus-4-6 | db_only | 1 | 19 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2025-11-180",
"actual": ""
}
] |
| single_product_multiple_shipment_complex.json | sonnet-4-5 | raw_only | 1 | 19 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
},
{
"path": "data[0].vendor_name",
"expected": "flamingos",
"actual": "Van Beethoven"
},
{
"path": "data[0].billing_address",
"expected": [],
"actual": "Leonardo da Vinci, da@vinci.com"
}
] |
| single_product_multiple_shipment_complex.json | opus-4-5 | raw_only | 1 | 19 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
},
{
"path": "data[0].vendor_name",
"expected": "flamingos",
"actual": "Van Beethoven"
},
{
"path": "data[0].billing_address",
"expected": [],
"actual": ""
}
] |
| single_product_multiple_shipment_complex.json | opus-4-5 | db_plus_raw | 1 | 19 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "FOB Singapore",
"actual": "FOB"
},
{
"path": "data[0].vendor_name",
"expected": "flamingos",
"actual": "Van Beethoven"
},
{
"path": "data[0].billing_address",
"expected": [],
"actual": ""
}
] |
| multiple_product_multiple_shipment_simple.json | sonnet-4-5 | raw_only | 1 | 19 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-200",
"actual": "PO-2025-11-200"
}
] |
| multiple_product_multiple_shipment_medium.json | sonnet-4-6 | raw_only | 1 | 19 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.75,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "EXW Singapore",
"actual": "EXW"
},
{
"path": "data[0].vendor_name",
"expected": "flamingos",
"actual": "Van Beethoven"
}
] |
| multiple_product_multiple_shipment_medium.json | opus-4-5 | raw_only | 1 | 19 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].unit_price",
"expected": 23.75,
"actual": 25.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "EXW Singapore",
"actual": "EXW"
},
{
"path": "data[0].vendor_name",
"expected": "flamingos",
"actual": "Van Beethoven"
}
] |
| multiple_product_multiple_shipment_simple.json | opus-4-6 | raw_only | 1 | 18 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-200",
"actual": "PO-2025-11-200"
}
] |
| multiple_product_multiple_shipment_simple.json | opus-4-6 | db_plus_raw | 1 | 18 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-200",
"actual": "PO-2025-11-200"
}
] |
| single_product_single_shipment_simple.json | sonnet-4-6 | db_only | 1 | 17 | [
{
"path": "data[0].items[0].description",
"expected": "KNM Coffee",
"actual": "BP102"
},
{
"path": "data[0].items[0].quantity",
"expected": 5.0,
"actual": 23.0
},
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "MT"
},
{
"path": "data[0].items[0].unit_price",
"expected": 25.0,
"actual": 1410.0
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/MT"
}
] |
| single_product_multiple_shipment_medium.json | sonnet-4-6 | raw_only | 1 | 17 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2025-11-180",
"actual": ""
}
] |
| single_product_multiple_shipment_medium.json | sonnet-4-6 | db_plus_raw | 1 | 17 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2025-11-180",
"actual": ""
}
] |
| single_product_multiple_shipment_medium.json | sonnet-4-6 | db_only | 1 | 17 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2025-11-180",
"actual": ""
}
] |
| single_product_multiple_shipment_medium.json | sonnet-4-5 | raw_only | 1 | 17 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2025-11-180",
"actual": ""
}
] |
| single_product_multiple_shipment_medium.json | sonnet-4-5 | db_only | 1 | 17 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2025-11-180",
"actual": ""
}
] |
| single_product_multiple_shipment_medium.json | opus-4-6 | raw_only | 1 | 17 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2025-11-180",
"actual": ""
}
] |
| single_product_multiple_shipment_medium.json | opus-4-6 | db_plus_raw | 1 | 17 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2025-11-180",
"actual": ""
}
] |
| single_product_multiple_shipment_medium.json | opus-4-5 | raw_only | 1 | 17 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2025-11-180",
"actual": ""
}
] |
| single_product_multiple_shipment_medium.json | opus-4-5 | db_plus_raw | 1 | 17 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2025-11-180",
"actual": ""
}
] |
| single_product_multiple_shipment_medium.json | opus-4-5 | db_only | 1 | 17 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2025-11-180",
"actual": ""
}
] |
| real_world_msgs_test_v3.json | sonnet-4-5 | raw_only | 1 | 17 | [
{
"path": "data[0].items[0].quantity",
"expected": 8.0,
"actual": 8000.0
},
{
"path": "data[0].items[0].quantity_unit",
"expected": "MT",
"actual": "KG"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/kg",
"actual": "USD/KG"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
}
] |
| real_world_msgs_test_v3.json | sonnet-4-5 | db_plus_raw | 1 | 17 | [
{
"path": "data[0].items[0].quantity",
"expected": 8.0,
"actual": 8000.0
},
{
"path": "data[0].items[0].quantity_unit",
"expected": "MT",
"actual": "KG"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/kg",
"actual": "USD/KG"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
}
] |
| real_world_msgs_test_v2.json | sonnet-4-5 | raw_only | 1 | 17 | [
{
"path": "data[0].items[0].quantity",
"expected": 23.0,
"actual": 23000.0
},
{
"path": "data[0].items[0].quantity_unit",
"expected": "MT",
"actual": "KG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Busan",
"actual": "CIF"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
}
] |
| real_world_msgs_test_v2.json | sonnet-4-6 | raw_only | 1 | 16 | [
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Busan",
"actual": "CIF"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": [
"CIF Busan",
"Busan"
],
"actual": ""
}
] |
| multiple_product_multiple_shipment_simple.json | sonnet-4-6 | raw_only | 1 | 16 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-200",
"actual": "PO-2025-11-200"
}
] |
| multiple_product_multiple_shipment_simple.json | sonnet-4-6 | db_plus_raw | 1 | 16 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-200",
"actual": "PO-2025-11-200"
}
] |
| multiple_product_multiple_shipment_simple.json | sonnet-4-6 | db_only | 1 | 16 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-200",
"actual": "PO-2025-11-200"
}
] |
| multiple_product_multiple_shipment_simple.json | sonnet-4-5 | db_plus_raw | 1 | 16 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-200",
"actual": "PO-2025-11-200"
}
] |
| multiple_product_multiple_shipment_simple.json | opus-4-5 | db_plus_raw | 1 | 16 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-200",
"actual": "PO-2025-11-200"
}
] |
| multiple_product_multiple_shipment_simple.json | opus-4-5 | db_only | 1 | 16 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-200",
"actual": "PO-2025-11-200"
}
] |
| real_world_msgs_test_v2.json | sonnet-4-6 | zero_shot | 1 | 15 | [
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Busan",
"actual": "CIF"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": [
"CIF Busan",
"Busan"
],
"actual": ""
},
{
"path": "data[0].billing_address",
"expected": null,
"actual": ""
}
] |
| single_product_multiple_shipment_simple.json | sonnet-4-6 | raw_only | 1 | 14 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 375.0,
"actual": 200.0
},
{
"path": "data[0].do_date",
"expected": "2026-02-28",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-150",
"actual": "PO-2025-11-150"
}
] |
| single_product_multiple_shipment_simple.json | sonnet-4-6 | db_plus_raw | 1 | 14 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 375.0,
"actual": 200.0
},
{
"path": "data[0].do_date",
"expected": "2026-02-28",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-150",
"actual": "PO-2025-11-150"
}
] |
| single_product_multiple_shipment_simple.json | sonnet-4-6 | db_only | 1 | 14 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 375.0,
"actual": 200.0
},
{
"path": "data[0].do_date",
"expected": "2026-02-28",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-150",
"actual": "PO-2025-11-150"
}
] |
| single_product_multiple_shipment_simple.json | sonnet-4-5 | db_plus_raw | 1 | 14 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 375.0,
"actual": 200.0
},
{
"path": "data[0].do_date",
"expected": "2026-02-28",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-150",
"actual": "PO-2025-11-150"
}
] |
| single_product_multiple_shipment_simple.json | sonnet-4-5 | db_only | 1 | 14 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 375.0,
"actual": 200.0
},
{
"path": "data[0].do_date",
"expected": "2026-02-28",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-150",
"actual": "PO-2025-11-150"
}
] |
| single_product_multiple_shipment_simple.json | opus-4-6 | raw_only | 1 | 14 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 375.0,
"actual": 200.0
},
{
"path": "data[0].do_date",
"expected": "2026-02-28",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-150",
"actual": "PO-2025-11-150"
}
] |
| single_product_multiple_shipment_simple.json | opus-4-6 | db_plus_raw | 1 | 14 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 375.0,
"actual": 200.0
},
{
"path": "data[0].do_date",
"expected": "2026-02-28",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-150",
"actual": "PO-2025-11-150"
}
] |
| single_product_multiple_shipment_simple.json | opus-4-5 | db_only | 1 | 14 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 375.0,
"actual": 200.0
},
{
"path": "data[0].do_date",
"expected": "2026-02-28",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-150",
"actual": "PO-2025-11-150"
}
] |
| real_world_msgs_test_v2.json | opus-4-6 | zero_shot | 1 | 14 | [
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Busan",
"actual": "CIF"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": [
"CIF Busan",
"Busan"
],
"actual": ""
},
{
"path": "data[0].billing_address",
"expected": null,
"actual": ""
}
] |
| multiple_product_multiple_shipment_simple.json | opus-4-5 | raw_only | 1 | 14 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Singapore",
"actual": "CIF"
},
{
"path": "data[0].do_date",
"expected": "2026-04-30",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-200",
"actual": "PO-2025-11-200"
}
] |
| single_product_multiple_shipment_simple.json | sonnet-4-5 | raw_only | 1 | 13 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 375.0,
"actual": 200.0
},
{
"path": "data[0].do_date",
"expected": "2026-02-28",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-150",
"actual": "PO-2025-11-150"
}
] |
| single_product_multiple_shipment_simple.json | opus-4-5 | db_plus_raw | 1 | 13 | [
{
"path": "data[0].items[0].quantity_unit",
"expected": "bags",
"actual": "BAGS"
},
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/bag",
"actual": "USD/BAG"
},
{
"path": "data[0].items[0].total",
"expected": 375.0,
"actual": 200.0
},
{
"path": "data[0].do_date",
"expected": "2026-02-28",
"actual": "2026-05-31"
},
{
"path": "data[0].po_ref_no",
"expected": "PO-2024-11-150",
"actual": "PO-2025-11-150"
}
] |
| real_world_msgs_test_v3.json | sonnet-4-6 | raw_only | 1 | 13 | [
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/kg",
"actual": "USD/KG"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": null,
"actual": ""
}
] |
| real_world_msgs_test_v3.json | sonnet-4-6 | db_plus_raw | 1 | 13 | [
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/kg",
"actual": "USD/KG"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": null,
"actual": ""
}
] |
| real_world_msgs_test_v3.json | sonnet-4-6 | db_only | 1 | 13 | [
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/kg",
"actual": "USD/KG"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": null,
"actual": ""
}
] |
| real_world_msgs_test_v3.json | sonnet-4-5 | db_only | 1 | 13 | [
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/kg",
"actual": "USD/KG"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": null,
"actual": ""
}
] |
| real_world_msgs_test_v3.json | opus-4-6 | raw_only | 1 | 13 | [
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/kg",
"actual": "USD/KG"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": null,
"actual": ""
}
] |
| real_world_msgs_test_v3.json | opus-4-6 | db_plus_raw | 1 | 13 | [
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/kg",
"actual": "USD/KG"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": null,
"actual": ""
}
] |
| real_world_msgs_test_v3.json | opus-4-6 | db_only | 1 | 13 | [
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/kg",
"actual": "USD/KG"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": null,
"actual": ""
}
] |
| real_world_msgs_test_v3.json | opus-4-5 | raw_only | 1 | 13 | [
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/kg",
"actual": "USD/KG"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": null,
"actual": ""
}
] |
| real_world_msgs_test_v3.json | opus-4-5 | db_plus_raw | 1 | 13 | [
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/kg",
"actual": "USD/KG"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": null,
"actual": ""
}
] |
| real_world_msgs_test_v3.json | opus-4-5 | db_only | 1 | 13 | [
{
"path": "data[0].items[0].pricing_unit",
"expected": "USD/kg",
"actual": "USD/KG"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": null,
"actual": ""
}
] |
| real_world_msgs_test_v2.json | sonnet-4-6 | db_plus_raw | 1 | 13 | [
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Busan",
"actual": "CIF"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": [
"CIF Busan",
"Busan"
],
"actual": ""
}
] |
| real_world_msgs_test_v2.json | sonnet-4-6 | db_only | 1 | 13 | [
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Busan",
"actual": "CIF"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": [
"CIF Busan",
"Busan"
],
"actual": ""
}
] |
| real_world_msgs_test_v2.json | sonnet-4-5 | zero_shot | 1 | 13 | [
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Busan",
"actual": "CIF"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": [
"CIF Busan",
"Busan"
],
"actual": ""
}
] |
| real_world_msgs_test_v2.json | sonnet-4-5 | db_plus_raw | 1 | 13 | [
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Busan",
"actual": "CIF"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": [
"CIF Busan",
"Busan"
],
"actual": ""
}
] |
| real_world_msgs_test_v2.json | opus-4-6 | raw_only | 1 | 13 | [
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Busan",
"actual": "CIF"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": [
"CIF Busan",
"Busan"
],
"actual": ""
}
] |
| real_world_msgs_test_v2.json | opus-4-6 | db_plus_raw | 1 | 13 | [
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Busan",
"actual": "CIF"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": [
"CIF Busan",
"Busan"
],
"actual": ""
}
] |
| real_world_msgs_test_v2.json | opus-4-6 | db_only | 1 | 13 | [
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Busan",
"actual": "CIF"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": [
"CIF Busan",
"Busan"
],
"actual": ""
}
] |
| real_world_msgs_test_v2.json | opus-4-5 | zero_shot | 1 | 13 | [
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Busan",
"actual": "CIF"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": [
"CIF Busan",
"Busan"
],
"actual": ""
}
] |
| real_world_msgs_test_v2.json | opus-4-5 | raw_only | 1 | 13 | [
{
"path": "data[0].items[0].ship_term",
"expected": "CIF Busan",
"actual": "CIF"
},
{
"path": "data[0].items[0].loading",
"expected": "",
"actual": "23MT/40'FCL"
},
{
"path": "data[0].po_ref_no",
"expected": null,
"actual": ""
},
{
"path": "data[0].payment_date",
"expected": null,
"actual": ""
},
{
"path": "data[0].shipping_address",
"expected": [
"CIF Busan",
"Busan"
],
"actual": ""
}
] |