{ "task1_global": { "per_pair_best": { "thinkhome-brick": { "matcher": "AML", "F1_Rplus": 0.7742, "P": 0.8276, "R": 0.7273 }, "brick-smartenv": { "matcher": "LogMap", "F1_Rplus": 0.8293, "P": 0.7391, "R": 0.9444 }, "city-brick": { "matcher": "LogMap", "F1_Rplus": 0.7568, "P": 0.7778, "R": 0.7368 }, "jc3iedm-mio": { "matcher": "LogMapLLM", "F1_Rplus": 0.928, "P": 0.9957, "R": 0.8689 }, "jc3iedm-brick": { "matcher": "LogMapLt", "F1_Rplus": 0.8188, "P": 0.8243, "R": 0.8133 }, "jc3iedm-facility": { "matcher": "LogMap", "F1_Rplus": 0.9767, "P": 0.9767, "R": 0.9767 } }, "aggregate": [ { "matcher": "ALOD2Vec", "n_pairs": 2, "micro_Rplus": { "P": 0.9634, "R": 0.2548, "F1": 0.4031, "TP": 79, "judgeable": 82, "reference": 310, "pairs": 2 }, "macro_Rplus": { "P": 0.9641, "R": 0.5089, "F1": 0.5918, "pairs": 2 }, "micro_Rapprox": { "P": 0.9634, "R": 0.2548, "F1": 0.4031, "TP": 79, "judgeable": 82, "reference": 310, "pairs": 2 }, "macro_Rapprox": { "P": 0.9641, "R": 0.5089, "F1": 0.5918, "pairs": 2 } }, { "matcher": "AML", "n_pairs": 6, "micro_Rplus": { "P": 0.8942, "R": 0.3463, "F1": 0.4993, "TP": 169, "judgeable": 189, "reference": 488, "pairs": 6 }, "macro_Rplus": { "P": 0.8789, "R": 0.5778, "F1": 0.6436, "pairs": 6 }, "micro_Rapprox": { "P": 0.9825, "R": 0.3678, "F1": 0.5352, "TP": 224, "judgeable": 228, "reference": 609, "pairs": 6 }, "macro_Rapprox": { "P": 0.9892, "R": 0.5099, "F1": 0.6346, "pairs": 6 } }, { "matcher": "ATMatcher", "n_pairs": 2, "micro_Rplus": { "P": 0.9259, "R": 0.2419, "F1": 0.3836, "TP": 75, "judgeable": 81, "reference": 310, "pairs": 2 }, "macro_Rplus": { "P": 0.9278, "R": 0.4234, "F1": 0.537, "pairs": 2 }, "micro_Rapprox": { "P": 0.9259, "R": 0.2419, "F1": 0.3836, "TP": 75, "judgeable": 81, "reference": 310, "pairs": 2 }, "macro_Rapprox": { "P": 0.9278, "R": 0.4234, "F1": 0.537, "pairs": 2 } }, { "matcher": "BertMap", "n_pairs": 6, "micro_Rplus": { "P": 0.9412, "R": 0.1967, "F1": 0.3254, "TP": 96, "judgeable": 102, "reference": 488, "pairs": 6 }, "macro_Rplus": { "P": 0.9232, "R": 0.3431, "F1": 0.4631, "pairs": 6 }, "micro_Rapprox": { "P": 0.9725, "R": 0.1741, "F1": 0.2953, "TP": 106, "judgeable": 109, "reference": 609, "pairs": 6 }, "macro_Rapprox": { "P": 0.9714, "R": 0.259, "F1": 0.3792, "pairs": 6 } }, { "matcher": "BertMapLt", "n_pairs": 6, "micro_Rplus": { "P": 0.906, "R": 0.2172, "F1": 0.3504, "TP": 106, "judgeable": 117, "reference": 488, "pairs": 6 }, "macro_Rplus": { "P": 0.859, "R": 0.3673, "F1": 0.4748, "pairs": 6 }, "micro_Rapprox": { "P": 1.0, "R": 0.2282, "F1": 0.3717, "TP": 139, "judgeable": 139, "reference": 609, "pairs": 6 }, "macro_Rapprox": { "P": 1.0, "R": 0.3423, "F1": 0.4793, "pairs": 6 } }, { "matcher": "CySecBertMap", "n_pairs": 6, "micro_Rplus": { "P": 0.9596, "R": 0.1947, "F1": 0.3237, "TP": 95, "judgeable": 99, "reference": 488, "pairs": 6 }, "macro_Rplus": { "P": 0.9394, "R": 0.3343, "F1": 0.459, "pairs": 6 }, "micro_Rapprox": { "P": 0.9906, "R": 0.1724, "F1": 0.2937, "TP": 105, "judgeable": 106, "reference": 609, "pairs": 6 }, "macro_Rapprox": { "P": 0.9912, "R": 0.2563, "F1": 0.3768, "pairs": 6 } }, { "matcher": "Fine-TOM", "n_pairs": 2, "micro_Rplus": { "P": 0.625, "R": 0.1613, "F1": 0.2564, "TP": 50, "judgeable": 80, "reference": 310, "pairs": 2 }, "macro_Rplus": { "P": 0.6021, "R": 0.201, "F1": 0.2633, "pairs": 2 }, "micro_Rapprox": { "P": 0.625, "R": 0.1613, "F1": 0.2564, "TP": 50, "judgeable": 80, "reference": 310, "pairs": 2 }, "macro_Rapprox": { "P": 0.6021, "R": 0.201, "F1": 0.2633, "pairs": 2 } }, { "matcher": "KGMatcher", "n_pairs": 2, "micro_Rplus": { "P": 1.0, "R": 0.0839, "F1": 0.1548, "TP": 26, "judgeable": 26, "reference": 310, "pairs": 2 }, "macro_Rplus": { "P": 1.0, "R": 0.1658, "F1": 0.268, "pairs": 2 }, "micro_Rapprox": { "P": 1.0, "R": 0.0839, "F1": 0.1548, "TP": 26, "judgeable": 26, "reference": 310, "pairs": 2 }, "macro_Rapprox": { "P": 1.0, "R": 0.1658, "F1": 0.268, "pairs": 2 } }, { "matcher": "LogMap", "n_pairs": 6, "micro_Rplus": { "P": 0.9571, "R": 0.7766, "F1": 0.8575, "TP": 379, "judgeable": 396, "reference": 488, "pairs": 6 }, "macro_Rplus": { "P": 0.8963, "R": 0.7708, "F1": 0.8126, "pairs": 6 }, "micro_Rapprox": { "P": 0.9737, "R": 0.6683, "F1": 0.7926, "TP": 407, "judgeable": 418, "reference": 609, "pairs": 6 }, "macro_Rapprox": { "P": 0.9498, "R": 0.6202, "F1": 0.7286, "pairs": 6 } }, { "matcher": "LogMapLLM", "n_pairs": 2, "micro_Rplus": { "P": 0.9963, "R": 0.8774, "F1": 0.9331, "TP": 272, "judgeable": 273, "reference": 310, "pairs": 2 }, "macro_Rplus": { "P": 0.9979, "R": 0.8996, "F1": 0.946, "pairs": 2 }, "micro_Rapprox": { "P": 0.9963, "R": 0.8774, "F1": 0.9331, "TP": 272, "judgeable": 273, "reference": 310, "pairs": 2 }, "macro_Rapprox": { "P": 0.9979, "R": 0.8996, "F1": 0.946, "pairs": 2 } }, { "matcher": "LogMapLt", "n_pairs": 6, "micro_Rplus": { "P": 0.7782, "R": 0.4242, "F1": 0.5491, "TP": 207, "judgeable": 266, "reference": 488, "pairs": 6 }, "macro_Rplus": { "P": 0.7895, "R": 0.6507, "F1": 0.6655, "pairs": 6 }, "micro_Rapprox": { "P": 0.8518, "R": 0.5189, "F1": 0.6449, "TP": 316, "judgeable": 371, "reference": 609, "pairs": 6 }, "macro_Rapprox": { "P": 0.8828, "R": 0.7049, "F1": 0.74, "pairs": 6 } }, { "matcher": "Matcha", "n_pairs": 6, "micro_Rplus": { "P": 0.4407, "R": 0.4877, "F1": 0.463, "TP": 238, "judgeable": 540, "reference": 488, "pairs": 6 }, "macro_Rplus": { "P": 0.5244, "R": 0.6896, "F1": 0.566, "pairs": 6 }, "micro_Rapprox": { "P": 0.3885, "R": 0.5747, "F1": 0.4636, "TP": 350, "judgeable": 901, "reference": 609, "pairs": 6 }, "macro_Rapprox": { "P": 0.4694, "R": 0.7637, "F1": 0.5356, "pairs": 6 } }, { "matcher": "SecureBertMap", "n_pairs": 6, "micro_Rplus": { "P": 0.9612, "R": 0.2029, "F1": 0.335, "TP": 99, "judgeable": 103, "reference": 488, "pairs": 6 }, "macro_Rplus": { "P": 0.9431, "R": 0.3498, "F1": 0.4686, "pairs": 6 }, "micro_Rapprox": { "P": 0.9909, "R": 0.179, "F1": 0.3032, "TP": 109, "judgeable": 110, "reference": 609, "pairs": 6 }, "macro_Rapprox": { "P": 0.9949, "R": 0.2718, "F1": 0.3864, "pairs": 6 } } ], "reference_sizes": { "thinkhome-brick": { "Rapprox": 116, "Rplus": 66, "coherent": false }, "brick-smartenv": { "Rapprox": 47, "Rplus": 18, "coherent": false }, "city-brick": { "Rapprox": 61, "Rplus": 19, "coherent": false }, "jc3iedm-mio": { "Rapprox": 267, "Rplus": 267, "coherent": true }, "jc3iedm-brick": { "Rapprox": 75, "Rplus": 75, "coherent": true }, "jc3iedm-facility": { "Rapprox": 43, "Rplus": 43, "coherent": true } } }, "task2_ranking": { "lexical": { "uco-stix": { "tau": 0.5, "n": 217, "overall": { "mrr": 0.9495, "queries": 217.0, "hits_at_1": 0.9171, "hits_at_3": 0.977, "hits_at_5": 0.9816, "hits_at_10": 1.0 }, "matched": { "mrr": 0.9768, "queries": 194.0, "hits_at_1": 0.9536, "hits_at_3": 1.0, "hits_at_5": 1.0, "hits_at_10": 1.0 }, "NIL": { "mrr": 0.7194, "queries": 23.0, "hits_at_1": 0.6087, "hits_at_3": 0.7826, "hits_at_5": 0.8261, "hits_at_10": 1.0 }, "per_type": { "CLS": { "mrr": 0.9433, "queries": 47.0, "hits_at_1": 0.8936, "hits_at_3": 1.0, "hits_at_5": 1.0, "hits_at_10": 1.0 }, "OPROP": { "mrr": 0.8333, "queries": 12.0, "hits_at_1": 0.6667, "hits_at_3": 1.0, "hits_at_5": 1.0, "hits_at_10": 1.0 }, "DPROP": { "mrr": 0.9602, "queries": 158.0, "hits_at_1": 0.943, "hits_at_3": 0.9684, "hits_at_5": 0.9747, "hits_at_10": 1.0 } } }, "stix-d3fend": { "tau": 0.5, "n": 73, "overall": { "mrr": 0.8489, "queries": 73.0, "hits_at_1": 0.7534, "hits_at_3": 0.9589, "hits_at_5": 0.9589, "hits_at_10": 0.9726 }, "matched": { "mrr": 0.8897, "queries": 38.0, "hits_at_1": 0.8158, "hits_at_3": 0.9737, "hits_at_5": 0.9737, "hits_at_10": 1.0 }, "NIL": { "mrr": 0.8046, "queries": 35.0, "hits_at_1": 0.6857, "hits_at_3": 0.9429, "hits_at_5": 0.9429, "hits_at_10": 0.9429 }, "per_type": { "CLS": { "mrr": 0.854, "queries": 32.0, "hits_at_1": 0.7812, "hits_at_3": 0.9375, "hits_at_5": 0.9375, "hits_at_10": 0.9375 }, "OPROP": { "mrr": 0.6667, "queries": 3.0, "hits_at_1": 0.3333, "hits_at_3": 1.0, "hits_at_5": 1.0, "hits_at_10": 1.0 }, "DPROP": { "mrr": 0.859, "queries": 38.0, "hits_at_1": 0.7632, "hits_at_3": 0.9737, "hits_at_5": 0.9737, "hits_at_10": 1.0 } } }, "MACRO": { "hits_at_1": 0.8353, "hits_at_3": 0.9679, "hits_at_5": 0.9703, "hits_at_10": 0.9863, "mrr": 0.8992 } }, "minilm": { "uco-stix": { "tau": 0.5, "n": 217, "overall": { "mrr": 0.8297, "queries": 217.0, "hits_at_1": 0.7281, "hits_at_3": 0.9263, "hits_at_5": 0.9539, "hits_at_10": 0.9816 }, "matched": { "mrr": 0.8729, "queries": 194.0, "hits_at_1": 0.7887, "hits_at_3": 0.9536, "hits_at_5": 0.9742, "hits_at_10": 0.9897 }, "NIL": { "mrr": 0.4648, "queries": 23.0, "hits_at_1": 0.2174, "hits_at_3": 0.6957, "hits_at_5": 0.7826, "hits_at_10": 0.913 }, "per_type": { "CLS": { "mrr": 0.8589, "queries": 47.0, "hits_at_1": 0.766, "hits_at_3": 0.9574, "hits_at_5": 0.9787, "hits_at_10": 1.0 }, "OPROP": { "mrr": 0.875, "queries": 12.0, "hits_at_1": 0.75, "hits_at_3": 1.0, "hits_at_5": 1.0, "hits_at_10": 1.0 }, "DPROP": { "mrr": 0.8176, "queries": 158.0, "hits_at_1": 0.7152, "hits_at_3": 0.9114, "hits_at_5": 0.943, "hits_at_10": 0.9747 } } }, "stix-d3fend": { "tau": 0.5, "n": 73, "overall": { "mrr": 0.7434, "queries": 73.0, "hits_at_1": 0.6027, "hits_at_3": 0.8356, "hits_at_5": 0.9315, "hits_at_10": 0.9589 }, "matched": { "mrr": 0.8708, "queries": 38.0, "hits_at_1": 0.7895, "hits_at_3": 0.9211, "hits_at_5": 0.9737, "hits_at_10": 0.9737 }, "NIL": { "mrr": 0.6051, "queries": 35.0, "hits_at_1": 0.4, "hits_at_3": 0.7429, "hits_at_5": 0.8857, "hits_at_10": 0.9429 }, "per_type": { "CLS": { "mrr": 0.766, "queries": 32.0, "hits_at_1": 0.6875, "hits_at_3": 0.7812, "hits_at_5": 0.875, "hits_at_10": 0.9375 }, "OPROP": { "mrr": 0.5, "queries": 3.0, "hits_at_1": 0.0, "hits_at_3": 1.0, "hits_at_5": 1.0, "hits_at_10": 1.0 }, "DPROP": { "mrr": 0.7436, "queries": 38.0, "hits_at_1": 0.5789, "hits_at_3": 0.8684, "hits_at_5": 0.9737, "hits_at_10": 0.9737 } } }, "MACRO": { "hits_at_1": 0.6654, "hits_at_3": 0.881, "hits_at_5": 0.9427, "hits_at_10": 0.9703, "mrr": 0.7865 } } } }