{ "schema": "autoarena/leaderboard/1", "tasks": [ { "id": "data", "title": "Training tokens to quality at validation BPB < 1.05", "metric": "tokens_to_target", "unit": "tokens", "direction": "minimize", "budget": 100, "comparison_step": null, "quality_gate": { "metric": "val_bpb", "operator": "<", "value": 1.05 }, "tiebreaks": [ { "metric": "updates_to_target", "direction": "minimize" } ], "runs": [ { "run_id": "sequential-search--data", "method_id": "sequential-search", "task_id": "data", "metric": "tokens_to_target", "unit": "tokens", "direction": "minimize", "best_value": 152455168, "reference_value": 394788864, "relative_improvement": 0.6138311337981408, "rank_key": [ 152455168.0, 2335.0, -0.00035004053235021537 ], "best_launch_seq": 94, "best_candidate_id": "c095", "best_is_reference": false, "budget": { "spent": 97, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 15.91517527777778, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 394788864, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "c001", "value": 273940480, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "c002", "value": 254017536, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "c003", "value": 247201792, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "c004", "value": 227540992, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "c005", "value": 246415360, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "c006", "value": 208666624, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "c007", "value": 188612608, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "c008", "value": 170393600, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "c009", "value": 187432960, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "c010", "value": 187170816, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "c012", "value": 189923328, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "c013", "value": 182321152, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "c014", "value": 207880192, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "c015", "value": 170393600, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "c016", "value": 171048960, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "c017", "value": 180748288, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "c018", "value": 169363456, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "c019", "value": 187582464, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "c020", "value": 173033472, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "c021", "value": 169363456, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "c022", "value": 170149888, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "c023", "value": 206981120, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "c024", "value": 188762112, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "c025", "value": 171853824, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "c026", "value": 185092096, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "c027", "value": 169363456, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "c028", "value": 189024256, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "c029", "value": 170018816, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "c030", "value": 187713536, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "c031", "value": 186402816, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "c032", "value": 169887744, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "c033", "value": 167659520, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "c034", "value": 167659520, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "c035", "value": 165300224, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "c036", "value": 183388160, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "c037", "value": 183519232, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "c038", "value": 183388160, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "c039", "value": 165300224, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "c040", "value": 178014208, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "c041", "value": 183584768, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "c042", "value": 168427520, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "c043", "value": 181946368, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "c044", "value": 182601728, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "c045", "value": 183257088, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "c046", "value": 183126016, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "c047", "value": 164513792, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "c048", "value": 165038080, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "c049", "value": 161138688, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "c050", "value": 181039104, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "c051", "value": 179324928, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "c052", "value": 178833408, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "c053", "value": 180602880, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "c054", "value": 178145280, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "c056", "value": 177620992, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "c058", "value": 163891200, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "c059", "value": 163203072, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "c060", "value": 180799488, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "c061", "value": 163006464, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "c062", "value": 161859584, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "c063", "value": 159197184, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "c064", "value": 158156800, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "c065", "value": 175785984, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "c066", "value": 158091264, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "c067", "value": 158091264, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "c068b", "value": 174868480, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "c069", "value": 157763584, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "c070", "value": 171591680, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "c071", "value": 189089792, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "c072", "value": 171722752, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "c073", "value": 171919360, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "c074", "value": 154421248, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "c075", "value": 167380992, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "c076", "value": 169560064, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "c077", "value": 170215424, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "c078", "value": 173033472, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "c079", "value": 156059648, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "c080", "value": 173033472, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "c081", "value": 155994112, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "c082", "value": 172640256, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "c083", "value": 172509184, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "c084", "value": 155600896, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "c085", "value": 155994112, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "c086", "value": 168335360, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "c087", "value": 165396480, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "c088", "value": 172785664, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "c089", "value": 154028032, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "c090", "value": 153700352, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "c091", "value": 152651776, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "c092", "value": 152913920, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "c093", "value": 154159104, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "c094", "value": 152717312, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "c095", "value": 152455168, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "c096", "value": 152651776, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "c097", "value": 152520704, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "c098", "value": 153044992, "is_reference": false } ], "curve": [ { "step": 1, "value": 394788864 }, { "step": 2, "value": 273940480 }, { "step": 3, "value": 254017536 }, { "step": 4, "value": 247201792 }, { "step": 5, "value": 227540992 }, { "step": 6, "value": 227540992 }, { "step": 7, "value": 208666624 }, { "step": 8, "value": 188612608 }, { "step": 9, "value": 170393600 }, { "step": 10, "value": 170393600 }, { "step": 11, "value": 170393600 }, { "step": 12, "value": 170393600 }, { "step": 13, "value": 170393600 }, { "step": 14, "value": 170393600 }, { "step": 15, "value": 170393600 }, { "step": 16, "value": 170393600 }, { "step": 17, "value": 170393600 }, { "step": 18, "value": 170393600 }, { "step": 19, "value": 169363456 }, { "step": 20, "value": 169363456 }, { "step": 21, "value": 169363456 }, { "step": 22, "value": 169363456 }, { "step": 23, "value": 169363456 }, { "step": 24, "value": 169363456 }, { "step": 25, "value": 169363456 }, { "step": 26, "value": 169363456 }, { "step": 27, "value": 169363456 }, { "step": 28, "value": 169363456 }, { "step": 29, "value": 169363456 }, { "step": 30, "value": 169363456 }, { "step": 31, "value": 169363456 }, { "step": 32, "value": 169363456 }, { "step": 33, "value": 169363456 }, { "step": 34, "value": 167659520 }, { "step": 35, "value": 167659520 }, { "step": 36, "value": 165300224 }, { "step": 37, "value": 165300224 }, { "step": 38, "value": 165300224 }, { "step": 39, "value": 165300224 }, { "step": 40, "value": 165300224 }, { "step": 41, "value": 165300224 }, { "step": 42, "value": 165300224 }, { "step": 43, "value": 165300224 }, { "step": 44, "value": 165300224 }, { "step": 45, "value": 165300224 }, { "step": 46, "value": 165300224 }, { "step": 47, "value": 165300224 }, { "step": 48, "value": 164513792 }, { "step": 49, "value": 164513792 }, { "step": 50, "value": 161138688 }, { "step": 51, "value": 161138688 }, { "step": 52, "value": 161138688 }, { "step": 53, "value": 161138688 }, { "step": 54, "value": 161138688 }, { "step": 55, "value": 161138688 }, { "step": 56, "value": 161138688 }, { "step": 57, "value": 161138688 }, { "step": 58, "value": 161138688 }, { "step": 59, "value": 161138688 }, { "step": 60, "value": 161138688 }, { "step": 61, "value": 161138688 }, { "step": 62, "value": 159197184 }, { "step": 63, "value": 158156800 }, { "step": 64, "value": 158156800 }, { "step": 65, "value": 158091264 }, { "step": 66, "value": 158091264 }, { "step": 67, "value": 158091264 }, { "step": 68, "value": 157763584 }, { "step": 69, "value": 157763584 }, { "step": 70, "value": 157763584 }, { "step": 71, "value": 157763584 }, { "step": 72, "value": 157763584 }, { "step": 73, "value": 154421248 }, { "step": 74, "value": 154421248 }, { "step": 75, "value": 154421248 }, { "step": 76, "value": 154421248 }, { "step": 77, "value": 154421248 }, { "step": 78, "value": 154421248 }, { "step": 79, "value": 154421248 }, { "step": 80, "value": 154421248 }, { "step": 81, "value": 154421248 }, { "step": 82, "value": 154421248 }, { "step": 83, "value": 154421248 }, { "step": 84, "value": 154421248 }, { "step": 85, "value": 154421248 }, { "step": 86, "value": 154421248 }, { "step": 87, "value": 154421248 }, { "step": 88, "value": 154028032 }, { "step": 89, "value": 153700352 }, { "step": 90, "value": 152651776 }, { "step": 91, "value": 152651776 }, { "step": 92, "value": 152651776 }, { "step": 93, "value": 152651776 }, { "step": 94, "value": 152455168 }, { "step": 95, "value": 152455168 }, { "step": 96, "value": 152455168 }, { "step": 97, "value": 152455168 } ], "submission_sha256": "f8693f5f1f9fd2ded68c073b66140d805f7f7596c8ad958cd84b154da2b5c744", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/sequential-search/data/submission.json", "comparison_value": 152455168, "comparison_rank_key": [ 152455168.0, 2335.0, -0.00035004053235021537 ], "rank": 1 }, { "run_id": "heuresis-map-elites--data", "method_id": "heuresis-map-elites", "task_id": "data", "metric": "tokens_to_target", "unit": "tokens", "direction": "minimize", "best_value": 156172288, "reference_value": 393740288, "relative_improvement": 0.6033621837549934, "rank_key": [ 156172288.0, 2383.0, -0.00014194283152213494 ], "best_launch_seq": 90, "best_candidate_id": "exp-0a9febf0fda90bf391ec0597", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 20.76471361111112, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 393740288, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-fdd29f3ab47868cb3a031cc5", "value": 254017536, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp-3457cb285acd43c92aab388e", "value": 227803136, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-e833d443aac38aa63cdf4900", "value": 189530112, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-fee9fed93499bd5a1216b213", "value": 171966464, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-ee8d4bcca6f3fa8f12bc5e54", "value": 171835392, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-9f4d6f0112686282d2f57a8a", "value": 165478400, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp-439cb22da1c64d0aa520471e", "value": 165412864, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp-db4e6f245d44ce158f472e72", "value": 165347328, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-f1bba4989c627eca9943bd1a", "value": 201981952, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-5c2e77e21fa424ecc9f9925a", "value": 164888576, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-28b44648b279439cb12ce62c", "value": 165085184, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-4076da6a4348fc9db958f610", "value": 349962240, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-05336566c6b719f605840e56", "value": 353632256, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp-11566c55071fccf1ab6d088d", "value": 238157824, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-e0ed15f96ccfe7a118e3694f", "value": 198311936, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-cbfb1e0b73d986d26c38bc69", "value": 200671232, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-1806cfa720d6114fc4bce216", "value": 208404480, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-760e12fa82c82bd74e519422", "value": 201326592, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp-e545599aa9962c218370e407", "value": 355467264, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp-73c864ab8346b08eafd7a3d5", "value": 203423744, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp-95854fbd198cdc8a009fe15a", "value": 213450752, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp-aa749a29ef98d4151aef619d", "value": 188743680, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-e024761e8b417bf93382d3e8", "value": 170000384, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp-a75120cef1e442146b3740c8", "value": 190054400, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp-0b70dc9aafcec439515f386f", "value": 190185472, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp-6abfb786bce69a16ae86e549", "value": 165085184, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp-6dbe2d24fb705f1c921baf29", "value": 165478400, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp-978a2b1e4160fcf174b594d6", "value": 228458496, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp-2c390033053d841e4d4a01f6", "value": 219807744, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-09e07b3a835bbe68c699da2d", "value": 220135424, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-df220f7a04e395889a52bade", "value": 220200960, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-4160b14c2a09c276efa7aaa1", "value": 239599616, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-980686729b803be8e7756799", "value": 198377472, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp-1eea06b55aadb32253917877", "value": 365428736, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp-eb882b6b1e0f49e246727dac", "value": 344457216, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "exp-4a9a281550bbc99cb36219fa", "value": 325582848, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp-f05b41cdf70db3efa4750d3a", "value": 365953024, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "exp-822141798a849c6bba6945ca", "value": 266207232, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp-882fe31f4736dfb7e6f8a1de", "value": 209321984, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp-3d07c749f5f37879d5b0b75b", "value": 364118016, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp-79cdf8985bb7f16aa6415085", "value": 346030080, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp-e0c01b194f4420857840a94e", "value": 327680000, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp-ab29bde98be72e8d49c2d665", "value": 362020864, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "exp-cf1b7c217d223bde1a1b7197", "value": 346292224, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp-da7a5cbd4285360d8e2dc066", "value": 346554368, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "exp-1edacfe8258b7c59a73c3190", "value": 187695104, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp-6d5c70d58ff4a9c9c9724ddd", "value": 172097536, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp-9643dc25ab9e71ca43e4f54c", "value": 227278848, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp-e078a57cc0086d124891d166", "value": 189399040, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp-22cc1f3c36008c733ffe63bb", "value": 171835392, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp-33fd2f433fb99074b68e9c1c", "value": 190971904, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "exp-0ca4e56fb508f7a2e42041f5", "value": 347602944, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "exp-61b03626bdde762f1af47eaf", "value": 328204288, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "exp-514fbdff581cba4ac98c6d24", "value": 229244928, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp-112f7ca77932d9e48eece097", "value": 210239488, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp-aa464e0cd3fddcbe7f6c4658", "value": 191102976, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "exp-47194ab36c1184fcb1823240", "value": 190971904, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp-6a099c7d26a39086fcbfa1a8", "value": 191234048, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "exp-3d4e7fd673452096ce08d13a", "value": 365428736, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "exp-7b40b271ad813658e44a1aaf", "value": 271974400, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "exp-a465060c9ad441410f885902", "value": 226951168, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "exp-268e4ff2280fb2cd0ae9367b", "value": 209453056, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp-74c2824e9c0bcd03339fb0e6", "value": 193953792, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "exp-3d749c69f9bf62f779ba3f30", "value": 193331200, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "exp-eb95cb8a12d95ec201f2ef08", "value": 241041408, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "exp-cdcfeb3e34a69cce861df38d", "value": 205914112, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "exp-82ee22a90cbfb729e9627091", "value": 196214784, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "exp-63e102f38518f2159ad7f9c1", "value": 194969600, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "exp-80e3d1ce092ddd5eb3b0ee04", "value": 195297280, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "exp-ba50cd00d0da6412f2cc2fe6", "value": 177864704, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "exp-135a1f1024244456579bee9d", "value": 193855488, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "exp-86baf23fd0d602223a6c57e5", "value": 199688192, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "exp-b56d6087ea57be6c3d3ba749", "value": 177602560, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "exp-814de617fb5f350dcbea7743", "value": 216006656, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "exp-86652372f93981490fcb63b5", "value": 190709760, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp-1d758301f0051982f6af7aa4", "value": 190709760, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "exp-64123c46d4dad33db6dec36f", "value": 173604864, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "exp-bf397dad4e532b795ee1521c", "value": 173670400, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "exp-5fb59a923d11ab4d28e17320", "value": 173604864, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "exp-0a9febf0fda90bf391ec0597", "value": 156172288, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "exp-22fb1e4f4007583455f2ddf7", "value": 243793920, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "exp-daf5ce06e32a8eadff223edd", "value": 204996608, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "exp-38e9e49c3f3a0b903f18e816", "value": 196411392, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "exp-a1e34a3dbe808d62c3acd936", "value": 196804608, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "exp-5190bbbdae6392e042e8c321", "value": 200474624, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "exp-0129a56b7e1de35ef6285ee2", "value": 182321152, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "exp-d377efb4279704e06ffe8acb", "value": 200146944, "is_reference": false }, { "step": 98, "launch_seq": 98, "candidate_id": "exp-7cc17e16ce6f6c40b7e9bee4", "value": 200736768, "is_reference": false }, { "step": 99, "launch_seq": 99, "candidate_id": "exp-e7284fcef87b5b70d98b03c1", "value": 250871808, "is_reference": false }, { "step": 100, "launch_seq": 100, "candidate_id": "exp-a3e2ce58d1fedb6b276e5651", "value": 221118464, "is_reference": false } ], "curve": [ { "step": 1, "value": 393740288 }, { "step": 2, "value": 254017536 }, { "step": 3, "value": 227803136 }, { "step": 4, "value": 189530112 }, { "step": 5, "value": 171966464 }, { "step": 6, "value": 171835392 }, { "step": 7, "value": 165478400 }, { "step": 8, "value": 165412864 }, { "step": 9, "value": 165347328 }, { "step": 10, "value": 165347328 }, { "step": 11, "value": 164888576 }, { "step": 12, "value": 164888576 }, { "step": 13, "value": 164888576 }, { "step": 14, "value": 164888576 }, { "step": 15, "value": 164888576 }, { "step": 16, "value": 164888576 }, { "step": 17, "value": 164888576 }, { "step": 18, "value": 164888576 }, { "step": 19, "value": 164888576 }, { "step": 20, "value": 164888576 }, { "step": 21, "value": 164888576 }, { "step": 22, "value": 164888576 }, { "step": 23, "value": 164888576 }, { "step": 24, "value": 164888576 }, { "step": 25, "value": 164888576 }, { "step": 26, "value": 164888576 }, { "step": 27, "value": 164888576 }, { "step": 28, "value": 164888576 }, { "step": 29, "value": 164888576 }, { "step": 30, "value": 164888576 }, { "step": 31, "value": 164888576 }, { "step": 32, "value": 164888576 }, { "step": 33, "value": 164888576 }, { "step": 34, "value": 164888576 }, { "step": 35, "value": 164888576 }, { "step": 36, "value": 164888576 }, { "step": 37, "value": 164888576 }, { "step": 38, "value": 164888576 }, { "step": 39, "value": 164888576 }, { "step": 40, "value": 164888576 }, { "step": 41, "value": 164888576 }, { "step": 42, "value": 164888576 }, { "step": 43, "value": 164888576 }, { "step": 44, "value": 164888576 }, { "step": 45, "value": 164888576 }, { "step": 46, "value": 164888576 }, { "step": 47, "value": 164888576 }, { "step": 48, "value": 164888576 }, { "step": 49, "value": 164888576 }, { "step": 50, "value": 164888576 }, { "step": 51, "value": 164888576 }, { "step": 52, "value": 164888576 }, { "step": 53, "value": 164888576 }, { "step": 54, "value": 164888576 }, { "step": 55, "value": 164888576 }, { "step": 56, "value": 164888576 }, { "step": 57, "value": 164888576 }, { "step": 58, "value": 164888576 }, { "step": 59, "value": 164888576 }, { "step": 60, "value": 164888576 }, { "step": 61, "value": 164888576 }, { "step": 62, "value": 164888576 }, { "step": 63, "value": 164888576 }, { "step": 64, "value": 164888576 }, { "step": 65, "value": 164888576 }, { "step": 66, "value": 164888576 }, { "step": 67, "value": 164888576 }, { "step": 68, "value": 164888576 }, { "step": 69, "value": 164888576 }, { "step": 70, "value": 164888576 }, { "step": 71, "value": 164888576 }, { "step": 72, "value": 164888576 }, { "step": 73, "value": 164888576 }, { "step": 74, "value": 164888576 }, { "step": 75, "value": 164888576 }, { "step": 76, "value": 164888576 }, { "step": 77, "value": 164888576 }, { "step": 78, "value": 164888576 }, { "step": 79, "value": 164888576 }, { "step": 80, "value": 164888576 }, { "step": 81, "value": 164888576 }, { "step": 82, "value": 164888576 }, { "step": 83, "value": 164888576 }, { "step": 84, "value": 164888576 }, { "step": 85, "value": 164888576 }, { "step": 86, "value": 164888576 }, { "step": 87, "value": 164888576 }, { "step": 88, "value": 164888576 }, { "step": 89, "value": 164888576 }, { "step": 90, "value": 156172288 }, { "step": 91, "value": 156172288 }, { "step": 92, "value": 156172288 }, { "step": 93, "value": 156172288 }, { "step": 94, "value": 156172288 }, { "step": 95, "value": 156172288 }, { "step": 96, "value": 156172288 }, { "step": 97, "value": 156172288 }, { "step": 98, "value": 156172288 }, { "step": 99, "value": 156172288 }, { "step": 100, "value": 156172288 } ], "submission_sha256": "2e0dcae34ab14e68258a62beb71a16cfc57adf0e30a13f7b7ee815a20aa18148", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/heuresis-map-elites/data/submission.json", "comparison_value": 156172288, "comparison_rank_key": [ 156172288.0, 2383.0, -0.00014194283152213494 ], "rank": 2 }, { "run_id": "arbor--data", "method_id": "arbor", "task_id": "data", "metric": "tokens_to_target", "unit": "tokens", "direction": "minimize", "best_value": 159776768, "reference_value": 382730240, "relative_improvement": 0.5825342465753425, "rank_key": [ 159776768.0, 2438.0, -0.0004458509263525201 ], "best_launch_seq": 43, "best_candidate_id": "exp-52d344f851216b2f3e46751d", "best_is_reference": false, "budget": { "spent": 44, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 8.401982222222221, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 382730240, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-246d15e14b27eb91bfd1b6d2", "value": 291241984, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-228cfbbcff2585e9970fb718", "value": 233570304, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-20a517a80caf33c9c21be7a2", "value": 281411584, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-c0f0653859497ef1f79c5300", "value": 195428352, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-42be607f9f4bef1732f1e04d", "value": 234487808, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp-f9a1d320d586f6ceb87f5842", "value": 233439232, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp-40f253f171ef96747a47dfa5", "value": 233177088, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-c48f73975449c4ce284ec6ac", "value": 332005376, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-22913be749335f5f0fc5205e", "value": 196083712, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-16cbdba04b8d3f6c1819dda2", "value": 215220224, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-e45e021e2f390fcea959dedb", "value": 195952640, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-348bf0f3c9d1c1036f1dc8d6", "value": 351272960, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-dc9f69fd944ea7ca2fe7d079", "value": 196083712, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-5384584cd1752852554363fd", "value": 195690496, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-5ca557cf86dfd43c49893ad4", "value": 195821568, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-023d45ea6e5aaddbc0431c53", "value": 176553984, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-7589774e7ec69e7e98417fea", "value": 215220224, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp-315a0fa3c285d16f03865b68", "value": 250478592, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp-b13861665c3d4e5c2cef9b3c", "value": 176553984, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp-f6180778c8dd6ca2b5acc43f", "value": 195559424, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp-ace8f4cad0fbcf04f467fe17", "value": 176291840, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-aeef4c065d6e1659a1c09ca0", "value": 195559424, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp-92eee51c7c2dd056013c4117", "value": 169410560, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp-3b558cf6bf6edbfa632c8b5c", "value": 195559424, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp-f8efc1a9f1ba93b66f67b29d", "value": 195690496, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp-d856838e99bb6fe02ad00069", "value": 188416000, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp-679ba9db962a950914ee3b78", "value": 198770688, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp-fb24794b2bbb3e84e6e40386", "value": 196935680, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-2ceac64cc9d81da4fb38360e", "value": 188547072, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-7f5ff24ceb3da690ae5404d6", "value": 186712064, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-203572dbea5fb696d7e06fc8", "value": 203161600, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-31ea0c68d6892bc372d049a1", "value": 186056704, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp-9619002d53141e57555d6d3f", "value": 171966464, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp-308ddbbc4bf0d14efddf53ef", "value": 164823040, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp-6e39ef56121afcdd51bc1017", "value": 173342720, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "exp-8c8eed6580b872b5fe59b154", "value": 172490752, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp-b43dcebbf70451dadee29673", "value": 162267136, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "exp-5f6625a9d19258b4a6cfa73e", "value": 196509696, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp-52d344f851216b2f3e46751d", "value": 159776768, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp-3c2b67086b3356f8fe0e4419", "value": 162398208, "is_reference": false } ], "curve": [ { "step": 1, "value": 382730240 }, { "step": 2, "value": 291241984 }, { "step": 3, "value": 291241984 }, { "step": 4, "value": 233570304 }, { "step": 5, "value": 233570304 }, { "step": 6, "value": 195428352 }, { "step": 7, "value": 195428352 }, { "step": 8, "value": 195428352 }, { "step": 9, "value": 195428352 }, { "step": 10, "value": 195428352 }, { "step": 11, "value": 195428352 }, { "step": 12, "value": 195428352 }, { "step": 13, "value": 195428352 }, { "step": 14, "value": 195428352 }, { "step": 15, "value": 195428352 }, { "step": 16, "value": 195428352 }, { "step": 17, "value": 195428352 }, { "step": 18, "value": 195428352 }, { "step": 19, "value": 195428352 }, { "step": 20, "value": 176553984 }, { "step": 21, "value": 176553984 }, { "step": 22, "value": 176553984 }, { "step": 23, "value": 176553984 }, { "step": 24, "value": 176553984 }, { "step": 25, "value": 176291840 }, { "step": 26, "value": 176291840 }, { "step": 27, "value": 169410560 }, { "step": 28, "value": 169410560 }, { "step": 29, "value": 169410560 }, { "step": 30, "value": 169410560 }, { "step": 31, "value": 169410560 }, { "step": 32, "value": 169410560 }, { "step": 33, "value": 169410560 }, { "step": 34, "value": 169410560 }, { "step": 35, "value": 169410560 }, { "step": 36, "value": 169410560 }, { "step": 37, "value": 169410560 }, { "step": 38, "value": 164823040 }, { "step": 39, "value": 164823040 }, { "step": 40, "value": 164823040 }, { "step": 41, "value": 162267136 }, { "step": 42, "value": 162267136 }, { "step": 43, "value": 159776768 }, { "step": 44, "value": 159776768 } ], "submission_sha256": "12f4bcb64fd086034c1357814f59312fce8a2e81ffc428ef00bcf1ef9987a789", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/arbor/data/submission.json", "comparison_value": 159776768, "comparison_rank_key": [ 159776768.0, 2438.0, -0.0004458509263525201 ], "rank": 3 }, { "run_id": "gear--data", "method_id": "gear", "task_id": "data", "metric": "tokens_to_target", "unit": "tokens", "direction": "minimize", "best_value": 164429824, "reference_value": 394788864, "relative_improvement": 0.5834993359893759, "rank_key": [ 164429824.0, 2509.0, -0.0008193790896249276 ], "best_launch_seq": 96, "best_candidate_id": "exp0096", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 17.10831083333333, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 394788864, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp0002", "value": 274464768, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp0003", "value": 286130176, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp0005", "value": 395313152, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp0006", "value": 254803968, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp0007", "value": 286261248, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp0008", "value": 290783232, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp0009", "value": 215744512, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp0010", "value": 229244928, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp0011", "value": 248250368, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp0012", "value": 232980480, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp0013", "value": 210370560, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp0014", "value": 210370560, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp0015", "value": 252248064, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp0016", "value": 209977344, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp0017", "value": 191365120, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp0018", "value": 230817792, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp0019", "value": 226885632, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp0020", "value": 208273408, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp0021", "value": 230227968, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp0022", "value": 207880192, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp0023", "value": 189399040, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp0024", "value": 189267968, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp0025", "value": 232521728, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp0026", "value": 207880192, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp0027", "value": 226623488, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp0028", "value": 207880192, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp0029", "value": 200278016, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp0030", "value": 182124544, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp0032", "value": 201850880, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp0033", "value": 182190080, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp0034", "value": 207749120, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp0035", "value": 183762944, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp0037", "value": 207618048, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp0038", "value": 183762944, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp0039", "value": 181927936, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "exp0040", "value": 189136896, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp0041", "value": 183304192, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "exp0042", "value": 185073664, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp0043", "value": 201457664, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp0044", "value": 183631872, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp0045", "value": 183631872, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp0047", "value": 183762944, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp0048", "value": 184156160, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "exp0049", "value": 183828480, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp0050", "value": 183828480, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp0051", "value": 183762944, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp0052", "value": 184221696, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "exp0053", "value": 184352768, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp0054", "value": 183959552, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp0055", "value": 184614912, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp0056", "value": 165871616, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp0057", "value": 184811520, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp0058", "value": 165675008, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "exp0059", "value": 184090624, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "exp0060", "value": 184745984, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "exp0061", "value": 184352768, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "exp0062", "value": 166789120, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp0063", "value": 184877056, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp0064", "value": 166133760, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "exp0065", "value": 167051264, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp0066", "value": 165871616, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "exp0067", "value": 166264832, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "exp0068", "value": 165675008, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "exp0069", "value": 165543936, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "exp0070", "value": 184352768, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp0071", "value": 183500800, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "exp0072", "value": 183435264, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "exp0073", "value": 166461440, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "exp0074", "value": 165675008, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "exp0075", "value": 165871616, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "exp0076", "value": 165478400, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "exp0077", "value": 184156160, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "exp0078", "value": 165543936, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "exp0079", "value": 165871616, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "exp0080", "value": 166068224, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "exp0081", "value": 165806080, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "exp0082", "value": 165871616, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "exp0083", "value": 165281792, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "exp0084", "value": 165150720, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "exp0085", "value": 165085184, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp0086", "value": 165347328, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "exp0087", "value": 165216256, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "exp0088", "value": 165347328, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "exp0089", "value": 165412864, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "exp0090", "value": 165478400, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "exp0091", "value": 165478400, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "exp0092", "value": 165085184, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "exp0093", "value": 165216256, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "exp0094", "value": 170131456, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "exp0095", "value": 183566336, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "exp0096", "value": 164429824, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "exp0097", "value": 164626432, "is_reference": false }, { "step": 98, "launch_seq": 98, "candidate_id": "exp0099", "value": 182714368, "is_reference": false }, { "step": 99, "launch_seq": 99, "candidate_id": "exp0100", "value": 164560896, "is_reference": false }, { "step": 100, "launch_seq": 100, "candidate_id": "exp0101", "value": 164954112, "is_reference": false } ], "curve": [ { "step": 1, "value": 394788864 }, { "step": 2, "value": 274464768 }, { "step": 3, "value": 274464768 }, { "step": 4, "value": 274464768 }, { "step": 5, "value": 274464768 }, { "step": 6, "value": 254803968 }, { "step": 7, "value": 254803968 }, { "step": 8, "value": 254803968 }, { "step": 9, "value": 215744512 }, { "step": 10, "value": 215744512 }, { "step": 11, "value": 215744512 }, { "step": 12, "value": 215744512 }, { "step": 13, "value": 210370560 }, { "step": 14, "value": 210370560 }, { "step": 15, "value": 210370560 }, { "step": 16, "value": 209977344 }, { "step": 17, "value": 191365120 }, { "step": 18, "value": 191365120 }, { "step": 19, "value": 191365120 }, { "step": 20, "value": 191365120 }, { "step": 21, "value": 191365120 }, { "step": 22, "value": 191365120 }, { "step": 23, "value": 189399040 }, { "step": 24, "value": 189267968 }, { "step": 25, "value": 189267968 }, { "step": 26, "value": 189267968 }, { "step": 27, "value": 189267968 }, { "step": 28, "value": 189267968 }, { "step": 29, "value": 189267968 }, { "step": 30, "value": 182124544 }, { "step": 31, "value": 182124544 }, { "step": 32, "value": 182124544 }, { "step": 33, "value": 182124544 }, { "step": 34, "value": 182124544 }, { "step": 35, "value": 182124544 }, { "step": 36, "value": 182124544 }, { "step": 37, "value": 182124544 }, { "step": 38, "value": 182124544 }, { "step": 39, "value": 181927936 }, { "step": 40, "value": 181927936 }, { "step": 41, "value": 181927936 }, { "step": 42, "value": 181927936 }, { "step": 43, "value": 181927936 }, { "step": 44, "value": 181927936 }, { "step": 45, "value": 181927936 }, { "step": 46, "value": 181927936 }, { "step": 47, "value": 181927936 }, { "step": 48, "value": 181927936 }, { "step": 49, "value": 181927936 }, { "step": 50, "value": 181927936 }, { "step": 51, "value": 181927936 }, { "step": 52, "value": 181927936 }, { "step": 53, "value": 181927936 }, { "step": 54, "value": 181927936 }, { "step": 55, "value": 181927936 }, { "step": 56, "value": 165871616 }, { "step": 57, "value": 165871616 }, { "step": 58, "value": 165675008 }, { "step": 59, "value": 165675008 }, { "step": 60, "value": 165675008 }, { "step": 61, "value": 165675008 }, { "step": 62, "value": 165675008 }, { "step": 63, "value": 165675008 }, { "step": 64, "value": 165675008 }, { "step": 65, "value": 165675008 }, { "step": 66, "value": 165675008 }, { "step": 67, "value": 165675008 }, { "step": 68, "value": 165675008 }, { "step": 69, "value": 165543936 }, { "step": 70, "value": 165543936 }, { "step": 71, "value": 165543936 }, { "step": 72, "value": 165543936 }, { "step": 73, "value": 165543936 }, { "step": 74, "value": 165543936 }, { "step": 75, "value": 165543936 }, { "step": 76, "value": 165478400 }, { "step": 77, "value": 165478400 }, { "step": 78, "value": 165478400 }, { "step": 79, "value": 165478400 }, { "step": 80, "value": 165478400 }, { "step": 81, "value": 165478400 }, { "step": 82, "value": 165478400 }, { "step": 83, "value": 165281792 }, { "step": 84, "value": 165150720 }, { "step": 85, "value": 165085184 }, { "step": 86, "value": 165085184 }, { "step": 87, "value": 165085184 }, { "step": 88, "value": 165085184 }, { "step": 89, "value": 165085184 }, { "step": 90, "value": 165085184 }, { "step": 91, "value": 165085184 }, { "step": 92, "value": 165085184 }, { "step": 93, "value": 165085184 }, { "step": 94, "value": 165085184 }, { "step": 95, "value": 165085184 }, { "step": 96, "value": 164429824 }, { "step": 97, "value": 164429824 }, { "step": 98, "value": 164429824 }, { "step": 99, "value": 164429824 }, { "step": 100, "value": 164429824 } ], "submission_sha256": "9e881c14a563ea1ec7bb68b453f9e7c373fdfc8daf9eca5f47e08331dc67098d", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/gear/data/submission.json", "comparison_value": 164429824, "comparison_rank_key": [ 164429824.0, 2509.0, -0.0008193790896249276 ], "rank": 4 }, { "run_id": "autoscientists--data", "method_id": "autoscientists", "task_id": "data", "metric": "tokens_to_target", "unit": "tokens", "direction": "minimize", "best_value": 184549376, "reference_value": 392167424, "relative_improvement": 0.5294117647058824, "rank_key": [ 184549376.0, 704.0, -3.9602161573837336e-05 ], "best_launch_seq": 48, "best_candidate_id": "exp-86d703393ca3318c61c79681", "best_is_reference": false, "budget": { "spent": 85, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 2, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 2 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 14.627287222222225, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 392167424, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-b97072adb1ea65d63e62b879", "value": 291241984, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp-72663189f6860e0a787f4b00", "value": 284295168, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-baaa1ad9d99bb6e4a3fa06b2", "value": 233570304, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-adace511f8237ce708393623", "value": 291504128, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-4a7f09c1a39bbc549f027289", "value": 214171648, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-32afa3a64c1048c6e88c81e2", "value": 233570304, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp-e6b05812463ff99f700c2ba7", "value": 195035136, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp-a3c4a6a03955e7c34b58b102", "value": 214433792, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-1add337f45012af494a2b46b", "value": 215220224, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-79d81dfd2816ca0b1713df05", "value": 195297280, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-90f02b1f0ed373d4e5401e14", "value": 215482368, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-1c54f5a6c4d6eac39c83803e", "value": 214695936, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-4c4fe3ab8561cb307e034d62", "value": 273154048, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-005b90f60be9afd4031dfbcd", "value": 215220224, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp-55879a91b2ced2287dcaf5de", "value": 194772992, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-e89972891468e8319842f3c5", "value": 196870144, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-cc1fc4a5cd3d6211cda245c2", "value": 218103808, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-20dff7511d0c4f5ec9ccd3ed", "value": 207355904, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-1672692dddb8c5f996fe6c2b", "value": 198705152, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-280c3c3d6a82dbd8357da56b", "value": 196608000, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp-09e9c1bd3ce6964365d3c243", "value": 198443008, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp-d0cfbccbae2a5bd504b9895d", "value": 215744512, "is_reference": false }, { "step": 25, "launch_seq": 26, "candidate_id": "exp-281f5e73286752b6634c312f", "value": 192413696, "is_reference": false }, { "step": 26, "launch_seq": 27, "candidate_id": "exp-d20f096c020125daed10719a", "value": 192937984, "is_reference": false }, { "step": 27, "launch_seq": 28, "candidate_id": "exp-08e208b2d7c160b84966078a", "value": 193462272, "is_reference": false }, { "step": 28, "launch_seq": 29, "candidate_id": "exp-0f58dceb1cdab650d49772d7", "value": 193462272, "is_reference": false }, { "step": 29, "launch_seq": 30, "candidate_id": "exp-ee5fd079e8e23669d9c275b5", "value": 211812352, "is_reference": false }, { "step": 30, "launch_seq": 31, "candidate_id": "exp-a59427c2295101a47e294d1a", "value": 188219392, "is_reference": false }, { "step": 31, "launch_seq": 32, "candidate_id": "exp-b5d0b741367a420a5995bc1e", "value": 187695104, "is_reference": false }, { "step": 32, "launch_seq": 33, "candidate_id": "exp-64556cd3e8cc69b0b93db88d", "value": 188481536, "is_reference": false }, { "step": 33, "launch_seq": 35, "candidate_id": "exp-0714c380eb70bb8983f323b6", "value": 206831616, "is_reference": false }, { "step": 34, "launch_seq": 36, "candidate_id": "exp-b3db621e3eca682d4add5272", "value": 187957248, "is_reference": false }, { "step": 35, "launch_seq": 37, "candidate_id": "exp-37fb5bd81856a7523156c920", "value": 188219392, "is_reference": false }, { "step": 36, "launch_seq": 38, "candidate_id": "exp-bf729eada4a3ac210c4192b1", "value": 189267968, "is_reference": false }, { "step": 37, "launch_seq": 39, "candidate_id": "exp-6f8388c03c176d06eeaca477", "value": 186908672, "is_reference": false }, { "step": 38, "launch_seq": 40, "candidate_id": "exp-25401b8421575a23936c7893", "value": 201588736, "is_reference": false }, { "step": 39, "launch_seq": 41, "candidate_id": "exp-1e73924d42193d7293eeb715", "value": 186908672, "is_reference": false }, { "step": 40, "launch_seq": 42, "candidate_id": "exp-3aed7cacd29259a0977ec6cc", "value": 185597952, "is_reference": false }, { "step": 41, "launch_seq": 43, "candidate_id": "exp-cd93a0530b4dba6884fe347d", "value": 203685888, "is_reference": false }, { "step": 42, "launch_seq": 44, "candidate_id": "exp-3c77e1190ac0deae24aa8c9a", "value": 204472320, "is_reference": false }, { "step": 44, "launch_seq": 46, "candidate_id": "exp-0467a32092094c388c7beec1", "value": 207093760, "is_reference": false }, { "step": 45, "launch_seq": 47, "candidate_id": "exp-17d10807a58c7c83a1ac55e7", "value": 187695104, "is_reference": false }, { "step": 46, "launch_seq": 48, "candidate_id": "exp-86d703393ca3318c61c79681", "value": 184549376, "is_reference": false }, { "step": 47, "launch_seq": 49, "candidate_id": "exp-9dc33daa638e0ad8ac1a808f", "value": 189005824, "is_reference": false }, { "step": 48, "launch_seq": 50, "candidate_id": "exp-36d8e1896b209a1cb2c1d780", "value": 189267968, "is_reference": false }, { "step": 49, "launch_seq": 51, "candidate_id": "exp-41ffe61f223c1e767a814d0d", "value": 187695104, "is_reference": false }, { "step": 50, "launch_seq": 52, "candidate_id": "exp-198edc5437ecb887f3c3d9ee", "value": 188743680, "is_reference": false }, { "step": 51, "launch_seq": 53, "candidate_id": "exp-986709957f1fc580b4c8301e", "value": 185073664, "is_reference": false }, { "step": 52, "launch_seq": 54, "candidate_id": "exp-cfe4b689ddfd9d3c8aec0041", "value": 188219392, "is_reference": false }, { "step": 53, "launch_seq": 55, "candidate_id": "exp-4d99e07c636ce7060f0471a2", "value": 187170816, "is_reference": false }, { "step": 54, "launch_seq": 56, "candidate_id": "exp-aa095a74e70e1e6f10c36982", "value": 187695104, "is_reference": false }, { "step": 55, "launch_seq": 57, "candidate_id": "exp-749d396ce217edcd347c5aed", "value": 185597952, "is_reference": false }, { "step": 56, "launch_seq": 58, "candidate_id": "exp-b3075fb1cf1e893a1cf7e867", "value": 184811520, "is_reference": false }, { "step": 57, "launch_seq": 59, "candidate_id": "exp-7d64bdec66ed052dfac255a4", "value": 187432960, "is_reference": false }, { "step": 58, "launch_seq": 60, "candidate_id": "exp-e35a8d22216173ee746d0b68", "value": 185335808, "is_reference": false }, { "step": 59, "launch_seq": 61, "candidate_id": "exp-d58734e4fc5b395eeba16e4a", "value": 203161600, "is_reference": false }, { "step": 60, "launch_seq": 62, "candidate_id": "exp-4a0a85e11fc0cfc47c128d4a", "value": 186384384, "is_reference": false }, { "step": 61, "launch_seq": 63, "candidate_id": "exp-5633780c3b45c2ce8a02f845", "value": 186908672, "is_reference": false }, { "step": 62, "launch_seq": 64, "candidate_id": "exp-89c493dc98725e626483d3a4", "value": 190054400, "is_reference": false }, { "step": 63, "launch_seq": 65, "candidate_id": "exp-a6ef0fd102a3c4e7805dd3f0", "value": 189792256, "is_reference": false }, { "step": 64, "launch_seq": 66, "candidate_id": "exp-3796ccffb74b395722f40cd1", "value": 189792256, "is_reference": false }, { "step": 65, "launch_seq": 67, "candidate_id": "exp-02727ec14b0b647f2e8bc282", "value": 203948032, "is_reference": false }, { "step": 69, "launch_seq": 71, "candidate_id": "exp-5130da970b85065c1efb0f6c", "value": 184811520, "is_reference": false }, { "step": 71, "launch_seq": 73, "candidate_id": "exp-c870f7963b180c86e2e8f770", "value": 202899456, "is_reference": false }, { "step": 73, "launch_seq": 75, "candidate_id": "exp-d4c9aefc15a65ea73821b9f9", "value": 201064448, "is_reference": false }, { "step": 74, "launch_seq": 76, "candidate_id": "exp-62bcb767c1023d968df6d328", "value": 201326592, "is_reference": false } ], "curve": [ { "step": 1, "value": 392167424 }, { "step": 2, "value": 291241984 }, { "step": 3, "value": 284295168 }, { "step": 4, "value": 233570304 }, { "step": 5, "value": 233570304 }, { "step": 6, "value": 214171648 }, { "step": 7, "value": 214171648 }, { "step": 8, "value": 195035136 }, { "step": 9, "value": 195035136 }, { "step": 10, "value": 195035136 }, { "step": 11, "value": 195035136 }, { "step": 12, "value": 195035136 }, { "step": 13, "value": 195035136 }, { "step": 14, "value": 195035136 }, { "step": 15, "value": 195035136 }, { "step": 16, "value": 194772992 }, { "step": 17, "value": 194772992 }, { "step": 18, "value": 194772992 }, { "step": 19, "value": 194772992 }, { "step": 20, "value": 194772992 }, { "step": 21, "value": 194772992 }, { "step": 22, "value": 194772992 }, { "step": 23, "value": 194772992 }, { "step": 24, "value": 194772992 }, { "step": 25, "value": 192413696 }, { "step": 26, "value": 192413696 }, { "step": 27, "value": 192413696 }, { "step": 28, "value": 192413696 }, { "step": 29, "value": 192413696 }, { "step": 30, "value": 188219392 }, { "step": 31, "value": 187695104 }, { "step": 32, "value": 187695104 }, { "step": 33, "value": 187695104 }, { "step": 34, "value": 187695104 }, { "step": 35, "value": 187695104 }, { "step": 36, "value": 187695104 }, { "step": 37, "value": 186908672 }, { "step": 38, "value": 186908672 }, { "step": 39, "value": 186908672 }, { "step": 40, "value": 185597952 }, { "step": 41, "value": 185597952 }, { "step": 42, "value": 185597952 }, { "step": 43, "value": 185597952 }, { "step": 44, "value": 185597952 }, { "step": 45, "value": 185597952 }, { "step": 46, "value": 184549376 }, { "step": 47, "value": 184549376 }, { "step": 48, "value": 184549376 }, { "step": 49, "value": 184549376 }, { "step": 50, "value": 184549376 }, { "step": 51, "value": 184549376 }, { "step": 52, "value": 184549376 }, { "step": 53, "value": 184549376 }, { "step": 54, "value": 184549376 }, { "step": 55, "value": 184549376 }, { "step": 56, "value": 184549376 }, { "step": 57, "value": 184549376 }, { "step": 58, "value": 184549376 }, { "step": 59, "value": 184549376 }, { "step": 60, "value": 184549376 }, { "step": 61, "value": 184549376 }, { "step": 62, "value": 184549376 }, { "step": 63, "value": 184549376 }, { "step": 64, "value": 184549376 }, { "step": 65, "value": 184549376 }, { "step": 66, "value": 184549376 }, { "step": 67, "value": 184549376 }, { "step": 68, "value": 184549376 }, { "step": 69, "value": 184549376 }, { "step": 70, "value": 184549376 }, { "step": 71, "value": 184549376 }, { "step": 72, "value": 184549376 }, { "step": 73, "value": 184549376 }, { "step": 74, "value": 184549376 }, { "step": 75, "value": 184549376 }, { "step": 76, "value": 184549376 }, { "step": 77, "value": 184549376 }, { "step": 78, "value": 184549376 }, { "step": 79, "value": 184549376 }, { "step": 80, "value": 184549376 }, { "step": 81, "value": 184549376 }, { "step": 82, "value": 184549376 }, { "step": 83, "value": 184549376 }, { "step": 84, "value": 184549376 }, { "step": 85, "value": 184549376 } ], "submission_sha256": "c548673476fb81b054cb38cf38686d8598969020ddb0aeab1af63e113dae7c04", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/autoscientists/data/submission.json", "comparison_value": 184549376, "comparison_rank_key": [ 184549376.0, 704.0, -3.9602161573837336e-05 ], "rank": 5 }, { "run_id": "beam-search--data", "method_id": "beam-search", "task_id": "data", "metric": "tokens_to_target", "unit": "tokens", "direction": "minimize", "best_value": 197656576, "reference_value": 400556032, "relative_improvement": 0.506544502617801, "rank_key": [ 197656576.0, 754.0, -0.00017593178897290507 ], "best_launch_seq": 95, "best_candidate_id": "g14-probe10-adam-moments-fp32-fixed", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 4, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 4 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 20.536948611111107, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 400556032, "is_reference": true }, { "step": 3, "launch_seq": 3, "candidate_id": "g1-update-granularity", "value": 299892736, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "g2b-anneal-horizon-tokens", "value": 257687552, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "g2b-window-depth-placement", "value": 372768768, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "g2b-swiglu-mlp-1344", "value": 390070272, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "g3-granularity-131072", "value": 269484032, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "g3-rotary-window-matched", "value": 244580352, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "g3-span-two-medium", "value": 374865920, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "g3-window-depth1-fine", "value": 257949696, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "g3-perchannel-resid-bf16", "value": 289406976, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "g3-no-logit-softcap-slsssssl", "value": 401604608, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "g4-rotary-cut-tabled", "value": 296222720, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "g4-rotary-cut-token-anneal", "value": 271581184, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "g4-long-window-depth0", "value": 255328256, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "g4-stride4-depth-skips", "value": 268828672, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "g4-rotary-cut-perlayer", "value": 294125568, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "g4-anneal-decay-absolute", "value": 252444672, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "g4-ve-second-half", "value": 258736128, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "g4-ve-all-depths-gqa2", "value": 282591232, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "g5-muon-shape-lr-symmetric", "value": 245628928, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "g5-anneal-plateau-quarter", "value": 251920384, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "g5-window-ramp-256", "value": 254541824, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "g5-normuon-off", "value": 257687552, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "g5-clock-anneal-matrices-only", "value": 259260416, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "g5-weight-decay-unramped", "value": 257425408, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "g5-ve-gate-fullwidth", "value": 271581184, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "g5-ve-gate-adamw", "value": 272891904, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "g6b-anneal-horizon-crossing-floor", "value": 234618880, "is_reference": false }, { "step": 37, "launch_seq": 38, "candidate_id": "g7-token-anneal-crossing-g3", "value": 215220224, "is_reference": false }, { "step": 38, "launch_seq": 39, "candidate_id": "g7-wd-ramp-plateau-clock", "value": 231997440, "is_reference": false }, { "step": 39, "launch_seq": 40, "candidate_id": "g7-muon-symmetric-token-frame", "value": 236453888, "is_reference": false }, { "step": 40, "launch_seq": 41, "candidate_id": "g7-anneal-compress-nofloor", "value": 212074496, "is_reference": false }, { "step": 41, "launch_seq": 42, "candidate_id": "g7-muon-width-scale-g3", "value": 243007488, "is_reference": false }, { "step": 42, "launch_seq": 43, "candidate_id": "g7-muon-momentum-300-updates", "value": 248512512, "is_reference": false }, { "step": 43, "launch_seq": 44, "candidate_id": "g7-anneal-leg-shift-1024", "value": 234356736, "is_reference": false }, { "step": 44, "launch_seq": 45, "candidate_id": "g7-microbatch-split-64-decay", "value": 291241984, "is_reference": false }, { "step": 45, "launch_seq": 46, "candidate_id": "g8-wd-ramp-half-horizon", "value": 215220224, "is_reference": false }, { "step": 46, "launch_seq": 47, "candidate_id": "g8-anneal-compress-821", "value": 217055232, "is_reference": false }, { "step": 47, "launch_seq": 48, "candidate_id": "g8-clock-no-plateau-rate-only", "value": 218103808, "is_reference": false }, { "step": 48, "launch_seq": 49, "candidate_id": "g8-momentum-ramp-300-updates", "value": 212598784, "is_reference": false }, { "step": 49, "launch_seq": 50, "candidate_id": "g8-muon-symmetric-mlp-only", "value": 212598784, "is_reference": false }, { "step": 50, "launch_seq": 51, "candidate_id": "g8-anneal-floor-binding-763", "value": 215482368, "is_reference": false }, { "step": 51, "launch_seq": 52, "candidate_id": "g8-clock-horizon-480-rate", "value": 217055232, "is_reference": false }, { "step": 52, "launch_seq": 53, "candidate_id": "g8-muon-symmetric-vegate-only", "value": 217841664, "is_reference": false }, { "step": 53, "launch_seq": 54, "candidate_id": "g9-momentum-ramp-leader", "value": 215744512, "is_reference": false }, { "step": 54, "launch_seq": 55, "candidate_id": "g9-momentum-window-token-matched", "value": 217317376, "is_reference": false }, { "step": 55, "launch_seq": 56, "candidate_id": "g9-muon-symmetric-full", "value": 218365952, "is_reference": false }, { "step": 56, "launch_seq": 57, "candidate_id": "g9-clock-rate-horizon-360", "value": 213385216, "is_reference": false }, { "step": 57, "launch_seq": 58, "candidate_id": "g9-momentum-wd-stack", "value": 218365952, "is_reference": false }, { "step": 58, "launch_seq": 59, "candidate_id": "g9-muon-polar-fp32", "value": 215482368, "is_reference": false }, { "step": 59, "launch_seq": 60, "candidate_id": "g9-momentum-ramp-symmetric-mlp", "value": 215482368, "is_reference": false }, { "step": 60, "launch_seq": 61, "candidate_id": "g9-token-rate-horizon-821", "value": 215482368, "is_reference": false }, { "step": 62, "launch_seq": 63, "candidate_id": "g10-rate-horizon-crossing-811", "value": 217055232, "is_reference": false }, { "step": 63, "launch_seq": 64, "candidate_id": "g10-wd-ramp-480-symmetric-mlp", "value": 218103808, "is_reference": false }, { "step": 64, "launch_seq": 65, "candidate_id": "g10-momentum-ramp-clock-360", "value": 213909504, "is_reference": false }, { "step": 65, "launch_seq": 66, "candidate_id": "g10-leader-stack-adamw-horizon", "value": 218628096, "is_reference": false }, { "step": 66, "launch_seq": 67, "candidate_id": "g10-momentum-window-doubled", "value": 217317376, "is_reference": false }, { "step": 67, "launch_seq": 68, "candidate_id": "g10-wd-ramp-1280-symmetric-mlp", "value": 216006656, "is_reference": false }, { "step": 68, "launch_seq": 69, "candidate_id": "g10-token-horizon-frame-matched-878", "value": 215744512, "is_reference": false }, { "step": 69, "launch_seq": 70, "candidate_id": "g11-momentum-ramp-150-leader", "value": 215744512, "is_reference": false }, { "step": 70, "launch_seq": 71, "candidate_id": "g11-rate-tail-probe10-738", "value": 217055232, "is_reference": false }, { "step": 71, "launch_seq": 72, "candidate_id": "g11-momentum-wd-stack-symmetric-mlp", "value": 218628096, "is_reference": false }, { "step": 72, "launch_seq": 73, "candidate_id": "g11-clock-tail-probe10-300", "value": 216530944, "is_reference": false }, { "step": 73, "launch_seq": 74, "candidate_id": "g11-embedding-horizon-809-leader", "value": 219152384, "is_reference": false }, { "step": 74, "launch_seq": 75, "candidate_id": "g11-wd-480-decoupled-momentum", "value": 217579520, "is_reference": false }, { "step": 75, "launch_seq": 76, "candidate_id": "g11-matrix-weight-average-symmetric-mlp", "value": 273416192, "is_reference": false }, { "step": 76, "launch_seq": 77, "candidate_id": "g11-wd-ramp-240-clock", "value": 214695936, "is_reference": false }, { "step": 77, "launch_seq": 78, "candidate_id": "g12-write-init-fanin-leader", "value": 215744512, "is_reference": false }, { "step": 78, "launch_seq": 79, "candidate_id": "g12-probe10-stack-symmetric-mlp", "value": 217055232, "is_reference": false }, { "step": 79, "launch_seq": 80, "candidate_id": "g12-mlp-write-init-symmetric", "value": 218628096, "is_reference": false }, { "step": 80, "launch_seq": 81, "candidate_id": "g12-clock-ramp300-write-init", "value": 216530944, "is_reference": false }, { "step": 81, "launch_seq": 82, "candidate_id": "g12-attn-write-init-leader", "value": 217055232, "is_reference": false }, { "step": 82, "launch_seq": 83, "candidate_id": "g12-probe10-stack-write-init", "value": 219152384, "is_reference": false }, { "step": 83, "launch_seq": 84, "candidate_id": "g12-muon-fanout-factor-neutral", "value": 234618880, "is_reference": false }, { "step": 87, "launch_seq": 88, "candidate_id": "g13-attn-write-both-symmetric", "value": 218628096, "is_reference": false }, { "step": 88, "launch_seq": 89, "candidate_id": "g13-lm-head-muon-clock", "value": 216268800, "is_reference": false }, { "step": 89, "launch_seq": 90, "candidate_id": "g13-attn-write-rate-half-leader", "value": 215482368, "is_reference": false }, { "step": 90, "launch_seq": 91, "candidate_id": "g13-probe10-attn-write-rate-half", "value": 217055232, "is_reference": false }, { "step": 91, "launch_seq": 92, "candidate_id": "g13-contract-tables-symmetric", "value": 236191744, "is_reference": false }, { "step": 93, "launch_seq": 94, "candidate_id": "g14-adam-moments-fp32-leader-fixed", "value": 215744512, "is_reference": false }, { "step": 94, "launch_seq": 95, "candidate_id": "g14-probe10-adam-moments-fp32-fixed", "value": 197656576, "is_reference": false }, { "step": 95, "launch_seq": 96, "candidate_id": "g14-rotary-window-base-symmetric", "value": 238288896, "is_reference": false }, { "step": 97, "launch_seq": 98, "candidate_id": "g14-softcap-10-leader", "value": 216793088, "is_reference": false }, { "step": 98, "launch_seq": 99, "candidate_id": "g14-probe10-softcap-10", "value": 215220224, "is_reference": false }, { "step": 99, "launch_seq": 100, "candidate_id": "g14-x0-rate-matched-symmetric", "value": 218890240, "is_reference": false }, { "step": 100, "launch_seq": 101, "candidate_id": "g14-clock-ramp300-x0-rate", "value": 214171648, "is_reference": false } ], "curve": [ { "step": 1, "value": 400556032 }, { "step": 2, "value": 400556032 }, { "step": 3, "value": 299892736 }, { "step": 4, "value": 257687552 }, { "step": 5, "value": 257687552 }, { "step": 6, "value": 257687552 }, { "step": 7, "value": 257687552 }, { "step": 8, "value": 257687552 }, { "step": 9, "value": 244580352 }, { "step": 10, "value": 244580352 }, { "step": 11, "value": 244580352 }, { "step": 12, "value": 244580352 }, { "step": 13, "value": 244580352 }, { "step": 14, "value": 244580352 }, { "step": 15, "value": 244580352 }, { "step": 16, "value": 244580352 }, { "step": 17, "value": 244580352 }, { "step": 18, "value": 244580352 }, { "step": 19, "value": 244580352 }, { "step": 20, "value": 244580352 }, { "step": 21, "value": 244580352 }, { "step": 22, "value": 244580352 }, { "step": 23, "value": 244580352 }, { "step": 24, "value": 244580352 }, { "step": 25, "value": 244580352 }, { "step": 26, "value": 244580352 }, { "step": 27, "value": 244580352 }, { "step": 28, "value": 244580352 }, { "step": 29, "value": 244580352 }, { "step": 30, "value": 244580352 }, { "step": 31, "value": 244580352 }, { "step": 32, "value": 244580352 }, { "step": 33, "value": 244580352 }, { "step": 34, "value": 244580352 }, { "step": 35, "value": 234618880 }, { "step": 36, "value": 234618880 }, { "step": 37, "value": 215220224 }, { "step": 38, "value": 215220224 }, { "step": 39, "value": 215220224 }, { "step": 40, "value": 212074496 }, { "step": 41, "value": 212074496 }, { "step": 42, "value": 212074496 }, { "step": 43, "value": 212074496 }, { "step": 44, "value": 212074496 }, { "step": 45, "value": 212074496 }, { "step": 46, "value": 212074496 }, { "step": 47, "value": 212074496 }, { "step": 48, "value": 212074496 }, { "step": 49, "value": 212074496 }, { "step": 50, "value": 212074496 }, { "step": 51, "value": 212074496 }, { "step": 52, "value": 212074496 }, { "step": 53, "value": 212074496 }, { "step": 54, "value": 212074496 }, { "step": 55, "value": 212074496 }, { "step": 56, "value": 212074496 }, { "step": 57, "value": 212074496 }, { "step": 58, "value": 212074496 }, { "step": 59, "value": 212074496 }, { "step": 60, "value": 212074496 }, { "step": 61, "value": 212074496 }, { "step": 62, "value": 212074496 }, { "step": 63, "value": 212074496 }, { "step": 64, "value": 212074496 }, { "step": 65, "value": 212074496 }, { "step": 66, "value": 212074496 }, { "step": 67, "value": 212074496 }, { "step": 68, "value": 212074496 }, { "step": 69, "value": 212074496 }, { "step": 70, "value": 212074496 }, { "step": 71, "value": 212074496 }, { "step": 72, "value": 212074496 }, { "step": 73, "value": 212074496 }, { "step": 74, "value": 212074496 }, { "step": 75, "value": 212074496 }, { "step": 76, "value": 212074496 }, { "step": 77, "value": 212074496 }, { "step": 78, "value": 212074496 }, { "step": 79, "value": 212074496 }, { "step": 80, "value": 212074496 }, { "step": 81, "value": 212074496 }, { "step": 82, "value": 212074496 }, { "step": 83, "value": 212074496 }, { "step": 84, "value": 212074496 }, { "step": 85, "value": 212074496 }, { "step": 86, "value": 212074496 }, { "step": 87, "value": 212074496 }, { "step": 88, "value": 212074496 }, { "step": 89, "value": 212074496 }, { "step": 90, "value": 212074496 }, { "step": 91, "value": 212074496 }, { "step": 92, "value": 212074496 }, { "step": 93, "value": 212074496 }, { "step": 94, "value": 197656576 }, { "step": 95, "value": 197656576 }, { "step": 96, "value": 197656576 }, { "step": 97, "value": 197656576 }, { "step": 98, "value": 197656576 }, { "step": 99, "value": 197656576 }, { "step": 100, "value": 197656576 } ], "submission_sha256": "e6111285209141c4b2c72e561e07ed74b3d8b2de1e6d970b47f07caedc9c1554", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/beam-search/data/submission.json", "comparison_value": 197656576, "comparison_rank_key": [ 197656576.0, 754.0, -0.00017593178897290507 ], "rank": 6 }, { "run_id": "tpe--data", "method_id": "tpe", "task_id": "data", "metric": "tokens_to_target", "unit": "tokens", "direction": "minimize", "best_value": 387710976, "reference_value": 395313152, "relative_improvement": 0.019230769230769232, "rank_key": [ 387710976.0, 1479.0, -0.004122906395126913 ], "best_launch_seq": 94, "best_candidate_id": "tpe-trial-0093", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 21.781381111111116, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 395313152, "is_reference": true }, { "step": 37, "launch_seq": 37, "candidate_id": "tpe-trial-0036", "value": 647495680, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "tpe-trial-0037", "value": 681312256, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "tpe-trial-0044", "value": 670302208, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "tpe-trial-0045", "value": 644087808, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "tpe-trial-0046", "value": 678952960, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "tpe-trial-0048", "value": 680525824, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "tpe-trial-0051", "value": 677380096, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "tpe-trial-0054", "value": 651689984, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "tpe-trial-0055", "value": 536608768, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "tpe-trial-0061", "value": 683409408, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "tpe-trial-0063", "value": 447741952, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "tpe-trial-0064", "value": 446431232, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "tpe-trial-0065", "value": 444858368, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "tpe-trial-0066", "value": 467140608, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "tpe-trial-0067", "value": 441974784, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "tpe-trial-0070", "value": 417333248, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "tpe-trial-0071", "value": 442236928, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "tpe-trial-0072", "value": 466092032, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "tpe-trial-0073", "value": 409468928, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "tpe-trial-0074", "value": 389021696, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "tpe-trial-0075", "value": 388497408, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "tpe-trial-0077", "value": 388759552, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "tpe-trial-0078", "value": 409993216, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "tpe-trial-0080", "value": 388235264, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "tpe-trial-0081", "value": 387973120, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "tpe-trial-0082", "value": 388497408, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "tpe-trial-0086", "value": 409731072, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "tpe-trial-0089", "value": 388497408, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "tpe-trial-0091", "value": 388235264, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "tpe-trial-0092", "value": 409206784, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "tpe-trial-0093", "value": 387710976, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "tpe-trial-0096", "value": 548405248, "is_reference": false }, { "step": 99, "launch_seq": 99, "candidate_id": "tpe-trial-0098", "value": 491257856, "is_reference": false } ], "curve": [ { "step": 1, "value": 395313152 }, { "step": 2, "value": 395313152 }, { "step": 3, "value": 395313152 }, { "step": 4, "value": 395313152 }, { "step": 5, "value": 395313152 }, { "step": 6, "value": 395313152 }, { "step": 7, "value": 395313152 }, { "step": 8, "value": 395313152 }, { "step": 9, "value": 395313152 }, { "step": 10, "value": 395313152 }, { "step": 11, "value": 395313152 }, { "step": 12, "value": 395313152 }, { "step": 13, "value": 395313152 }, { "step": 14, "value": 395313152 }, { "step": 15, "value": 395313152 }, { "step": 16, "value": 395313152 }, { "step": 17, "value": 395313152 }, { "step": 18, "value": 395313152 }, { "step": 19, "value": 395313152 }, { "step": 20, "value": 395313152 }, { "step": 21, "value": 395313152 }, { "step": 22, "value": 395313152 }, { "step": 23, "value": 395313152 }, { "step": 24, "value": 395313152 }, { "step": 25, "value": 395313152 }, { "step": 26, "value": 395313152 }, { "step": 27, "value": 395313152 }, { "step": 28, "value": 395313152 }, { "step": 29, "value": 395313152 }, { "step": 30, "value": 395313152 }, { "step": 31, "value": 395313152 }, { "step": 32, "value": 395313152 }, { "step": 33, "value": 395313152 }, { "step": 34, "value": 395313152 }, { "step": 35, "value": 395313152 }, { "step": 36, "value": 395313152 }, { "step": 37, "value": 395313152 }, { "step": 38, "value": 395313152 }, { "step": 39, "value": 395313152 }, { "step": 40, "value": 395313152 }, { "step": 41, "value": 395313152 }, { "step": 42, "value": 395313152 }, { "step": 43, "value": 395313152 }, { "step": 44, "value": 395313152 }, { "step": 45, "value": 395313152 }, { "step": 46, "value": 395313152 }, { "step": 47, "value": 395313152 }, { "step": 48, "value": 395313152 }, { "step": 49, "value": 395313152 }, { "step": 50, "value": 395313152 }, { "step": 51, "value": 395313152 }, { "step": 52, "value": 395313152 }, { "step": 53, "value": 395313152 }, { "step": 54, "value": 395313152 }, { "step": 55, "value": 395313152 }, { "step": 56, "value": 395313152 }, { "step": 57, "value": 395313152 }, { "step": 58, "value": 395313152 }, { "step": 59, "value": 395313152 }, { "step": 60, "value": 395313152 }, { "step": 61, "value": 395313152 }, { "step": 62, "value": 395313152 }, { "step": 63, "value": 395313152 }, { "step": 64, "value": 395313152 }, { "step": 65, "value": 395313152 }, { "step": 66, "value": 395313152 }, { "step": 67, "value": 395313152 }, { "step": 68, "value": 395313152 }, { "step": 69, "value": 395313152 }, { "step": 70, "value": 395313152 }, { "step": 71, "value": 395313152 }, { "step": 72, "value": 395313152 }, { "step": 73, "value": 395313152 }, { "step": 74, "value": 395313152 }, { "step": 75, "value": 389021696 }, { "step": 76, "value": 388497408 }, { "step": 77, "value": 388497408 }, { "step": 78, "value": 388497408 }, { "step": 79, "value": 388497408 }, { "step": 80, "value": 388497408 }, { "step": 81, "value": 388235264 }, { "step": 82, "value": 387973120 }, { "step": 83, "value": 387973120 }, { "step": 84, "value": 387973120 }, { "step": 85, "value": 387973120 }, { "step": 86, "value": 387973120 }, { "step": 87, "value": 387973120 }, { "step": 88, "value": 387973120 }, { "step": 89, "value": 387973120 }, { "step": 90, "value": 387973120 }, { "step": 91, "value": 387973120 }, { "step": 92, "value": 387973120 }, { "step": 93, "value": 387973120 }, { "step": 94, "value": 387710976 }, { "step": 95, "value": 387710976 }, { "step": 96, "value": 387710976 }, { "step": 97, "value": 387710976 }, { "step": 98, "value": 387710976 }, { "step": 99, "value": 387710976 }, { "step": 100, "value": 387710976 } ], "submission_sha256": "98ce4fc7ff2c77e5e7e160cd655225f260dda50ef0e810404955cfaec39f06ee", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/tpe/data/submission.json", "comparison_value": 387710976, "comparison_rank_key": [ 387710976.0, 1479.0, -0.004122906395126913 ], "rank": 7 } ] }, { "id": "decode", "title": "Decode latency at validation BPB < 1.05", "metric": "nopref_request_ms_median", "unit": "ms", "direction": "minimize", "budget": 100, "comparison_step": null, "quality_gate": { "metric": "val_bpb", "operator": "<", "value": 1.05 }, "tiebreaks": [ { "metric": "request_ms_median", "direction": "minimize" } ], "runs": [ { "run_id": "autoscientists--decode", "method_id": "autoscientists", "task_id": "decode", "metric": "nopref_request_ms_median", "unit": "ms", "direction": "minimize", "best_value": 49.695491790771484, "reference_value": 465.54434299468994, "relative_improvement": 0.8932529359693276, "rank_key": [ 49.695491790771484, 83.9395523071289, -0.001501603364794546 ], "best_launch_seq": 85, "best_candidate_id": "exp-e46658b04d8f1fe38e73eb5a", "best_is_reference": false, "budget": { "spent": 84, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 2, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 2 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 16.39203527777778, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 465.54434299468994, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-7820d55b3e7fb33cada87a9b", "value": 261.6335153579712, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp-bfd628028be5762ce7b87897", "value": 405.4243564605713, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-efefb583c2a96e9de6056601", "value": 254.93311882019043, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-50a8a0ac7005c0bcf762acd7", "value": 336.3419771194458, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-3da4ba8ba48b91f856a1deb1", "value": 251.4479160308838, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp-3da0803d642619c54dd6be5d", "value": 240.82183837890625, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-062a92318ad9837bd3bef54a", "value": 220.78418731689453, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-501d9b2fe73588059ee58b73", "value": 209.62560176849365, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-34eb07aed1eac3a1808b529e", "value": 198.29082489013672, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-49b2db0234c75a0c9104070f", "value": 177.02817916870117, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-e1df9ab361596f872eeaa391", "value": 178.76017093658447, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp-d7b1f405b5473ba3d2ee099e", "value": 154.22296524047852, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-8289e9393212a32537f2b570", "value": 156.1335325241089, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-ebe03a55c082c2156dc611aa", "value": 161.34214401245117, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-7fd2e554ac735fd0a8a548ba", "value": 133.18860530853271, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-bd76eff222b0336adf5b2256", "value": 125.27990341186523, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-a5a78de02dbe324a5b81f885", "value": 118.77357959747314, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp-f8da07e2a62a763ddcdb8407", "value": 110.91053485870361, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp-75b25e5ca7318dd4259025e7", "value": 111.02139949798584, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp-0b669718705617a2574864a1", "value": 108.09659957885742, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp-b1343e89ecac731b8ea3c7ba", "value": 100.38650035858154, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-e63e182771b4754208c5d143", "value": 107.80596733093262, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp-91050363a30bef83a7f4681e", "value": 104.89368438720703, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp-b5a36ea4913c86f0aa6cdaa5", "value": 107.30218887329102, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp-ddd75def189d886ce1e81bac", "value": 102.91540622711182, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp-aebe70ada91ad9f5a8b85541", "value": 96.21739387512207, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp-cfcbb7fad7e44f344defa514", "value": 101.68170928955078, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp-e0b40482dddcee3b91fc96b2", "value": 91.0407304763794, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-4e4c087d524d9dea35421ad9", "value": 104.5076847076416, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-d857cf2c16007124c523d858", "value": 88.98413181304932, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-c755934afe9dc760f03633ea", "value": 86.3497257232666, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-ebc44a57b8d07283f611c68d", "value": 86.3194465637207, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp-81a6b398d1c79f5b1cb8f29f", "value": 87.72122859954834, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp-7fc5157e2b30154db6c7e151", "value": 85.8837366104126, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp-9ee8ef84b9dfc2ae5d6ad714", "value": 83.89627933502197, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "exp-efbb7569fb447907f7f50bcb", "value": 81.11441135406494, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "exp-8d0384ba42a6058fb20174ee", "value": 82.57770538330078, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp-4c9bd7dee75df988cc2c3fa4", "value": 80.95800876617432, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp-101282fde9aa8d4af77b7380", "value": 76.74431800842285, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp-2403aebee3a27cf64e73c0a7", "value": 75.691819190979, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp-0d5010149b850f751879abe0", "value": 74.0443468093872, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp-9387262662cb41fe7ce26838", "value": 73.73356819152832, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp-647d4b2f458b446645a9feb4", "value": 73.49860668182373, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp-2b26a67b1dc80b3872040091", "value": 71.6170072555542, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp-f3fdd6abe94ce80204a1bc4f", "value": 71.5632438659668, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "exp-11f9e3935887e949e92ea6bf", "value": 68.90130043029785, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp-b41b37008ed87732465efbef", "value": 67.4518346786499, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp-8eca6ea0ae22674112b9abf1", "value": 69.7108507156372, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp-3d30daec0956e8cf774b3c9c", "value": 66.69604778289795, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp-dce2b22a6654a9045289393d", "value": 66.89321994781494, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp-c07c3681860c6fa7462ac137", "value": 65.8409595489502, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "exp-bda1654ccb8806676e34d914", "value": 64.77224826812744, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "exp-620b63b2ccb6b43e34071370", "value": 64.60070610046387, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "exp-78938548a315f6a99f2cb5b9", "value": 63.225388526916504, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "exp-c6df7b1dd46cd2fec7085186", "value": 66.46728515625, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp-de032d612ca46d675ad25f8a", "value": 66.55216217041016, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp-926e2953b7bfd799586467ff", "value": 64.55183029174805, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "exp-ce82e7a56447d851d6bc82d4", "value": 63.898324966430664, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp-12989028dcd11ed7fb50aee7", "value": 63.396453857421875, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "exp-4d97c7af0aa859e1c4face9d", "value": 64.22996520996094, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "exp-de92a44917fb10432b257b7f", "value": 64.77761268615723, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "exp-78598168d48fbfcb46ad2eeb", "value": 62.95478343963623, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "exp-3093cda96a565a2deb122b92", "value": 62.26706504821777, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp-2738e8ee75a93f08b00c6fe2", "value": 61.62524223327637, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "exp-fe2fd4c3b5d7a7a4e5e93259", "value": 60.965895652770996, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "exp-03c18b9854c6de7bfcca84a2", "value": 61.290621757507324, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "exp-23c2acf7524b7c416abb6b17", "value": 62.65711784362793, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "exp-3969a734ee5b7a8ecd04401e", "value": 59.92996692657471, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "exp-fb43cc0a4f05482f3d3b97e7", "value": 57.09207057952881, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "exp-4ab0d92166fab96adf4f3801", "value": 54.9931526184082, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "exp-ff41a9519e5be558f54761bc", "value": 52.382588386535645, "is_reference": false }, { "step": 84, "launch_seq": 85, "candidate_id": "exp-e46658b04d8f1fe38e73eb5a", "value": 49.695491790771484, "is_reference": false } ], "curve": [ { "step": 1, "value": 465.54434299468994 }, { "step": 2, "value": 261.6335153579712 }, { "step": 3, "value": 261.6335153579712 }, { "step": 4, "value": 254.93311882019043 }, { "step": 5, "value": 254.93311882019043 }, { "step": 6, "value": 254.93311882019043 }, { "step": 7, "value": 251.4479160308838 }, { "step": 8, "value": 251.4479160308838 }, { "step": 9, "value": 240.82183837890625 }, { "step": 10, "value": 220.78418731689453 }, { "step": 11, "value": 209.62560176849365 }, { "step": 12, "value": 198.29082489013672 }, { "step": 13, "value": 198.29082489013672 }, { "step": 14, "value": 177.02817916870117 }, { "step": 15, "value": 177.02817916870117 }, { "step": 16, "value": 154.22296524047852 }, { "step": 17, "value": 154.22296524047852 }, { "step": 18, "value": 154.22296524047852 }, { "step": 19, "value": 133.18860530853271 }, { "step": 20, "value": 125.27990341186523 }, { "step": 21, "value": 118.77357959747314 }, { "step": 22, "value": 110.91053485870361 }, { "step": 23, "value": 110.91053485870361 }, { "step": 24, "value": 108.09659957885742 }, { "step": 25, "value": 100.38650035858154 }, { "step": 26, "value": 100.38650035858154 }, { "step": 27, "value": 100.38650035858154 }, { "step": 28, "value": 100.38650035858154 }, { "step": 29, "value": 100.38650035858154 }, { "step": 30, "value": 96.21739387512207 }, { "step": 31, "value": 96.21739387512207 }, { "step": 32, "value": 91.0407304763794 }, { "step": 33, "value": 91.0407304763794 }, { "step": 34, "value": 88.98413181304932 }, { "step": 35, "value": 86.3497257232666 }, { "step": 36, "value": 86.3194465637207 }, { "step": 37, "value": 86.3194465637207 }, { "step": 38, "value": 85.8837366104126 }, { "step": 39, "value": 83.89627933502197 }, { "step": 40, "value": 81.11441135406494 }, { "step": 41, "value": 81.11441135406494 }, { "step": 42, "value": 81.11441135406494 }, { "step": 43, "value": 80.95800876617432 }, { "step": 44, "value": 76.74431800842285 }, { "step": 45, "value": 75.691819190979 }, { "step": 46, "value": 74.0443468093872 }, { "step": 47, "value": 73.73356819152832 }, { "step": 48, "value": 73.49860668182373 }, { "step": 49, "value": 73.49860668182373 }, { "step": 50, "value": 73.49860668182373 }, { "step": 51, "value": 71.6170072555542 }, { "step": 52, "value": 71.5632438659668 }, { "step": 53, "value": 68.90130043029785 }, { "step": 54, "value": 67.4518346786499 }, { "step": 55, "value": 67.4518346786499 }, { "step": 56, "value": 66.69604778289795 }, { "step": 57, "value": 66.69604778289795 }, { "step": 58, "value": 65.8409595489502 }, { "step": 59, "value": 64.77224826812744 }, { "step": 60, "value": 64.60070610046387 }, { "step": 61, "value": 63.225388526916504 }, { "step": 62, "value": 63.225388526916504 }, { "step": 63, "value": 63.225388526916504 }, { "step": 64, "value": 63.225388526916504 }, { "step": 65, "value": 63.225388526916504 }, { "step": 66, "value": 63.225388526916504 }, { "step": 67, "value": 63.225388526916504 }, { "step": 68, "value": 63.225388526916504 }, { "step": 69, "value": 62.95478343963623 }, { "step": 70, "value": 62.26706504821777 }, { "step": 71, "value": 61.62524223327637 }, { "step": 72, "value": 60.965895652770996 }, { "step": 73, "value": 60.965895652770996 }, { "step": 74, "value": 60.965895652770996 }, { "step": 75, "value": 60.965895652770996 }, { "step": 76, "value": 59.92996692657471 }, { "step": 77, "value": 59.92996692657471 }, { "step": 78, "value": 57.09207057952881 }, { "step": 79, "value": 57.09207057952881 }, { "step": 80, "value": 57.09207057952881 }, { "step": 81, "value": 54.9931526184082 }, { "step": 82, "value": 52.382588386535645 }, { "step": 83, "value": 52.382588386535645 }, { "step": 84, "value": 49.695491790771484 } ], "submission_sha256": "3981899cae6fea15d690b8892ed3ebabf7bb30ef0186748a8780f073870874cf", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/autoscientists/decode/submission.json", "comparison_value": 49.695491790771484, "comparison_rank_key": [ 49.695491790771484, 83.9395523071289, -0.001501603364794546 ], "rank": 1 }, { "run_id": "gear--decode", "method_id": "gear", "task_id": "decode", "metric": "nopref_request_ms_median", "unit": "ms", "direction": "minimize", "best_value": 56.014180183410645, "reference_value": 462.87715435028076, "relative_improvement": 0.8789869414444634, "rank_key": [ 56.014180183410645, 66.02013111114502, -0.0114157954632077 ], "best_launch_seq": 99, "best_candidate_id": "exp0098-final-config-fifteenth", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": null }, "evaluation_gpu_hours": 19.23525833333333, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 462.87715435028076, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp0002-compile-step", "value": 329.27846908569336, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp0003-compile-body", "value": 326.9953727722168, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp0004-fused-projection", "value": 428.49254608154297, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp0005-stacked-projection", "value": 431.3185214996338, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp0006b-autotune-step", "value": 197.0045566558838, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp0007-compile-plus-projection", "value": 264.5317316055298, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp0008-attention-splits", "value": 418.2302951812744, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp0009-autotune-plus-projection", "value": 199.0112066268921, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp0010-merged-qk-norm", "value": 273.9264965057373, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp0011-merged-qk-autotuned", "value": 208.79197120666504, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp0012-fusion-options", "value": 197.6567506790161, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp0013-fusion-no-autotune", "value": 263.9356851577759, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp0014-triton-prepare", "value": 200.23632049560547, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp0015-compiled-prefill", "value": 207.13067054748535, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp0016-splits-autotuned", "value": 184.22460556030273, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp0017-prepare-plus-splits", "value": 187.14439868927002, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp0018-triton-attention", "value": 212.53681182861328, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp0019-split-triton-attention", "value": 138.67104053497314, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp0020-combo-kernels", "value": 183.1272840499878, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp0021-split-attn-merged-qk", "value": 146.87275886535645, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp0022-epilogue-first", "value": 197.49009609222412, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp0023-combo-plus-split-attn", "value": 141.5715217590332, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp0024-own-gemv", "value": 338.01352977752686, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp0025-split-attn-epilogue", "value": 138.56983184814453, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp0026-sixteen-splits", "value": 135.39385795593262, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp0027-sixteen-splits-best", "value": 132.75301456451416, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp0028-library-sixteen", "value": 183.77232551574707, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp0029-thirtytwo-splits", "value": 135.68854331970215, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp0031-thirtytwo-on-best", "value": 136.88945770263672, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp0032-walk-launch-config", "value": 136.63053512573242, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp0033-tuned-tiling-two-warps", "value": 138.99171352386475, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp0034-eight-warps", "value": 140.8144235610962, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp0035-best-replicate", "value": 132.97057151794434, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp0036-depth-seven", "value": 119.52924728393555, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp0037-depth-seven-combo", "value": 121.16575241088867, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp0038-depth-six", "value": 103.74116897583008, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "exp0039-depth-six-twowarp", "value": 104.21383380889893, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp0040-depth-four", "value": 73.4243392944336, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "exp0041-depth-four-best", "value": 71.62833213806152, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp0042-depth-three", "value": 57.58988857269287, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp0043-depth-three-combo", "value": 57.46650695800781, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp0044-half-batch", "value": 133.07058811187744, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp0045-depth-three-half-batch", "value": 57.7392578125, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp0047-depth-three-best-path", "value": 56.29932880401611, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp0049-depth-three-half-batch-combo", "value": 57.944655418395996, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp0050-resized-for-clock", "value": 56.62667751312256, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp0051-reported-replicate", "value": 56.862831115722656, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "exp0052-longer-warmdown", "value": 58.29918384552002, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp0053-double-matrix-lr", "value": 56.36155605316162, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp0054-half-matrix-lr", "value": 57.41775035858154, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp0055-half-matrix-lr-full-batch", "value": 56.434035301208496, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp0056-matrix-lr-001", "value": 56.23829364776611, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp0057-matrix-lr-003", "value": 56.327104568481445, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "exp0058-half-embedding-lr", "value": 56.30683898925781, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "exp0059-embedding-lr-substrate-pair", "value": 56.85436725616455, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "exp0060-no-weight-decay", "value": 57.524681091308594, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "exp0061-double-weight-decay", "value": 56.53643608093262, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp0062-double-unembedding-lr", "value": 56.8467378616333, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp0063-paired-batch-and-rates", "value": 56.55562877655029, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "exp0064-adam-beta1-09", "value": 57.00337886810303, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp0065-quarter-batch-diagonal", "value": 57.05440044403076, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "exp0066-half-scalar-lr", "value": 56.53512477874756, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "exp0067-best-config-replicate", "value": 56.594252586364746, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "exp0068-warmup-three-percent", "value": 56.433796882629395, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "exp0069-measured-optimizer-config", "value": 56.764841079711914, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp0070-paired-batch-second-reading", "value": 56.461453437805176, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "exp0071-reported-third-reading", "value": 56.28383159637451, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "exp0072-reported-fourth-reading", "value": 56.581854820251465, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "exp0073-narrow-window", "value": 57.04045295715332, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "exp0074-narrow-window-second", "value": 56.908607482910156, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "exp0075-narrow-window-reported", "value": 56.83314800262451, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "exp0076-window-256", "value": 56.23829364776611, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "exp0077-reported-window-256", "value": 56.595683097839355, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "exp0078-final-config-second", "value": 56.46538734436035, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "exp0079-window512-muon001", "value": 57.758450508117676, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "exp0080-final-config-third", "value": 56.40137195587158, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "exp0081-final-config-fourth", "value": 56.352853775024414, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "exp0082-window512-substrate-second", "value": 56.864261627197266, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "exp0083-final-config-fifth", "value": 57.51168727874756, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "exp0084-final-config-sixth", "value": 56.51450157165527, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp0085-final-config-seventh", "value": 56.160569190979004, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "exp0086-final-config-eighth", "value": 57.37006664276123, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "exp0087-final-lr-frac", "value": 56.24282360076904, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "exp0088-final-lr-frac-second", "value": 56.55980110168457, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "exp0089-final-config-ninth", "value": 56.430816650390625, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "exp0090-final-config-tenth", "value": 57.190895080566406, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "exp0091-two-microbatches", "value": 56.76758289337158, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "exp0092-two-microbatches-second", "value": 56.328415870666504, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "exp0093-final-config-eleventh", "value": 56.551337242126465, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "exp0094-final-config-twelfth", "value": 56.304216384887695, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "exp0095-final-config-thirteenth", "value": 56.409239768981934, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "exp0096-final-config-fourteenth", "value": 57.392001152038574, "is_reference": false }, { "step": 98, "launch_seq": 98, "candidate_id": "exp0097-window256-second-on-pair", "value": 56.59365653991699, "is_reference": false }, { "step": 99, "launch_seq": 99, "candidate_id": "exp0098-final-config-fifteenth", "value": 56.014180183410645, "is_reference": false }, { "step": 100, "launch_seq": 100, "candidate_id": "exp0099-final-config-sixteenth", "value": 56.53941631317139, "is_reference": false } ], "curve": [ { "step": 1, "value": 462.87715435028076 }, { "step": 2, "value": 329.27846908569336 }, { "step": 3, "value": 326.9953727722168 }, { "step": 4, "value": 326.9953727722168 }, { "step": 5, "value": 326.9953727722168 }, { "step": 6, "value": 326.9953727722168 }, { "step": 7, "value": 197.0045566558838 }, { "step": 8, "value": 197.0045566558838 }, { "step": 9, "value": 197.0045566558838 }, { "step": 10, "value": 197.0045566558838 }, { "step": 11, "value": 197.0045566558838 }, { "step": 12, "value": 197.0045566558838 }, { "step": 13, "value": 197.0045566558838 }, { "step": 14, "value": 197.0045566558838 }, { "step": 15, "value": 197.0045566558838 }, { "step": 16, "value": 197.0045566558838 }, { "step": 17, "value": 184.22460556030273 }, { "step": 18, "value": 184.22460556030273 }, { "step": 19, "value": 184.22460556030273 }, { "step": 20, "value": 138.67104053497314 }, { "step": 21, "value": 138.67104053497314 }, { "step": 22, "value": 138.67104053497314 }, { "step": 23, "value": 138.67104053497314 }, { "step": 24, "value": 138.67104053497314 }, { "step": 25, "value": 138.67104053497314 }, { "step": 26, "value": 138.56983184814453 }, { "step": 27, "value": 135.39385795593262 }, { "step": 28, "value": 132.75301456451416 }, { "step": 29, "value": 132.75301456451416 }, { "step": 30, "value": 132.75301456451416 }, { "step": 31, "value": 132.75301456451416 }, { "step": 32, "value": 132.75301456451416 }, { "step": 33, "value": 132.75301456451416 }, { "step": 34, "value": 132.75301456451416 }, { "step": 35, "value": 132.75301456451416 }, { "step": 36, "value": 132.75301456451416 }, { "step": 37, "value": 119.52924728393555 }, { "step": 38, "value": 119.52924728393555 }, { "step": 39, "value": 103.74116897583008 }, { "step": 40, "value": 103.74116897583008 }, { "step": 41, "value": 73.4243392944336 }, { "step": 42, "value": 71.62833213806152 }, { "step": 43, "value": 57.58988857269287 }, { "step": 44, "value": 57.46650695800781 }, { "step": 45, "value": 57.46650695800781 }, { "step": 46, "value": 57.46650695800781 }, { "step": 47, "value": 57.46650695800781 }, { "step": 48, "value": 56.29932880401611 }, { "step": 49, "value": 56.29932880401611 }, { "step": 50, "value": 56.29932880401611 }, { "step": 51, "value": 56.29932880401611 }, { "step": 52, "value": 56.29932880401611 }, { "step": 53, "value": 56.29932880401611 }, { "step": 54, "value": 56.29932880401611 }, { "step": 55, "value": 56.29932880401611 }, { "step": 56, "value": 56.29932880401611 }, { "step": 57, "value": 56.23829364776611 }, { "step": 58, "value": 56.23829364776611 }, { "step": 59, "value": 56.23829364776611 }, { "step": 60, "value": 56.23829364776611 }, { "step": 61, "value": 56.23829364776611 }, { "step": 62, "value": 56.23829364776611 }, { "step": 63, "value": 56.23829364776611 }, { "step": 64, "value": 56.23829364776611 }, { "step": 65, "value": 56.23829364776611 }, { "step": 66, "value": 56.23829364776611 }, { "step": 67, "value": 56.23829364776611 }, { "step": 68, "value": 56.23829364776611 }, { "step": 69, "value": 56.23829364776611 }, { "step": 70, "value": 56.23829364776611 }, { "step": 71, "value": 56.23829364776611 }, { "step": 72, "value": 56.23829364776611 }, { "step": 73, "value": 56.23829364776611 }, { "step": 74, "value": 56.23829364776611 }, { "step": 75, "value": 56.23829364776611 }, { "step": 76, "value": 56.23829364776611 }, { "step": 77, "value": 56.23829364776611 }, { "step": 78, "value": 56.23829364776611 }, { "step": 79, "value": 56.23829364776611 }, { "step": 80, "value": 56.23829364776611 }, { "step": 81, "value": 56.23829364776611 }, { "step": 82, "value": 56.23829364776611 }, { "step": 83, "value": 56.23829364776611 }, { "step": 84, "value": 56.23829364776611 }, { "step": 85, "value": 56.23829364776611 }, { "step": 86, "value": 56.160569190979004 }, { "step": 87, "value": 56.160569190979004 }, { "step": 88, "value": 56.160569190979004 }, { "step": 89, "value": 56.160569190979004 }, { "step": 90, "value": 56.160569190979004 }, { "step": 91, "value": 56.160569190979004 }, { "step": 92, "value": 56.160569190979004 }, { "step": 93, "value": 56.160569190979004 }, { "step": 94, "value": 56.160569190979004 }, { "step": 95, "value": 56.160569190979004 }, { "step": 96, "value": 56.160569190979004 }, { "step": 97, "value": 56.160569190979004 }, { "step": 98, "value": 56.160569190979004 }, { "step": 99, "value": 56.014180183410645 }, { "step": 100, "value": 56.014180183410645 } ], "submission_sha256": "e473225b50af17e5e5277baf4ef5a4df43595e631205401781c6879db203e480", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/gear/decode/submission.json", "comparison_value": 56.014180183410645, "comparison_rank_key": [ 56.014180183410645, 66.02013111114502, -0.0114157954632077 ], "rank": 2 }, { "run_id": "arbor--decode", "method_id": "arbor", "task_id": "decode", "metric": "nopref_request_ms_median", "unit": "ms", "direction": "minimize", "best_value": 57.08944797515869, "reference_value": 477.38349437713623, "relative_improvement": 0.8804117682165659, "rank_key": [ 57.08944797515869, 82.65674114227295, -0.0014702022368526801 ], "best_launch_seq": 37, "best_candidate_id": "exp-6cc16cf9f24121a0af0ec527", "best_is_reference": false, "budget": { "spent": 39, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 7.828529166666669, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 477.38349437713623, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-9889e1cfa970607e1345dbae", "value": 377.9470920562744, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp-30deda1d84b6d915fada7b47", "value": 220.79086303710938, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-29f7ac1f32b178e4ef372897", "value": 183.4656000137329, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-8524463b911a667814d1f48e", "value": 214.73073959350586, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-23d30921cbffec1b9181aa71", "value": 176.6587495803833, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-c6782ed21c19d741432b0265", "value": 175.284743309021, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp-81a9b35ce0d9c2ffd7ec51a4", "value": 135.95616817474365, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-8950d0c4e897f5e77bf01717", "value": 182.4880838394165, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-cfb551368247ee295853d05c", "value": 122.67565727233887, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-ae9722328de955852c4b9516", "value": 114.16471004486084, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-8d348478285ae3162643f018", "value": 137.33220100402832, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-b5e19a04c66ca998a6852421", "value": 122.81155586242676, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp-3aa00ba1f297819aad87593f", "value": 132.90643692016602, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-0a9cf66b676951e0c2213e3d", "value": 110.75544357299805, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-11204ea3256ce37180670350", "value": 122.8102445602417, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-b15a11d548501908f56196bc", "value": 122.44284152984619, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp-bff24a4494db62d09e9d2d44", "value": 101.47833824157715, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp-c2a7110d0c9879f62a370ede", "value": 79.28037643432617, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp-bed2f37378a2198304122408", "value": 104.06756401062012, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-98cb48df4e68d07006f7a530", "value": 79.29694652557373, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp-cf1e777e4acf2d8ccc5a6242", "value": 77.72445678710938, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp-9232e4303ca9ab523d4a039f", "value": 78.78732681274414, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp-18ccbea35f1c3b0902a3774b", "value": 72.30174541473389, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp-31d16a9a11d9dbce324b575a", "value": 72.23713397979736, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp-ed33d6e18c9dba282583b05d", "value": 70.74534893035889, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-e074f14e6e59147f3881eeca", "value": 72.60382175445557, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-4faa36c7560dc2fe216e2925", "value": 66.72990322113037, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-b24b720ef2e5404fdc035f21", "value": 62.110185623168945, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-fa08341ef668c204d729b514", "value": 57.6854944229126, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp-6cc16cf9f24121a0af0ec527", "value": 57.08944797515869, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp-09258402aa228615eb992b6e", "value": 64.72635269165039, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp-ed6cb6d2f24cf7cafb1ffa89", "value": 64.21637535095215, "is_reference": false } ], "curve": [ { "step": 1, "value": 477.38349437713623 }, { "step": 2, "value": 377.9470920562744 }, { "step": 3, "value": 220.79086303710938 }, { "step": 4, "value": 183.4656000137329 }, { "step": 5, "value": 183.4656000137329 }, { "step": 6, "value": 176.6587495803833 }, { "step": 7, "value": 175.284743309021 }, { "step": 8, "value": 135.95616817474365 }, { "step": 9, "value": 135.95616817474365 }, { "step": 10, "value": 135.95616817474365 }, { "step": 11, "value": 122.67565727233887 }, { "step": 12, "value": 114.16471004486084 }, { "step": 13, "value": 114.16471004486084 }, { "step": 14, "value": 114.16471004486084 }, { "step": 15, "value": 114.16471004486084 }, { "step": 16, "value": 114.16471004486084 }, { "step": 17, "value": 110.75544357299805 }, { "step": 18, "value": 110.75544357299805 }, { "step": 19, "value": 110.75544357299805 }, { "step": 20, "value": 110.75544357299805 }, { "step": 21, "value": 110.75544357299805 }, { "step": 22, "value": 101.47833824157715 }, { "step": 23, "value": 101.47833824157715 }, { "step": 24, "value": 79.28037643432617 }, { "step": 25, "value": 79.28037643432617 }, { "step": 26, "value": 79.28037643432617 }, { "step": 27, "value": 77.72445678710938 }, { "step": 28, "value": 77.72445678710938 }, { "step": 29, "value": 72.30174541473389 }, { "step": 30, "value": 72.23713397979736 }, { "step": 31, "value": 70.74534893035889 }, { "step": 32, "value": 70.74534893035889 }, { "step": 33, "value": 70.74534893035889 }, { "step": 34, "value": 66.72990322113037 }, { "step": 35, "value": 62.110185623168945 }, { "step": 36, "value": 57.6854944229126 }, { "step": 37, "value": 57.08944797515869 }, { "step": 38, "value": 57.08944797515869 }, { "step": 39, "value": 57.08944797515869 } ], "submission_sha256": "7634bc0f49185e29e1464fda962325c853450779b35d2a24816be89ca43168d5", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/arbor/decode/submission.json", "comparison_value": 57.08944797515869, "comparison_rank_key": [ 57.08944797515869, 82.65674114227295, -0.0014702022368526801 ], "rank": 3 }, { "run_id": "beam-search--decode", "method_id": "beam-search", "task_id": "decode", "metric": "nopref_request_ms_median", "unit": "ms", "direction": "minimize", "best_value": 65.14537334442139, "reference_value": 463.8998508453369, "relative_improvement": 0.8595701782923386, "rank_key": [ 65.14537334442139, 84.04147624969482, -0.0018213579218031573 ], "best_launch_seq": 90, "best_candidate_id": "g13c1-rotary-position-lookup-inside-region-one", "best_is_reference": false, "budget": { "spent": 97, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 4, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 4 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 19.25613833333333, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 463.8998508453369, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "g01c1-fused-decode-pointwise", "value": 260.22160053253174, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "g01c2-split-decode-attention", "value": 455.14893531799316, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "g02c1-whole-body-decode-graph", "value": 328.0966281890869, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "g02c3-compiled-regions-on-split-attn", "value": 249.44496154785156, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "g02c5-attention-free-blocks", "value": 389.37878608703613, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "g02c6-block-for-attention-span", "value": 410.3057384490967, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "g03c1-decode-only-fused-input-projection", "value": 219.27392482757568, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "g03c3-depth-seven-same-window-pattern", "value": 229.02965545654297, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "g03c5-rotary-island-alone", "value": 349.3342399597168, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "g03c7-compiled-decode-regions-on-attention-free", "value": 209.1289758682251, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "g03c6-autotuned-decode-lowering", "value": 202.34739780426025, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "g03c8-fused-input-projection-in-model", "value": 367.6074743270874, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "g04c1-fused-decode-input-projection-under-autotuned-graph", "value": 202.1700143814087, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "g04c3-autotuned-gemv-regions-with-eager-kvcache", "value": 160.43341159820557, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "g04c5-fused-decode-projection-absorbs-ve-gate", "value": 214.97833728790283, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "g04c2-depth-seven-on-autotuned-leader", "value": 175.70197582244873, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "g04c4-fused-input-projection-parameter-on-attention-free", "value": 186.51270866394043, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "g04c6-attention-free-blocks-on-fused-decode-projection", "value": 185.71996688842773, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "g04c8-unpinned-kvcache-reduction-split-at-depth-seven", "value": 220.4723358154297, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "g05c1-depth-seven-on-partitioned-autotuned-leader", "value": 136.79683208465576, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "g05c3-partitioned-regions-unpin-split-at-depth-seven", "value": 172.50430583953857, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "g05c5-autotuned-gemv-regions-on-fused-decode-projection", "value": 161.47148609161377, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "g05c7-third-attention-free-block", "value": 169.91662979125977, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "g05c2-bf16-decode-weight-copies", "value": 149.36912059783936, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "g05c4-paired-kv-stream-at-depth-seven", "value": 173.15971851348877, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "g05c6-narrower-mlp-expansion", "value": 184.0987205505371, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "g05c8-halved-short-attention-window", "value": 186.34748458862305, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "g06c1-bf16-decode-weights-at-depth-seven-r2", "value": 126.00934505462646, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "g06c3-transposed-derived-decode-weights-r2", "value": 197.18825817108154, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "g06c5-replicate-of-g04c3-r2", "value": 160.20464897155762, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "g06c7-bf16-decode-weights-for-the-rest-of-the-step-r2", "value": 152.45306491851807, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "g06c2-fourth-attending-block-removed-at-depth-seven-r2", "value": 118.85368824005127, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "g06c4-six-blocks-at-held-width-r2", "value": 106.86230659484863, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "g06c6-one-gather-derived-embedding-table-r2", "value": 155.46000003814697, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "g06c8-triton-only-gemm-backend-for-decode-regions-r2", "value": 160.58874130249023, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "g07c1-depth-five-at-held-width", "value": 81.9476842880249, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "g07c3-bf16-decode-unembedding-only", "value": 117.0046329498291, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "g07c5-fourth-attending-block-removed-on-bf16-body", "value": 109.26854610443115, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "g07c7-six-blocks-at-held-width-without-decode-copies", "value": 116.42229557037354, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "g07c2-replicate-of-g06c4", "value": 106.20486736297607, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "g07c4-bf16-decode-weights-except-unembedding", "value": 113.15643787384033, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "g07c6-one-gather-derived-embedding-table-at-depth-seven", "value": 123.6119270324707, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "g08c1-decode-head-region", "value": 81.16650581359863, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "g08c3-narrower-body-at-held-depth-six", "value": 97.56577014923096, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "g08c5-int8-derived-decode-unembedding", "value": 105.04257678985596, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "g08c7-depth-six-at-held-width-on-three-free-blocks", "value": 90.17229080200195, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "g08c2-one-optimizer-update-per-microbatch", "value": 82.66854286193848, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "g08c4-one-gather-derived-embedding-table-at-depth-six", "value": 103.91223430633545, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "g08c6-third-reading-of-the-depth-six-computation", "value": 108.09326171875, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "g08c8-fifth-attention-free-block-at-depth-seven", "value": 94.49529647827148, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "g09c3-half-microbatch-one-update-per-forward", "value": 82.30996131896973, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "g09c5-depth-four-at-held-width", "value": 79.1388750076294, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "g09c7-depth-five-at-held-width-on-three-free-blocks", "value": 81.6279649734497, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "g09c4-whole-budget-linear-anneal", "value": 82.0080041885376, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "g09c6-mlp-free-attending-block", "value": 77.60894298553467, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "g10c1-head-granularity-at-held-width", "value": 73.38297367095947, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "g10c3-depth-three-at-held-attender-count", "value": 73.00424575805664, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "g10c5-one-update-per-microbatch-on-the-decode-head-body", "value": 81.72059059143066, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "g10c7-shorter-warmdown-on-a-twice-measured-computation", "value": 82.64565467834473, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "g10c2-second-reading-of-the-leader", "value": 76.86400413513184, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "g10c4-mlp-free-attending-block-at-depth-four", "value": 72.79527187347412, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "g10c6-value-embeddings-on-the-attending-blocks", "value": 85.5417251586914, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "g10c8-depth-four-at-held-width-on-three-free-blocks", "value": 78.84228229522705, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "g11c1-ungated-value-residual-at-held-placement-r2", "value": 71.47812843322754, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "g11c5-second-reading-of-the-thin-margin-member-r2", "value": 73.45366477966309, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "g11c2-head-granularity-on-the-leader-r2", "value": 71.82955741882324, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "g11c6-value-residual-on-every-attender-ungated-r2", "value": 73.97949695587158, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "g11c8-autotuned-training-lowering-r2", "value": 77.11076736450195, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "g12c1-one-normalisation-for-the-query-and-key-streams", "value": 73.9591121673584, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "g12c3-value-embedding-gather-inside-region-one", "value": 69.4890022277832, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "g12c5-the-gate-gemv-absorbed-into-the-value-projection", "value": 75.81257820129395, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "g12c7-head-granularity-at-depth-three", "value": 69.7932243347168, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "g12c2-a-value-residual-stream-for-every-attending-block", "value": 71.28989696502686, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "g12c4-a-value-residual-gate-that-sees-the-whole-residual", "value": 71.63071632385254, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "g12c6-per-channel-residual-and-shortcut-gains", "value": 73.70305061340332, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "g13c1-rotary-position-lookup-inside-region-one", "value": 65.14537334442139, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "g13c3-token-embedding-gather-and-top-norm-inside-the-first-block", "value": 69.03302669525146, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "g13c5-three-value-embedding-gathers-inside-region-one", "value": 69.07963752746582, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "g13c7-two-value-embedding-gathers-inside-region-one-ungated", "value": 68.43411922454834, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "g13c4-the-unembedding-as-two-half-vocabulary-gemvs", "value": 72.34072685241699, "is_reference": false } ], "curve": [ { "step": 1, "value": 463.8998508453369 }, { "step": 2, "value": 260.22160053253174 }, { "step": 3, "value": 260.22160053253174 }, { "step": 4, "value": 260.22160053253174 }, { "step": 5, "value": 249.44496154785156 }, { "step": 6, "value": 249.44496154785156 }, { "step": 7, "value": 249.44496154785156 }, { "step": 8, "value": 249.44496154785156 }, { "step": 9, "value": 249.44496154785156 }, { "step": 10, "value": 219.27392482757568 }, { "step": 11, "value": 219.27392482757568 }, { "step": 12, "value": 219.27392482757568 }, { "step": 13, "value": 209.1289758682251 }, { "step": 14, "value": 209.1289758682251 }, { "step": 15, "value": 209.1289758682251 }, { "step": 16, "value": 202.34739780426025 }, { "step": 17, "value": 202.34739780426025 }, { "step": 18, "value": 202.1700143814087 }, { "step": 19, "value": 160.43341159820557 }, { "step": 20, "value": 160.43341159820557 }, { "step": 21, "value": 160.43341159820557 }, { "step": 22, "value": 160.43341159820557 }, { "step": 23, "value": 160.43341159820557 }, { "step": 24, "value": 160.43341159820557 }, { "step": 25, "value": 160.43341159820557 }, { "step": 26, "value": 136.79683208465576 }, { "step": 27, "value": 136.79683208465576 }, { "step": 28, "value": 136.79683208465576 }, { "step": 29, "value": 136.79683208465576 }, { "step": 30, "value": 136.79683208465576 }, { "step": 31, "value": 136.79683208465576 }, { "step": 32, "value": 136.79683208465576 }, { "step": 33, "value": 136.79683208465576 }, { "step": 34, "value": 126.00934505462646 }, { "step": 35, "value": 126.00934505462646 }, { "step": 36, "value": 126.00934505462646 }, { "step": 37, "value": 126.00934505462646 }, { "step": 38, "value": 118.85368824005127 }, { "step": 39, "value": 106.86230659484863 }, { "step": 40, "value": 106.86230659484863 }, { "step": 41, "value": 106.86230659484863 }, { "step": 42, "value": 81.9476842880249 }, { "step": 43, "value": 81.9476842880249 }, { "step": 44, "value": 81.9476842880249 }, { "step": 45, "value": 81.9476842880249 }, { "step": 46, "value": 81.9476842880249 }, { "step": 47, "value": 81.9476842880249 }, { "step": 48, "value": 81.9476842880249 }, { "step": 49, "value": 81.9476842880249 }, { "step": 50, "value": 81.16650581359863 }, { "step": 51, "value": 81.16650581359863 }, { "step": 52, "value": 81.16650581359863 }, { "step": 53, "value": 81.16650581359863 }, { "step": 54, "value": 81.16650581359863 }, { "step": 55, "value": 81.16650581359863 }, { "step": 56, "value": 81.16650581359863 }, { "step": 57, "value": 81.16650581359863 }, { "step": 58, "value": 81.16650581359863 }, { "step": 59, "value": 81.16650581359863 }, { "step": 60, "value": 79.1388750076294 }, { "step": 61, "value": 79.1388750076294 }, { "step": 62, "value": 79.1388750076294 }, { "step": 63, "value": 79.1388750076294 }, { "step": 64, "value": 77.60894298553467 }, { "step": 65, "value": 77.60894298553467 }, { "step": 66, "value": 73.38297367095947 }, { "step": 67, "value": 73.00424575805664 }, { "step": 68, "value": 73.00424575805664 }, { "step": 69, "value": 73.00424575805664 }, { "step": 70, "value": 73.00424575805664 }, { "step": 71, "value": 72.79527187347412 }, { "step": 72, "value": 72.79527187347412 }, { "step": 73, "value": 72.79527187347412 }, { "step": 74, "value": 71.47812843322754 }, { "step": 75, "value": 71.47812843322754 }, { "step": 76, "value": 71.47812843322754 }, { "step": 77, "value": 71.47812843322754 }, { "step": 78, "value": 71.47812843322754 }, { "step": 79, "value": 71.47812843322754 }, { "step": 80, "value": 71.47812843322754 }, { "step": 81, "value": 71.47812843322754 }, { "step": 82, "value": 71.47812843322754 }, { "step": 83, "value": 69.4890022277832 }, { "step": 84, "value": 69.4890022277832 }, { "step": 85, "value": 69.4890022277832 }, { "step": 86, "value": 69.4890022277832 }, { "step": 87, "value": 69.4890022277832 }, { "step": 88, "value": 69.4890022277832 }, { "step": 89, "value": 69.4890022277832 }, { "step": 90, "value": 65.14537334442139 }, { "step": 91, "value": 65.14537334442139 }, { "step": 92, "value": 65.14537334442139 }, { "step": 93, "value": 65.14537334442139 }, { "step": 94, "value": 65.14537334442139 }, { "step": 95, "value": 65.14537334442139 }, { "step": 96, "value": 65.14537334442139 }, { "step": 97, "value": 65.14537334442139 } ], "submission_sha256": "058607a5d383f4b575826da1c67add45dc54e305491abd30c31f1a9fbd6d7874", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/beam-search/decode/submission.json", "comparison_value": 65.14537334442139, "comparison_rank_key": [ 65.14537334442139, 84.04147624969482, -0.0018213579218031573 ], "rank": 4 }, { "run_id": "heuresis-map-elites--decode", "method_id": "heuresis-map-elites", "task_id": "decode", "metric": "nopref_request_ms_median", "unit": "ms", "direction": "minimize", "best_value": 97.81789779663086, "reference_value": 466.0876989364624, "relative_improvement": 0.7901298446197236, "rank_key": [ 97.81789779663086, 204.54061031341553, -0.0006320947797109344 ], "best_launch_seq": 83, "best_candidate_id": "exp-278c626e1578548bb751b121", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 15.545087777777773, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 466.0876989364624, "is_reference": true }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-a0772a20b417845978e06209", "value": 302.24549770355225, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-1d2bcbbaef365d2d3f322fa7", "value": 183.9437484741211, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp-ab7ce796278fc0d4ee416878", "value": 179.27098274230957, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp-8f825f0caf91c67a49d30099", "value": 178.98285388946533, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-9047508c61cda4dfac1032d2", "value": 162.59562969207764, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-099dbb34743ec0fe6f2e626d", "value": 136.67058944702148, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-c40301ab8d133bd2ff06659c", "value": 194.0518617630005, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp-f250e0abf60e29979eba2d0a", "value": 188.61567974090576, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-109b3c6197ed9084c3d20d82", "value": 198.4156370162964, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-88e6c83ae57a41d9eaa8f8c2", "value": 181.39982223510742, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-b7e98220eec1059933056ad6", "value": 174.04890060424805, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp-9bbc5940f0bf2a9c25e50366", "value": 195.74689865112305, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp-0e83dccbdb44b4da9f5f95a0", "value": 448.23157787323, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp-c1d19ef29d8a3b60f0a9adf3", "value": 192.834734916687, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp-2c32f83c216273bf6a511a1a", "value": 371.2337017059326, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-8409b250118989b33df874b4", "value": 370.87440490722656, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp-b0e93512e1b7082f84d9fa28", "value": 280.7813882827759, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp-ffa5e586acbb8fdf7d18205b", "value": 175.90391635894775, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp-2270f776b65f10427357dd51", "value": 234.62295532226562, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-0fd997b73cf994d58ee7ac61", "value": 285.15565395355225, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-8f1ec794beb00775e8d25263", "value": 320.9989070892334, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-73c745b2c3d624a3c88456d9", "value": 311.31553649902344, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-295f6cb5a2b9efc38201c2f0", "value": 311.173677444458, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp-c14526b8b3b3e356f94fe34e", "value": 311.1379146575928, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp-eca1f7fe93850f84e9cef02d", "value": 264.4389867782593, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "exp-3699f19cbc71886cce8c7fdd", "value": 281.866192817688, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp-f6abd2b4972d52e564209aa9", "value": 262.9474401473999, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp-edc5cb43938654c3853ad30c", "value": 282.2885513305664, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp-526221f0bb89aa3c9941d6cc", "value": 258.00442695617676, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp-947bdcc8378cbb334fd691e6", "value": 293.62332820892334, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp-eca4f302e9760c263d001051", "value": 425.08387565612793, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "exp-ca9ed6693c8f70b3f2cfba0d", "value": 370.52738666534424, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp-1477d726480836cfd31cc084", "value": 302.12533473968506, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp-03022a9f050126e51cbd9dc1", "value": 205.56139945983887, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp-27e59a8d9b2e0cd80ccb1652", "value": 471.3674783706665, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "exp-2939c79c91c063c60cae6c8f", "value": 392.9262161254883, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp-ff508d83a282e06e8cba1dda", "value": 383.758544921875, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp-73600514da49973b4523f711", "value": 382.8634023666382, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp-40cb4cd38ea07c69cdfbec38", "value": 472.15771675109863, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp-a7ad442fc65793d91ef620c0", "value": 383.3949565887451, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp-bab1d52de9dd8fe276df6cd5", "value": 472.234845161438, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "exp-08b32cccd1ce5156837e01bf", "value": 372.42591381073, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "exp-f9fbe91ad835655f0164d3a8", "value": 370.9298372268677, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp-19833a22eaea5eb329236917", "value": 453.58550548553467, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp-07bdfdb117f0255e28092541", "value": 352.5916337966919, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp-8677f9d111f8542824605476", "value": 114.57192897796631, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "exp-b7d993796da58dc6ca8f2e89", "value": 112.02108860015869, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "exp-2b77daf8ffa787c4821f6276", "value": 250.99313259124756, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "exp-329cb3759a005f7c4790820c", "value": 106.12249374389648, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "exp-0538b84a17752e2d16f32ef2", "value": 260.8811855316162, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "exp-36f8e4440e84d2914fac8efc", "value": 113.30699920654297, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "exp-0a969301832ccc6066d75c0e", "value": 105.65853118896484, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "exp-172aff8b382989e43a84c648", "value": 102.77175903320312, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "exp-278c626e1578548bb751b121", "value": 97.81789779663086, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp-9a07d9f323f1278bbef2d132", "value": 137.72857189178467, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "exp-691b7143a0763406336da731", "value": 115.90564250946045, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "exp-eeb0b981f5e450630824a5cd", "value": 110.48769950866699, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "exp-bafb68c590d5b571f3e032c2", "value": 110.29505729675293, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "exp-dc336bab809bc8b23a7c3dd6", "value": 110.10658740997314, "is_reference": false }, { "step": 100, "launch_seq": 100, "candidate_id": "exp-a61916bc16a90710df70a4d3", "value": 126.57701969146729, "is_reference": false } ], "curve": [ { "step": 1, "value": 466.0876989364624 }, { "step": 2, "value": 466.0876989364624 }, { "step": 3, "value": 466.0876989364624 }, { "step": 4, "value": 302.24549770355225 }, { "step": 5, "value": 302.24549770355225 }, { "step": 6, "value": 183.9437484741211 }, { "step": 7, "value": 183.9437484741211 }, { "step": 8, "value": 179.27098274230957 }, { "step": 9, "value": 178.98285388946533 }, { "step": 10, "value": 178.98285388946533 }, { "step": 11, "value": 162.59562969207764 }, { "step": 12, "value": 162.59562969207764 }, { "step": 13, "value": 136.67058944702148 }, { "step": 14, "value": 136.67058944702148 }, { "step": 15, "value": 136.67058944702148 }, { "step": 16, "value": 136.67058944702148 }, { "step": 17, "value": 136.67058944702148 }, { "step": 18, "value": 136.67058944702148 }, { "step": 19, "value": 136.67058944702148 }, { "step": 20, "value": 136.67058944702148 }, { "step": 21, "value": 136.67058944702148 }, { "step": 22, "value": 136.67058944702148 }, { "step": 23, "value": 136.67058944702148 }, { "step": 24, "value": 136.67058944702148 }, { "step": 25, "value": 136.67058944702148 }, { "step": 26, "value": 136.67058944702148 }, { "step": 27, "value": 136.67058944702148 }, { "step": 28, "value": 136.67058944702148 }, { "step": 29, "value": 136.67058944702148 }, { "step": 30, "value": 136.67058944702148 }, { "step": 31, "value": 136.67058944702148 }, { "step": 32, "value": 136.67058944702148 }, { "step": 33, "value": 136.67058944702148 }, { "step": 34, "value": 136.67058944702148 }, { "step": 35, "value": 136.67058944702148 }, { "step": 36, "value": 136.67058944702148 }, { "step": 37, "value": 136.67058944702148 }, { "step": 38, "value": 136.67058944702148 }, { "step": 39, "value": 136.67058944702148 }, { "step": 40, "value": 136.67058944702148 }, { "step": 41, "value": 136.67058944702148 }, { "step": 42, "value": 136.67058944702148 }, { "step": 43, "value": 136.67058944702148 }, { "step": 44, "value": 136.67058944702148 }, { "step": 45, "value": 136.67058944702148 }, { "step": 46, "value": 136.67058944702148 }, { "step": 47, "value": 136.67058944702148 }, { "step": 48, "value": 136.67058944702148 }, { "step": 49, "value": 136.67058944702148 }, { "step": 50, "value": 136.67058944702148 }, { "step": 51, "value": 136.67058944702148 }, { "step": 52, "value": 136.67058944702148 }, { "step": 53, "value": 136.67058944702148 }, { "step": 54, "value": 136.67058944702148 }, { "step": 55, "value": 136.67058944702148 }, { "step": 56, "value": 136.67058944702148 }, { "step": 57, "value": 136.67058944702148 }, { "step": 58, "value": 136.67058944702148 }, { "step": 59, "value": 136.67058944702148 }, { "step": 60, "value": 136.67058944702148 }, { "step": 61, "value": 136.67058944702148 }, { "step": 62, "value": 136.67058944702148 }, { "step": 63, "value": 136.67058944702148 }, { "step": 64, "value": 136.67058944702148 }, { "step": 65, "value": 136.67058944702148 }, { "step": 66, "value": 114.57192897796631 }, { "step": 67, "value": 114.57192897796631 }, { "step": 68, "value": 114.57192897796631 }, { "step": 69, "value": 114.57192897796631 }, { "step": 70, "value": 112.02108860015869 }, { "step": 71, "value": 112.02108860015869 }, { "step": 72, "value": 112.02108860015869 }, { "step": 73, "value": 106.12249374389648 }, { "step": 74, "value": 106.12249374389648 }, { "step": 75, "value": 106.12249374389648 }, { "step": 76, "value": 106.12249374389648 }, { "step": 77, "value": 106.12249374389648 }, { "step": 78, "value": 105.65853118896484 }, { "step": 79, "value": 105.65853118896484 }, { "step": 80, "value": 102.77175903320312 }, { "step": 81, "value": 102.77175903320312 }, { "step": 82, "value": 102.77175903320312 }, { "step": 83, "value": 97.81789779663086 }, { "step": 84, "value": 97.81789779663086 }, { "step": 85, "value": 97.81789779663086 }, { "step": 86, "value": 97.81789779663086 }, { "step": 87, "value": 97.81789779663086 }, { "step": 88, "value": 97.81789779663086 }, { "step": 89, "value": 97.81789779663086 }, { "step": 90, "value": 97.81789779663086 }, { "step": 91, "value": 97.81789779663086 }, { "step": 92, "value": 97.81789779663086 }, { "step": 93, "value": 97.81789779663086 }, { "step": 94, "value": 97.81789779663086 }, { "step": 95, "value": 97.81789779663086 }, { "step": 96, "value": 97.81789779663086 }, { "step": 97, "value": 97.81789779663086 }, { "step": 98, "value": 97.81789779663086 }, { "step": 99, "value": 97.81789779663086 }, { "step": 100, "value": 97.81789779663086 } ], "submission_sha256": "aac9c967dcf67a3bcc2e6a2818e10d42e718ba81a15bde0a0467b6923ee579a2", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/heuresis-map-elites/decode/submission.json", "comparison_value": 97.81789779663086, "comparison_rank_key": [ 97.81789779663086, 204.54061031341553, -0.0006320947797109344 ], "rank": 5 }, { "run_id": "sequential-search--decode", "method_id": "sequential-search", "task_id": "decode", "metric": "nopref_request_ms_median", "unit": "ms", "direction": "minimize", "best_value": 98.5555648803711, "reference_value": 464.036226272583, "relative_improvement": 0.7876123472685992, "rank_key": [ 98.5555648803711, 157.6099395751953, -0.004025025737517396 ], "best_launch_seq": 101, "best_candidate_id": "cand-0098", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "evaluation_capacity_gpus": 1, "gpu_instance_type": null, "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "gpus_per_evaluation": 1, "workers": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 19.58621805555555, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 464.036226272583, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "cand-0001", "value": 469.7606563568115, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "cand-0002", "value": 466.29559993743896, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "cand-0003", "value": 419.38936710357666, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "cand-0004", "value": 394.2350149154663, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "cand-0005", "value": 394.9235677719116, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "cand-0006", "value": 396.6104984283447, "is_reference": false }, { "step": 8, "launch_seq": 10, "candidate_id": "cand-0007", "value": 366.68097972869873, "is_reference": false }, { "step": 9, "launch_seq": 11, "candidate_id": "cand-0008", "value": 330.06560802459717, "is_reference": false }, { "step": 10, "launch_seq": 12, "candidate_id": "cand-0009", "value": 332.3906660079956, "is_reference": false }, { "step": 11, "launch_seq": 13, "candidate_id": "cand-0010", "value": 324.3753910064697, "is_reference": false }, { "step": 12, "launch_seq": 14, "candidate_id": "cand-0011", "value": 320.3918933868408, "is_reference": false }, { "step": 13, "launch_seq": 15, "candidate_id": "cand-0012", "value": 322.668194770813, "is_reference": false }, { "step": 14, "launch_seq": 16, "candidate_id": "cand-0013", "value": 321.72834873199463, "is_reference": false }, { "step": 15, "launch_seq": 17, "candidate_id": "cand-0014", "value": 297.2910404205322, "is_reference": false }, { "step": 16, "launch_seq": 18, "candidate_id": "cand-0015", "value": 327.85701751708984, "is_reference": false }, { "step": 17, "launch_seq": 19, "candidate_id": "cand-0016", "value": 298.3052730560303, "is_reference": false }, { "step": 18, "launch_seq": 20, "candidate_id": "cand-0017", "value": 290.1875972747803, "is_reference": false }, { "step": 19, "launch_seq": 21, "candidate_id": "cand-0018", "value": 290.2867794036865, "is_reference": false }, { "step": 20, "launch_seq": 22, "candidate_id": "cand-0019", "value": 291.47374629974365, "is_reference": false }, { "step": 21, "launch_seq": 23, "candidate_id": "cand-0020", "value": 282.85837173461914, "is_reference": false }, { "step": 22, "launch_seq": 24, "candidate_id": "cand-0021", "value": 276.51381492614746, "is_reference": false }, { "step": 23, "launch_seq": 25, "candidate_id": "cand-0022", "value": 278.0110836029053, "is_reference": false }, { "step": 24, "launch_seq": 26, "candidate_id": "cand-0023", "value": 265.8872604370117, "is_reference": false }, { "step": 25, "launch_seq": 27, "candidate_id": "cand-0024", "value": 264.6036148071289, "is_reference": false }, { "step": 26, "launch_seq": 28, "candidate_id": "cand-0025", "value": 265.4578685760498, "is_reference": false }, { "step": 27, "launch_seq": 29, "candidate_id": "cand-0026", "value": 258.6721181869507, "is_reference": false }, { "step": 28, "launch_seq": 30, "candidate_id": "cand-0027", "value": 258.9457035064697, "is_reference": false }, { "step": 29, "launch_seq": 31, "candidate_id": "cand-0028", "value": 237.16914653778076, "is_reference": false }, { "step": 30, "launch_seq": 32, "candidate_id": "cand-0029", "value": 231.2905788421631, "is_reference": false }, { "step": 31, "launch_seq": 34, "candidate_id": "cand-0030", "value": 231.90748691558838, "is_reference": false }, { "step": 32, "launch_seq": 35, "candidate_id": "cand-0031", "value": 205.60765266418457, "is_reference": false }, { "step": 33, "launch_seq": 36, "candidate_id": "cand-0032", "value": 210.91365814208984, "is_reference": false }, { "step": 34, "launch_seq": 37, "candidate_id": "cand-0033", "value": 205.48903942108154, "is_reference": false }, { "step": 35, "launch_seq": 38, "candidate_id": "cand-0034", "value": 170.2418327331543, "is_reference": false }, { "step": 36, "launch_seq": 39, "candidate_id": "cand-0035", "value": 170.0592041015625, "is_reference": false }, { "step": 37, "launch_seq": 40, "candidate_id": "cand-0036", "value": 169.34728622436523, "is_reference": false }, { "step": 38, "launch_seq": 41, "candidate_id": "cand-0037", "value": 176.42641067504883, "is_reference": false }, { "step": 39, "launch_seq": 42, "candidate_id": "cand-0038", "value": 170.13967037200928, "is_reference": false }, { "step": 40, "launch_seq": 43, "candidate_id": "cand-0039", "value": 168.10071468353271, "is_reference": false }, { "step": 41, "launch_seq": 44, "candidate_id": "cand-0040", "value": 158.85674953460693, "is_reference": false }, { "step": 42, "launch_seq": 46, "candidate_id": "cand-0042", "value": 155.5318832397461, "is_reference": false }, { "step": 43, "launch_seq": 47, "candidate_id": "cand-0043", "value": 150.39825439453125, "is_reference": false }, { "step": 44, "launch_seq": 48, "candidate_id": "cand-0044", "value": 146.20482921600342, "is_reference": false }, { "step": 45, "launch_seq": 49, "candidate_id": "cand-0045", "value": 147.1569538116455, "is_reference": false }, { "step": 46, "launch_seq": 50, "candidate_id": "cand-0046", "value": 152.89855003356934, "is_reference": false }, { "step": 47, "launch_seq": 51, "candidate_id": "cand-0047", "value": 147.11952209472656, "is_reference": false }, { "step": 48, "launch_seq": 52, "candidate_id": "cand-0048", "value": 152.57644653320312, "is_reference": false }, { "step": 49, "launch_seq": 53, "candidate_id": "cand-0049", "value": 137.27259635925293, "is_reference": false }, { "step": 50, "launch_seq": 54, "candidate_id": "cand-0050", "value": 137.63856887817383, "is_reference": false }, { "step": 51, "launch_seq": 55, "candidate_id": "cand-0051", "value": 135.370135307312, "is_reference": false }, { "step": 52, "launch_seq": 56, "candidate_id": "cand-0052", "value": 135.70404052734375, "is_reference": false }, { "step": 53, "launch_seq": 57, "candidate_id": "cand-0053", "value": 129.47475910186768, "is_reference": false }, { "step": 54, "launch_seq": 58, "candidate_id": "cand-0054", "value": 130.71560859680176, "is_reference": false }, { "step": 55, "launch_seq": 59, "candidate_id": "cand-0055", "value": 125.9465217590332, "is_reference": false }, { "step": 56, "launch_seq": 60, "candidate_id": "cand-0056", "value": 117.9434061050415, "is_reference": false }, { "step": 57, "launch_seq": 61, "candidate_id": "cand-0057", "value": 118.9652681350708, "is_reference": false }, { "step": 58, "launch_seq": 62, "candidate_id": "cand-0058", "value": 117.61581897735596, "is_reference": false }, { "step": 59, "launch_seq": 63, "candidate_id": "cand-0059", "value": 117.45810508728027, "is_reference": false }, { "step": 60, "launch_seq": 64, "candidate_id": "cand-0060", "value": 114.54606056213379, "is_reference": false }, { "step": 61, "launch_seq": 65, "candidate_id": "cand-0061", "value": 114.64858055114746, "is_reference": false }, { "step": 62, "launch_seq": 66, "candidate_id": "cand-0062", "value": 114.66813087463379, "is_reference": false }, { "step": 63, "launch_seq": 67, "candidate_id": "cand-0063", "value": 115.09013175964355, "is_reference": false }, { "step": 64, "launch_seq": 68, "candidate_id": "cand-0064", "value": 114.59636688232422, "is_reference": false }, { "step": 65, "launch_seq": 69, "candidate_id": "cand-0065", "value": 114.94767665863037, "is_reference": false }, { "step": 66, "launch_seq": 70, "candidate_id": "cand-0066", "value": 114.97390270233154, "is_reference": false }, { "step": 67, "launch_seq": 71, "candidate_id": "cand-0067", "value": 114.80152606964111, "is_reference": false }, { "step": 68, "launch_seq": 72, "candidate_id": "cand-0068", "value": 114.53878879547119, "is_reference": false }, { "step": 69, "launch_seq": 73, "candidate_id": "cand-0069", "value": 111.89568042755127, "is_reference": false }, { "step": 70, "launch_seq": 74, "candidate_id": "cand-0070", "value": 111.91976070404053, "is_reference": false }, { "step": 71, "launch_seq": 75, "candidate_id": "cand-0071", "value": 107.56909847259521, "is_reference": false }, { "step": 72, "launch_seq": 76, "candidate_id": "cand-0072", "value": 104.70235347747803, "is_reference": false }, { "step": 73, "launch_seq": 77, "candidate_id": "cand-0073", "value": 104.39729690551758, "is_reference": false }, { "step": 75, "launch_seq": 79, "candidate_id": "cand-0075", "value": 103.18303108215332, "is_reference": false }, { "step": 76, "launch_seq": 80, "candidate_id": "cand-0076", "value": 137.16864585876465, "is_reference": false }, { "step": 77, "launch_seq": 81, "candidate_id": "cand-0077", "value": 102.8977632522583, "is_reference": false }, { "step": 78, "launch_seq": 82, "candidate_id": "cand-0078", "value": 110.76581478118896, "is_reference": false }, { "step": 79, "launch_seq": 83, "candidate_id": "cand-0079", "value": 102.8357744216919, "is_reference": false }, { "step": 80, "launch_seq": 84, "candidate_id": "cand-0081", "value": 102.76734828948975, "is_reference": false }, { "step": 81, "launch_seq": 85, "candidate_id": "cand-0082", "value": 103.43444347381592, "is_reference": false }, { "step": 82, "launch_seq": 86, "candidate_id": "cand-0083", "value": 102.77330875396729, "is_reference": false }, { "step": 83, "launch_seq": 87, "candidate_id": "cand-0084", "value": 103.0651330947876, "is_reference": false }, { "step": 84, "launch_seq": 88, "candidate_id": "cand-0085", "value": 102.59675979614258, "is_reference": false }, { "step": 85, "launch_seq": 89, "candidate_id": "cand-0086", "value": 102.56218910217285, "is_reference": false }, { "step": 86, "launch_seq": 90, "candidate_id": "cand-0087", "value": 106.69159889221191, "is_reference": false }, { "step": 87, "launch_seq": 91, "candidate_id": "cand-0088", "value": 101.0277271270752, "is_reference": false }, { "step": 88, "launch_seq": 92, "candidate_id": "cand-0089", "value": 101.20439529418945, "is_reference": false }, { "step": 89, "launch_seq": 93, "candidate_id": "cand-0090", "value": 101.19938850402832, "is_reference": false }, { "step": 90, "launch_seq": 94, "candidate_id": "cand-0091", "value": 101.2953519821167, "is_reference": false }, { "step": 91, "launch_seq": 95, "candidate_id": "cand-0092", "value": 108.46376419067383, "is_reference": false }, { "step": 93, "launch_seq": 97, "candidate_id": "cand-0094", "value": 102.11753845214844, "is_reference": false }, { "step": 94, "launch_seq": 98, "candidate_id": "cand-0095", "value": 101.43542289733887, "is_reference": false }, { "step": 95, "launch_seq": 99, "candidate_id": "cand-0096", "value": 101.79698467254639, "is_reference": false }, { "step": 96, "launch_seq": 100, "candidate_id": "cand-0097", "value": 101.5768051147461, "is_reference": false }, { "step": 97, "launch_seq": 101, "candidate_id": "cand-0098", "value": 98.5555648803711, "is_reference": false }, { "step": 98, "launch_seq": 102, "candidate_id": "cand-0099", "value": 100.5789041519165, "is_reference": false }, { "step": 99, "launch_seq": 103, "candidate_id": "cand-0100", "value": 103.11830043792725, "is_reference": false }, { "step": 100, "launch_seq": 104, "candidate_id": "cand-0101", "value": 99.54404830932617, "is_reference": false } ], "curve": [ { "step": 1, "value": 464.036226272583 }, { "step": 2, "value": 464.036226272583 }, { "step": 3, "value": 464.036226272583 }, { "step": 4, "value": 419.38936710357666 }, { "step": 5, "value": 394.2350149154663 }, { "step": 6, "value": 394.2350149154663 }, { "step": 7, "value": 394.2350149154663 }, { "step": 8, "value": 366.68097972869873 }, { "step": 9, "value": 330.06560802459717 }, { "step": 10, "value": 330.06560802459717 }, { "step": 11, "value": 324.3753910064697 }, { "step": 12, "value": 320.3918933868408 }, { "step": 13, "value": 320.3918933868408 }, { "step": 14, "value": 320.3918933868408 }, { "step": 15, "value": 297.2910404205322 }, { "step": 16, "value": 297.2910404205322 }, { "step": 17, "value": 297.2910404205322 }, { "step": 18, "value": 290.1875972747803 }, { "step": 19, "value": 290.1875972747803 }, { "step": 20, "value": 290.1875972747803 }, { "step": 21, "value": 282.85837173461914 }, { "step": 22, "value": 276.51381492614746 }, { "step": 23, "value": 276.51381492614746 }, { "step": 24, "value": 265.8872604370117 }, { "step": 25, "value": 264.6036148071289 }, { "step": 26, "value": 264.6036148071289 }, { "step": 27, "value": 258.6721181869507 }, { "step": 28, "value": 258.6721181869507 }, { "step": 29, "value": 237.16914653778076 }, { "step": 30, "value": 231.2905788421631 }, { "step": 31, "value": 231.2905788421631 }, { "step": 32, "value": 205.60765266418457 }, { "step": 33, "value": 205.60765266418457 }, { "step": 34, "value": 205.48903942108154 }, { "step": 35, "value": 170.2418327331543 }, { "step": 36, "value": 170.0592041015625 }, { "step": 37, "value": 169.34728622436523 }, { "step": 38, "value": 169.34728622436523 }, { "step": 39, "value": 169.34728622436523 }, { "step": 40, "value": 168.10071468353271 }, { "step": 41, "value": 158.85674953460693 }, { "step": 42, "value": 155.5318832397461 }, { "step": 43, "value": 150.39825439453125 }, { "step": 44, "value": 146.20482921600342 }, { "step": 45, "value": 146.20482921600342 }, { "step": 46, "value": 146.20482921600342 }, { "step": 47, "value": 146.20482921600342 }, { "step": 48, "value": 146.20482921600342 }, { "step": 49, "value": 137.27259635925293 }, { "step": 50, "value": 137.27259635925293 }, { "step": 51, "value": 135.370135307312 }, { "step": 52, "value": 135.370135307312 }, { "step": 53, "value": 129.47475910186768 }, { "step": 54, "value": 129.47475910186768 }, { "step": 55, "value": 125.9465217590332 }, { "step": 56, "value": 117.9434061050415 }, { "step": 57, "value": 117.9434061050415 }, { "step": 58, "value": 117.61581897735596 }, { "step": 59, "value": 117.45810508728027 }, { "step": 60, "value": 114.54606056213379 }, { "step": 61, "value": 114.54606056213379 }, { "step": 62, "value": 114.54606056213379 }, { "step": 63, "value": 114.54606056213379 }, { "step": 64, "value": 114.54606056213379 }, { "step": 65, "value": 114.54606056213379 }, { "step": 66, "value": 114.54606056213379 }, { "step": 67, "value": 114.54606056213379 }, { "step": 68, "value": 114.53878879547119 }, { "step": 69, "value": 111.89568042755127 }, { "step": 70, "value": 111.89568042755127 }, { "step": 71, "value": 107.56909847259521 }, { "step": 72, "value": 104.70235347747803 }, { "step": 73, "value": 104.39729690551758 }, { "step": 74, "value": 104.39729690551758 }, { "step": 75, "value": 103.18303108215332 }, { "step": 76, "value": 103.18303108215332 }, { "step": 77, "value": 102.8977632522583 }, { "step": 78, "value": 102.8977632522583 }, { "step": 79, "value": 102.8357744216919 }, { "step": 80, "value": 102.76734828948975 }, { "step": 81, "value": 102.76734828948975 }, { "step": 82, "value": 102.76734828948975 }, { "step": 83, "value": 102.76734828948975 }, { "step": 84, "value": 102.59675979614258 }, { "step": 85, "value": 102.56218910217285 }, { "step": 86, "value": 102.56218910217285 }, { "step": 87, "value": 101.0277271270752 }, { "step": 88, "value": 101.0277271270752 }, { "step": 89, "value": 101.0277271270752 }, { "step": 90, "value": 101.0277271270752 }, { "step": 91, "value": 101.0277271270752 }, { "step": 92, "value": 101.0277271270752 }, { "step": 93, "value": 101.0277271270752 }, { "step": 94, "value": 101.0277271270752 }, { "step": 95, "value": 101.0277271270752 }, { "step": 96, "value": 101.0277271270752 }, { "step": 97, "value": 98.5555648803711 }, { "step": 98, "value": 98.5555648803711 }, { "step": 99, "value": 98.5555648803711 }, { "step": 100, "value": 98.5555648803711 } ], "submission_sha256": "1455c281ab803da6fb7fc51bb41ed63b4839172044c6966f6e5d96344b203297", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/sequential-search/decode/submission.json", "comparison_value": 98.5555648803711, "comparison_rank_key": [ 98.5555648803711, 157.6099395751953, -0.004025025737517396 ], "rank": 6 }, { "run_id": "tpe--decode", "method_id": "tpe", "task_id": "decode", "metric": "nopref_request_ms_median", "unit": "ms", "direction": "minimize", "best_value": 230.74018955230713, "reference_value": 465.35277366638184, "relative_improvement": 0.504160708585938, "rank_key": [ 230.74018955230713, 306.46753311157227, -0.004582426109573712 ], "best_launch_seq": 96, "best_candidate_id": "tpe-trial-0094", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 17.222510555555555, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 465.35277366638184, "is_reference": true }, { "step": 14, "launch_seq": 14, "candidate_id": "tpe-trial-0013", "value": 500.9126663208008, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "tpe-trial-0014", "value": 551.0368347167969, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "tpe-trial-0015", "value": 536.1863374710083, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "tpe-trial-0021", "value": 500.96988677978516, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "tpe-trial-0022", "value": 500.50950050354004, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "tpe-trial-0024", "value": 329.36251163482666, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "tpe-trial-0026", "value": 402.0453691482544, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "tpe-trial-0027", "value": 328.4255266189575, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "tpe-trial-0028", "value": 344.2208766937256, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "tpe-trial-0031", "value": 400.67219734191895, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "tpe-trial-0036", "value": 292.4659252166748, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "tpe-trial-0037", "value": 290.8724546432495, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "tpe-trial-0038", "value": 236.26017570495605, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "tpe-trial-0039", "value": 304.0804862976074, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "tpe-trial-0040", "value": 236.31823062896729, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "tpe-trial-0041", "value": 236.15491390228271, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "tpe-trial-0042", "value": 231.36472702026367, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "tpe-trial-0043", "value": 231.42766952514648, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "tpe-trial-0044", "value": 231.66584968566895, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "tpe-trial-0045", "value": 231.37378692626953, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "tpe-trial-0046", "value": 244.50886249542236, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "tpe-trial-0050", "value": 244.22109127044678, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "tpe-trial-0051", "value": 231.49359226226807, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "tpe-trial-0053", "value": 231.64081573486328, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "tpe-trial-0057", "value": 231.5828800201416, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "tpe-trial-0062", "value": 231.89020156860352, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "tpe-trial-0063", "value": 303.77447605133057, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "tpe-trial-0065", "value": 231.50885105133057, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "tpe-trial-0069", "value": 244.56119537353516, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "tpe-trial-0071", "value": 231.4542531967163, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "tpe-trial-0073", "value": 231.5387725830078, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "tpe-trial-0077", "value": 244.49050426483154, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "tpe-trial-0081", "value": 231.37080669403076, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "tpe-trial-0083-r1", "value": 230.7490110397339, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "tpe-trial-0084", "value": 230.90732097625732, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "tpe-trial-0091", "value": 230.8502197265625, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "tpe-trial-0092", "value": 230.81040382385254, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "tpe-trial-0094", "value": 230.74018955230713, "is_reference": false }, { "step": 99, "launch_seq": 99, "candidate_id": "tpe-trial-0097", "value": 230.8046817779541, "is_reference": false } ], "curve": [ { "step": 1, "value": 465.35277366638184 }, { "step": 2, "value": 465.35277366638184 }, { "step": 3, "value": 465.35277366638184 }, { "step": 4, "value": 465.35277366638184 }, { "step": 5, "value": 465.35277366638184 }, { "step": 6, "value": 465.35277366638184 }, { "step": 7, "value": 465.35277366638184 }, { "step": 8, "value": 465.35277366638184 }, { "step": 9, "value": 465.35277366638184 }, { "step": 10, "value": 465.35277366638184 }, { "step": 11, "value": 465.35277366638184 }, { "step": 12, "value": 465.35277366638184 }, { "step": 13, "value": 465.35277366638184 }, { "step": 14, "value": 465.35277366638184 }, { "step": 15, "value": 465.35277366638184 }, { "step": 16, "value": 465.35277366638184 }, { "step": 17, "value": 465.35277366638184 }, { "step": 18, "value": 465.35277366638184 }, { "step": 19, "value": 465.35277366638184 }, { "step": 20, "value": 465.35277366638184 }, { "step": 21, "value": 465.35277366638184 }, { "step": 22, "value": 465.35277366638184 }, { "step": 23, "value": 465.35277366638184 }, { "step": 24, "value": 465.35277366638184 }, { "step": 25, "value": 329.36251163482666 }, { "step": 26, "value": 329.36251163482666 }, { "step": 27, "value": 329.36251163482666 }, { "step": 28, "value": 328.4255266189575 }, { "step": 29, "value": 328.4255266189575 }, { "step": 30, "value": 328.4255266189575 }, { "step": 31, "value": 328.4255266189575 }, { "step": 32, "value": 328.4255266189575 }, { "step": 33, "value": 328.4255266189575 }, { "step": 34, "value": 328.4255266189575 }, { "step": 35, "value": 328.4255266189575 }, { "step": 36, "value": 328.4255266189575 }, { "step": 37, "value": 292.4659252166748 }, { "step": 38, "value": 290.8724546432495 }, { "step": 39, "value": 236.26017570495605 }, { "step": 40, "value": 236.26017570495605 }, { "step": 41, "value": 236.26017570495605 }, { "step": 42, "value": 236.15491390228271 }, { "step": 43, "value": 231.36472702026367 }, { "step": 44, "value": 231.36472702026367 }, { "step": 45, "value": 231.36472702026367 }, { "step": 46, "value": 231.36472702026367 }, { "step": 47, "value": 231.36472702026367 }, { "step": 48, "value": 231.36472702026367 }, { "step": 49, "value": 231.36472702026367 }, { "step": 50, "value": 231.36472702026367 }, { "step": 51, "value": 231.36472702026367 }, { "step": 52, "value": 231.36472702026367 }, { "step": 53, "value": 231.36472702026367 }, { "step": 54, "value": 231.36472702026367 }, { "step": 55, "value": 231.36472702026367 }, { "step": 56, "value": 231.36472702026367 }, { "step": 57, "value": 231.36472702026367 }, { "step": 58, "value": 231.36472702026367 }, { "step": 59, "value": 231.36472702026367 }, { "step": 60, "value": 231.36472702026367 }, { "step": 61, "value": 231.36472702026367 }, { "step": 62, "value": 231.36472702026367 }, { "step": 63, "value": 231.36472702026367 }, { "step": 64, "value": 231.36472702026367 }, { "step": 65, "value": 231.36472702026367 }, { "step": 66, "value": 231.36472702026367 }, { "step": 67, "value": 231.36472702026367 }, { "step": 68, "value": 231.36472702026367 }, { "step": 69, "value": 231.36472702026367 }, { "step": 70, "value": 231.36472702026367 }, { "step": 71, "value": 231.36472702026367 }, { "step": 72, "value": 231.36472702026367 }, { "step": 73, "value": 231.36472702026367 }, { "step": 74, "value": 231.36472702026367 }, { "step": 75, "value": 231.36472702026367 }, { "step": 76, "value": 231.36472702026367 }, { "step": 77, "value": 231.36472702026367 }, { "step": 78, "value": 231.36472702026367 }, { "step": 79, "value": 231.36472702026367 }, { "step": 80, "value": 231.36472702026367 }, { "step": 81, "value": 231.36472702026367 }, { "step": 82, "value": 231.36472702026367 }, { "step": 83, "value": 231.36472702026367 }, { "step": 84, "value": 231.36472702026367 }, { "step": 85, "value": 230.7490110397339 }, { "step": 86, "value": 230.7490110397339 }, { "step": 87, "value": 230.7490110397339 }, { "step": 88, "value": 230.7490110397339 }, { "step": 89, "value": 230.7490110397339 }, { "step": 90, "value": 230.7490110397339 }, { "step": 91, "value": 230.7490110397339 }, { "step": 92, "value": 230.7490110397339 }, { "step": 93, "value": 230.7490110397339 }, { "step": 94, "value": 230.7490110397339 }, { "step": 95, "value": 230.7490110397339 }, { "step": 96, "value": 230.74018955230713 }, { "step": 97, "value": 230.74018955230713 }, { "step": 98, "value": 230.74018955230713 }, { "step": 99, "value": 230.74018955230713 }, { "step": 100, "value": 230.74018955230713 } ], "submission_sha256": "88a8e89201a4add14cbc94c13db0281f8c5b69dfe756b80d818f309e11e4e09d", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/tpe/decode/submission.json", "comparison_value": 230.74018955230713, "comparison_rank_key": [ 230.74018955230713, 306.46753311157227, -0.004582426109573712 ], "rank": 7 } ] }, { "id": "flops", "title": "Counted training FLOPs per token at validation BPB < 1.05", "metric": "flops_per_token_measured", "unit": "FLOPs/token", "direction": "minimize", "budget": 100, "comparison_step": null, "quality_gate": { "metric": "val_bpb", "operator": "<", "value": 1.05 }, "tiebreaks": [ { "metric": "total_tokens", "direction": "minimize" } ], "runs": [ { "run_id": "autoscientists--flops", "method_id": "autoscientists", "task_id": "flops", "metric": "flops_per_token_measured", "unit": "FLOPs/token", "direction": "minimize", "best_value": 52148760, "reference_value": 239078400, "relative_improvement": 0.7818759034693222, "rank_key": [ 52148760.0, 1073479680.0, -0.0008626639992499463 ], "best_launch_seq": 100, "best_candidate_id": "exp-1ba9d8174dcf3345e024ef26", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 2, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 2 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 19.990911944444438, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 239078400, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-1df9eb83be3334c02e3cbd52", "value": 213912576, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp-360431e45fee5eb15c7fd886", "value": 220204032, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-88749888bcfd88873a23b066", "value": 135661824, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-bf8159a86ffdfe2d753be606", "value": 185601024, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-993a42fae456297a91c2c4bf", "value": 102041856, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp-d78a9f26748a2887c71fe477", "value": 89655552, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp-c7357d8fdec5ab6e5a709734", "value": 96143616, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-d1de4f6142e2c41f4085d302", "value": 83757312, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-2080a5f4ee1b05dd3b4eddad", "value": 76089600, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-a0d878b4d05dc37c0e764b53", "value": 74615040, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp-32b55b1c6c00d365d44a7e22", "value": 74613888, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-fd11384be21ff84f848aecc0", "value": 68420736, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-8fc592a840be7e0e98fc3f50", "value": 66012288, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-fe79db843ff642ad692c887e", "value": 65840256, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-31ee44b90ff73486a4801ce3", "value": 64832640, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp-3331515762a0a3f2d386be18", "value": 64832064, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp-f75939579b4d17d659581113", "value": 64537152, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp-f7662c5dd10dbdb21362875b", "value": 62890560, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-c0c1bbdd893dd9ca45b70a64", "value": 62890488, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp-cc083ce59f1bc2aaf7b2d0f1", "value": 61440576, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-67ff1e97fa3986e0af43bf55", "value": 61182528, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-6b220c26b8368dbbac5db740", "value": 61440504, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp-996562b4f3c610b913850acb", "value": 60924408, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp-b7edfcb469cca61c5c2011ff", "value": 60666360, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp-55d3874b86eb6ba95b7e5c95", "value": 60666288, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp-c50ea6f9e50ac4304b99594e", "value": 59634096, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp-d1378fc55fc9293e2177b20c", "value": 58343856, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp-c869e1a69fae5b31db57ed04", "value": 58343856, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp-262a433bed57d4b53e211063", "value": 58085808, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp-d34b96a03ae1675eb27b2200", "value": 57643440, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp-aa22ea0ab5e0cc70b6a13119", "value": 57828084, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp-d3d6b8352fc274be4a7239ce", "value": 57127344, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp-3a7c950a1e10e40677b3e456", "value": 57127344, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp-61e512efd317887a877589ce", "value": 57125040, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "exp-1d3ef0c36b563d7a94fbff7c", "value": 57124896, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "exp-29bf3f1365b4bdca27a6cc62", "value": 56866848, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "exp-d24ca1b859efc2b5fc798b93", "value": 56866848, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp-3f4330924464e5567bb1e2de", "value": 56719392, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "exp-8759db7d37409b4758061c45", "value": 56664384, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "exp-98f0bd51c03962dcce5d75b3", "value": 56239872, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "exp-d361babb8f8b50f99cfe6afa", "value": 56101632, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "exp-40355a98a63c3419cffd9755", "value": 55843872, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "exp-62fcf4b0264e690b70da709e", "value": 55070592, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "exp-eec9e5c6de1ea152a346580c", "value": 55070808, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "exp-9034e2990a011ffc609a504f", "value": 54997080, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp-44959268fbe4eab773dc3d6a", "value": 54997080, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "exp-dd3bbf69bdc832d5cd56be14", "value": 54886488, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "exp-cc23812eb751c2006c7a3159", "value": 54849624, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "exp-8cacd7a8f9900815ac596173", "value": 54849624, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "exp-1ad80ff82c03412f8ddf9de2", "value": 54775896, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "exp-f0469f5d30f347c3d0619d6d", "value": 54739032, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "exp-53b51a5a2e93851aabd671b2", "value": 54333528, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "exp-7542e4456160bfb547f27367", "value": 54434328, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "exp-ea4b52e476ed03dfd735247c", "value": 54102552, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "exp-b95be5d9b30b89f4008c3fd4", "value": 53596248, "is_reference": false }, { "step": 98, "launch_seq": 98, "candidate_id": "exp-d797b1182279871b060de549", "value": 53733912, "is_reference": false }, { "step": 99, "launch_seq": 99, "candidate_id": "exp-6ec392d74374489f7914d868", "value": 53301336, "is_reference": false }, { "step": 100, "launch_seq": 100, "candidate_id": "exp-1ba9d8174dcf3345e024ef26", "value": 52148760, "is_reference": false } ], "curve": [ { "step": 1, "value": 239078400 }, { "step": 2, "value": 213912576 }, { "step": 3, "value": 213912576 }, { "step": 4, "value": 135661824 }, { "step": 5, "value": 135661824 }, { "step": 6, "value": 102041856 }, { "step": 7, "value": 102041856 }, { "step": 8, "value": 89655552 }, { "step": 9, "value": 89655552 }, { "step": 10, "value": 89655552 }, { "step": 11, "value": 83757312 }, { "step": 12, "value": 83757312 }, { "step": 13, "value": 83757312 }, { "step": 14, "value": 76089600 }, { "step": 15, "value": 74615040 }, { "step": 16, "value": 74613888 }, { "step": 17, "value": 74613888 }, { "step": 18, "value": 68420736 }, { "step": 19, "value": 66012288 }, { "step": 20, "value": 65840256 }, { "step": 21, "value": 64832640 }, { "step": 22, "value": 64832064 }, { "step": 23, "value": 64832064 }, { "step": 24, "value": 64537152 }, { "step": 25, "value": 62890560 }, { "step": 26, "value": 62890488 }, { "step": 27, "value": 62890488 }, { "step": 28, "value": 62890488 }, { "step": 29, "value": 62890488 }, { "step": 30, "value": 61440576 }, { "step": 31, "value": 61440576 }, { "step": 32, "value": 61440576 }, { "step": 33, "value": 61440576 }, { "step": 34, "value": 61440576 }, { "step": 35, "value": 61182528 }, { "step": 36, "value": 61182528 }, { "step": 37, "value": 60924408 }, { "step": 38, "value": 60924408 }, { "step": 39, "value": 60666360 }, { "step": 40, "value": 60666360 }, { "step": 41, "value": 60666360 }, { "step": 42, "value": 60666360 }, { "step": 43, "value": 60666288 }, { "step": 44, "value": 60666288 }, { "step": 45, "value": 60666288 }, { "step": 46, "value": 59634096 }, { "step": 47, "value": 58343856 }, { "step": 48, "value": 58343856 }, { "step": 49, "value": 58343856 }, { "step": 50, "value": 58343856 }, { "step": 51, "value": 58085808 }, { "step": 52, "value": 58085808 }, { "step": 53, "value": 58085808 }, { "step": 54, "value": 58085808 }, { "step": 55, "value": 57643440 }, { "step": 56, "value": 57643440 }, { "step": 57, "value": 57643440 }, { "step": 58, "value": 57127344 }, { "step": 59, "value": 57127344 }, { "step": 60, "value": 57127344 }, { "step": 61, "value": 57127344 }, { "step": 62, "value": 57127344 }, { "step": 63, "value": 57127344 }, { "step": 64, "value": 57127344 }, { "step": 65, "value": 57127344 }, { "step": 66, "value": 57125040 }, { "step": 67, "value": 57125040 }, { "step": 68, "value": 57124896 }, { "step": 69, "value": 56866848 }, { "step": 70, "value": 56866848 }, { "step": 71, "value": 56719392 }, { "step": 72, "value": 56664384 }, { "step": 73, "value": 56664384 }, { "step": 74, "value": 56239872 }, { "step": 75, "value": 56101632 }, { "step": 76, "value": 56101632 }, { "step": 77, "value": 55843872 }, { "step": 78, "value": 55843872 }, { "step": 79, "value": 55070592 }, { "step": 80, "value": 55070592 }, { "step": 81, "value": 55070592 }, { "step": 82, "value": 55070592 }, { "step": 83, "value": 55070592 }, { "step": 84, "value": 55070592 }, { "step": 85, "value": 54997080 }, { "step": 86, "value": 54997080 }, { "step": 87, "value": 54886488 }, { "step": 88, "value": 54886488 }, { "step": 89, "value": 54849624 }, { "step": 90, "value": 54849624 }, { "step": 91, "value": 54775896 }, { "step": 92, "value": 54775896 }, { "step": 93, "value": 54739032 }, { "step": 94, "value": 54333528 }, { "step": 95, "value": 54333528 }, { "step": 96, "value": 54102552 }, { "step": 97, "value": 53596248 }, { "step": 98, "value": 53596248 }, { "step": 99, "value": 53301336 }, { "step": 100, "value": 52148760 } ], "submission_sha256": "58c1d78324f5abab442bc135d7e7a783a158e8762253fa8ad3568082beba8e3f", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/autoscientists/flops/submission.json", "comparison_value": 52148760, "comparison_rank_key": [ 52148760.0, 1073479680.0, -0.0008626639992499463 ], "rank": 1 }, { "run_id": "sequential-search--flops", "method_id": "sequential-search", "task_id": "flops", "metric": "flops_per_token_measured", "unit": "FLOPs/token", "direction": "minimize", "best_value": 56663808, "reference_value": 239078400, "relative_improvement": 0.7629906842274333, "rank_key": [ 56663808.0, 728760320.0, -0.0005253409552423705 ], "best_launch_seq": 66, "best_candidate_id": "r70-hand-the-kernel-the-views-it-already-accepts", "best_is_reference": false, "budget": { "spent": 81, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 15.493141111111111, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 239078400, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "r01-narrow-attention-pathway", "value": 182458368, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "r02-gathered-mlp-features", "value": 132132864, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "r03-block-diagonal-mlp", "value": 106967040, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "r04-cross-group-mlp-writeback", "value": 106967040, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "r05-local-span-prefix-mean", "value": 92811264, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "r07-readout-context-free-prior", "value": 92811264, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "r08-shared-kv-read", "value": 86519808, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "r11-learned-attention-sharpness", "value": 86519808, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "r12-span-matched-head-budget", "value": 77080320, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "r14-per-channel-residual-mix", "value": 77080320, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "r15-per-head-reach-ladder", "value": 70395648, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "r17-per-head-rotary-mix", "value": 70395648, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "r22-key-salience-restoration", "value": 70395648, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "r25-one-update-per-staged-batch", "value": 70395648, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "r26-cross-group-attention-read", "value": 68036352, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "r27-grouped-attention-write", "value": 66856704, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "r30-depth-blocked-token-features", "value": 66856704, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "r32-retrieval-layers-join-the-grouped-path", "value": 64497408, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "r35-third-depth-block-of-token-features", "value": 64497408, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "r41-per-token-key-identity-at-a-halved-local-radius", "value": 63317760, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "r42-one-reach-ladder-across-the-stack", "value": 60761856, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "r43-reach-matched-rotary-band", "value": 60761856, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "r44-charge-a-query-block-its-own-causal-reach", "value": 60073728, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "r46-one-key-table-for-every-floor-reach-head", "value": 60073728, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "r52-every-gate-reads-the-whole-residual-at-two-resolutions", "value": 60073728, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "r53-the-readout-reads-every-channel-at-two-resolutions", "value": 56928000, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "r63-charge-at-the-ladders-finest-rung", "value": 56755968, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "r69-admit-the-floor-to-the-charge-rung-set", "value": 56663808, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "r70-hand-the-kernel-the-views-it-already-accepts", "value": 56663808, "is_reference": false } ], "curve": [ { "step": 1, "value": 239078400 }, { "step": 2, "value": 182458368 }, { "step": 3, "value": 132132864 }, { "step": 4, "value": 106967040 }, { "step": 5, "value": 106967040 }, { "step": 6, "value": 92811264 }, { "step": 7, "value": 92811264 }, { "step": 8, "value": 92811264 }, { "step": 9, "value": 86519808 }, { "step": 10, "value": 86519808 }, { "step": 11, "value": 86519808 }, { "step": 12, "value": 86519808 }, { "step": 13, "value": 77080320 }, { "step": 14, "value": 77080320 }, { "step": 15, "value": 77080320 }, { "step": 16, "value": 70395648 }, { "step": 17, "value": 70395648 }, { "step": 18, "value": 70395648 }, { "step": 19, "value": 70395648 }, { "step": 20, "value": 70395648 }, { "step": 21, "value": 70395648 }, { "step": 22, "value": 70395648 }, { "step": 23, "value": 70395648 }, { "step": 24, "value": 70395648 }, { "step": 25, "value": 70395648 }, { "step": 26, "value": 70395648 }, { "step": 27, "value": 68036352 }, { "step": 28, "value": 66856704 }, { "step": 29, "value": 66856704 }, { "step": 30, "value": 66856704 }, { "step": 31, "value": 66856704 }, { "step": 32, "value": 64497408 }, { "step": 33, "value": 64497408 }, { "step": 34, "value": 64497408 }, { "step": 35, "value": 64497408 }, { "step": 36, "value": 64497408 }, { "step": 37, "value": 64497408 }, { "step": 38, "value": 64497408 }, { "step": 39, "value": 64497408 }, { "step": 40, "value": 64497408 }, { "step": 41, "value": 63317760 }, { "step": 42, "value": 60761856 }, { "step": 43, "value": 60761856 }, { "step": 44, "value": 60073728 }, { "step": 45, "value": 60073728 }, { "step": 46, "value": 60073728 }, { "step": 47, "value": 60073728 }, { "step": 48, "value": 60073728 }, { "step": 49, "value": 60073728 }, { "step": 50, "value": 60073728 }, { "step": 51, "value": 56928000 }, { "step": 52, "value": 56928000 }, { "step": 53, "value": 56928000 }, { "step": 54, "value": 56928000 }, { "step": 55, "value": 56928000 }, { "step": 56, "value": 56928000 }, { "step": 57, "value": 56928000 }, { "step": 58, "value": 56928000 }, { "step": 59, "value": 56755968 }, { "step": 60, "value": 56755968 }, { "step": 61, "value": 56755968 }, { "step": 62, "value": 56755968 }, { "step": 63, "value": 56755968 }, { "step": 64, "value": 56755968 }, { "step": 65, "value": 56663808 }, { "step": 66, "value": 56663808 }, { "step": 67, "value": 56663808 }, { "step": 68, "value": 56663808 }, { "step": 69, "value": 56663808 }, { "step": 70, "value": 56663808 }, { "step": 71, "value": 56663808 }, { "step": 72, "value": 56663808 }, { "step": 73, "value": 56663808 }, { "step": 74, "value": 56663808 }, { "step": 75, "value": 56663808 }, { "step": 76, "value": 56663808 }, { "step": 77, "value": 56663808 }, { "step": 78, "value": 56663808 }, { "step": 79, "value": 56663808 }, { "step": 80, "value": 56663808 }, { "step": 81, "value": 56663808 } ], "submission_sha256": "a5b7cd827a47cd01199e9d24df8f485a28db50d90d3c0286bc31ad547c134688", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/sequential-search/flops/submission.json", "comparison_value": 56663808, "comparison_rank_key": [ 56663808.0, 728760320.0, -0.0005253409552423705 ], "rank": 2 }, { "run_id": "arbor--flops", "method_id": "arbor", "task_id": "flops", "metric": "flops_per_token_measured", "unit": "FLOPs/token", "direction": "minimize", "best_value": 58296576, "reference_value": 239078400, "relative_improvement": 0.7561612592354642, "rank_key": [ 58296576.0, 632029184.0, -0.00014790548889664912 ], "best_launch_seq": 34, "best_candidate_id": "exp-4d49a7477fa13eb156f60986", "best_is_reference": false, "budget": { "spent": 65, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 12.965350833333334, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 239078400, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-52d79a82fb970a7f27fc416e", "value": 187173888, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-f805fd3289c339525129e622", "value": 185604096, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp-723d6f9c47940b965a23f6c2", "value": 169875456, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-81ec0060d226d242c82beca0", "value": 157292544, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-a8c40027bbfb6a3f7d62a58c", "value": 213912384, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-365315197f0a0b5d515ad167", "value": 188746560, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-dea79939625da6f7fb281408", "value": 176163648, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-8418074cbdd8723c86028771", "value": 125045760, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp-d359b638a5598c55827a5a44", "value": 136842240, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-ac403a930d02302708fed54b", "value": 97519104, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-8487ebe191f0d35bbf51aeeb", "value": 112462848, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp-0d0954db3494012139213b21", "value": 97519104, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp-d305871397402bc4903d1269", "value": 83364864, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp-4b52da94fa165fbada0bf187", "value": 83364864, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-8cd73d0c8c30b28350c8fa80", "value": 83364864, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp-6ab0836bc844e6cda76016ad", "value": 83364864, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp-e7cf680a3a18c4b961426533", "value": 70683648, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp-b1cbfa7c15e23fb286462418", "value": 67144704, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-b6fe5b03cce284a1fcb939be", "value": 62720256, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-4d49a7477fa13eb156f60986", "value": 58296576, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp-3f76532e59e03cd3eb10938d", "value": 58296576, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp-6da676c7475b0c1bf7043b02", "value": 58296576, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp-79ef3e9b4697ca6853f3ace3", "value": 58296576, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "exp-3a79e9a49a1cb24b68e2d019", "value": 58296576, "is_reference": false } ], "curve": [ { "step": 1, "value": 239078400 }, { "step": 2, "value": 187173888 }, { "step": 3, "value": 187173888 }, { "step": 4, "value": 187173888 }, { "step": 5, "value": 187173888 }, { "step": 6, "value": 185604096 }, { "step": 7, "value": 185604096 }, { "step": 8, "value": 169875456 }, { "step": 9, "value": 169875456 }, { "step": 10, "value": 157292544 }, { "step": 11, "value": 157292544 }, { "step": 12, "value": 157292544 }, { "step": 13, "value": 157292544 }, { "step": 14, "value": 157292544 }, { "step": 15, "value": 125045760 }, { "step": 16, "value": 125045760 }, { "step": 17, "value": 125045760 }, { "step": 18, "value": 97519104 }, { "step": 19, "value": 97519104 }, { "step": 20, "value": 97519104 }, { "step": 21, "value": 97519104 }, { "step": 22, "value": 97519104 }, { "step": 23, "value": 83364864 }, { "step": 24, "value": 83364864 }, { "step": 25, "value": 83364864 }, { "step": 26, "value": 83364864 }, { "step": 27, "value": 83364864 }, { "step": 28, "value": 83364864 }, { "step": 29, "value": 83364864 }, { "step": 30, "value": 70683648 }, { "step": 31, "value": 67144704 }, { "step": 32, "value": 67144704 }, { "step": 33, "value": 62720256 }, { "step": 34, "value": 58296576 }, { "step": 35, "value": 58296576 }, { "step": 36, "value": 58296576 }, { "step": 37, "value": 58296576 }, { "step": 38, "value": 58296576 }, { "step": 39, "value": 58296576 }, { "step": 40, "value": 58296576 }, { "step": 41, "value": 58296576 }, { "step": 42, "value": 58296576 }, { "step": 43, "value": 58296576 }, { "step": 44, "value": 58296576 }, { "step": 45, "value": 58296576 }, { "step": 46, "value": 58296576 }, { "step": 47, "value": 58296576 }, { "step": 48, "value": 58296576 }, { "step": 49, "value": 58296576 }, { "step": 50, "value": 58296576 }, { "step": 51, "value": 58296576 }, { "step": 52, "value": 58296576 }, { "step": 53, "value": 58296576 }, { "step": 54, "value": 58296576 }, { "step": 55, "value": 58296576 }, { "step": 56, "value": 58296576 }, { "step": 57, "value": 58296576 }, { "step": 58, "value": 58296576 }, { "step": 59, "value": 58296576 }, { "step": 60, "value": 58296576 }, { "step": 61, "value": 58296576 }, { "step": 62, "value": 58296576 }, { "step": 63, "value": 58296576 }, { "step": 64, "value": 58296576 }, { "step": 65, "value": 58296576 } ], "submission_sha256": "1cbf6e98c9a7e219828d6cad966e01754f933abf95fe333cfa2d782a1406dcbc", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/arbor/flops/submission.json", "comparison_value": 58296576, "comparison_rank_key": [ 58296576.0, 632029184.0, -0.00014790548889664912 ], "rank": 3 }, { "run_id": "gear--flops", "method_id": "gear", "task_id": "flops", "metric": "flops_per_token_measured", "unit": "FLOPs/token", "direction": "minimize", "best_value": 62233344, "reference_value": 239078400, "relative_improvement": 0.7396948281400578, "rank_key": [ 62233344.0, 913833984.0, -0.0003695269950472202 ], "best_launch_seq": 97, "best_candidate_id": "exp0095", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": null }, "evaluation_gpu_hours": 18.53159083333334, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 239078400, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp0002", "value": 220204032, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp0003", "value": 210766848, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp0004", "value": 213912576, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp0005", "value": 188746752, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp0006", "value": 110888640, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp0007", "value": 169872384, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp0008", "value": 150997248, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp0009", "value": 89654976, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp0010", "value": 213912576, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp0011", "value": 122685696, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp0012", "value": 169872384, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp0013", "value": 89654976, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp0014", "value": 101452032, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp0015", "value": 160435200, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp0016", "value": 89656704, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp0017", "value": 89656704, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp0018", "value": 80219520, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp0019", "value": 89656704, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp0020", "value": 75500928, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp0021", "value": 68423040, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp0023", "value": 68423040, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp0024", "value": 89656704, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp0025", "value": 89654976, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp0026", "value": 68423040, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp0029", "value": 68426496, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp0030", "value": 68426496, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp0031", "value": 68426496, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp0032", "value": 68423040, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp0033", "value": 68426496, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp0034", "value": 68423040, "is_reference": false }, { "step": 35, "launch_seq": 36, "candidate_id": "exp0035b", "value": 64884096, "is_reference": false }, { "step": 37, "launch_seq": 38, "candidate_id": "exp0037", "value": 64884096, "is_reference": false }, { "step": 38, "launch_seq": 39, "candidate_id": "exp0038", "value": 66653568, "is_reference": false }, { "step": 39, "launch_seq": 40, "candidate_id": "exp0039", "value": 68423040, "is_reference": false }, { "step": 40, "launch_seq": 41, "candidate_id": "exp0040", "value": 64884096, "is_reference": false }, { "step": 41, "launch_seq": 42, "candidate_id": "exp0041", "value": 66653568, "is_reference": false }, { "step": 42, "launch_seq": 43, "candidate_id": "exp0042", "value": 64884096, "is_reference": false }, { "step": 44, "launch_seq": 45, "candidate_id": "exp0044", "value": 64884096, "is_reference": false }, { "step": 48, "launch_seq": 49, "candidate_id": "exp0048", "value": 66653568, "is_reference": false }, { "step": 51, "launch_seq": 52, "candidate_id": "exp0051", "value": 64887552, "is_reference": false }, { "step": 52, "launch_seq": 53, "candidate_id": "exp0052", "value": 64887552, "is_reference": false }, { "step": 53, "launch_seq": 54, "candidate_id": "exp0053", "value": 64887552, "is_reference": false }, { "step": 58, "launch_seq": 59, "candidate_id": "exp0058", "value": 64887552, "is_reference": false }, { "step": 59, "launch_seq": 60, "candidate_id": "exp0059", "value": 64884096, "is_reference": false }, { "step": 60, "launch_seq": 61, "candidate_id": "exp0060", "value": 64884096, "is_reference": false }, { "step": 63, "launch_seq": 64, "candidate_id": "exp0063", "value": 64887552, "is_reference": false }, { "step": 67, "launch_seq": 68, "candidate_id": "exp0067", "value": 63118080, "is_reference": false }, { "step": 70, "launch_seq": 71, "candidate_id": "exp0070", "value": 63118080, "is_reference": false }, { "step": 71, "launch_seq": 72, "candidate_id": "exp0071", "value": 64884096, "is_reference": false }, { "step": 76, "launch_seq": 77, "candidate_id": "exp0076", "value": 63118080, "is_reference": false }, { "step": 80, "launch_seq": 82, "candidate_id": "exp0080b", "value": 63118080, "is_reference": false }, { "step": 83, "launch_seq": 85, "candidate_id": "exp0083", "value": 63118080, "is_reference": false }, { "step": 84, "launch_seq": 86, "candidate_id": "exp0084", "value": 63118080, "is_reference": false }, { "step": 85, "launch_seq": 87, "candidate_id": "exp0085", "value": 63118080, "is_reference": false }, { "step": 86, "launch_seq": 88, "candidate_id": "exp0086", "value": 63118080, "is_reference": false }, { "step": 88, "launch_seq": 90, "candidate_id": "exp0088", "value": 63118080, "is_reference": false }, { "step": 90, "launch_seq": 92, "candidate_id": "exp0090", "value": 63118080, "is_reference": false }, { "step": 91, "launch_seq": 93, "candidate_id": "exp0091", "value": 63118080, "is_reference": false }, { "step": 92, "launch_seq": 94, "candidate_id": "exp0092", "value": 63118080, "is_reference": false }, { "step": 93, "launch_seq": 95, "candidate_id": "exp0093", "value": 62675712, "is_reference": false }, { "step": 94, "launch_seq": 96, "candidate_id": "exp0094", "value": 62675712, "is_reference": false }, { "step": 95, "launch_seq": 97, "candidate_id": "exp0095", "value": 62233344, "is_reference": false }, { "step": 96, "launch_seq": 98, "candidate_id": "exp0096", "value": 62233344, "is_reference": false }, { "step": 100, "launch_seq": 102, "candidate_id": "exp0100", "value": 62233344, "is_reference": false } ], "curve": [ { "step": 1, "value": 239078400 }, { "step": 2, "value": 220204032 }, { "step": 3, "value": 210766848 }, { "step": 4, "value": 210766848 }, { "step": 5, "value": 188746752 }, { "step": 6, "value": 110888640 }, { "step": 7, "value": 110888640 }, { "step": 8, "value": 110888640 }, { "step": 9, "value": 89654976 }, { "step": 10, "value": 89654976 }, { "step": 11, "value": 89654976 }, { "step": 12, "value": 89654976 }, { "step": 13, "value": 89654976 }, { "step": 14, "value": 89654976 }, { "step": 15, "value": 89654976 }, { "step": 16, "value": 89654976 }, { "step": 17, "value": 89654976 }, { "step": 18, "value": 80219520 }, { "step": 19, "value": 80219520 }, { "step": 20, "value": 75500928 }, { "step": 21, "value": 68423040 }, { "step": 22, "value": 68423040 }, { "step": 23, "value": 68423040 }, { "step": 24, "value": 68423040 }, { "step": 25, "value": 68423040 }, { "step": 26, "value": 68423040 }, { "step": 27, "value": 68423040 }, { "step": 28, "value": 68423040 }, { "step": 29, "value": 68423040 }, { "step": 30, "value": 68423040 }, { "step": 31, "value": 68423040 }, { "step": 32, "value": 68423040 }, { "step": 33, "value": 68423040 }, { "step": 34, "value": 68423040 }, { "step": 35, "value": 64884096 }, { "step": 36, "value": 64884096 }, { "step": 37, "value": 64884096 }, { "step": 38, "value": 64884096 }, { "step": 39, "value": 64884096 }, { "step": 40, "value": 64884096 }, { "step": 41, "value": 64884096 }, { "step": 42, "value": 64884096 }, { "step": 43, "value": 64884096 }, { "step": 44, "value": 64884096 }, { "step": 45, "value": 64884096 }, { "step": 46, "value": 64884096 }, { "step": 47, "value": 64884096 }, { "step": 48, "value": 64884096 }, { "step": 49, "value": 64884096 }, { "step": 50, "value": 64884096 }, { "step": 51, "value": 64884096 }, { "step": 52, "value": 64884096 }, { "step": 53, "value": 64884096 }, { "step": 54, "value": 64884096 }, { "step": 55, "value": 64884096 }, { "step": 56, "value": 64884096 }, { "step": 57, "value": 64884096 }, { "step": 58, "value": 64884096 }, { "step": 59, "value": 64884096 }, { "step": 60, "value": 64884096 }, { "step": 61, "value": 64884096 }, { "step": 62, "value": 64884096 }, { "step": 63, "value": 64884096 }, { "step": 64, "value": 64884096 }, { "step": 65, "value": 64884096 }, { "step": 66, "value": 64884096 }, { "step": 67, "value": 63118080 }, { "step": 68, "value": 63118080 }, { "step": 69, "value": 63118080 }, { "step": 70, "value": 63118080 }, { "step": 71, "value": 63118080 }, { "step": 72, "value": 63118080 }, { "step": 73, "value": 63118080 }, { "step": 74, "value": 63118080 }, { "step": 75, "value": 63118080 }, { "step": 76, "value": 63118080 }, { "step": 77, "value": 63118080 }, { "step": 78, "value": 63118080 }, { "step": 79, "value": 63118080 }, { "step": 80, "value": 63118080 }, { "step": 81, "value": 63118080 }, { "step": 82, "value": 63118080 }, { "step": 83, "value": 63118080 }, { "step": 84, "value": 63118080 }, { "step": 85, "value": 63118080 }, { "step": 86, "value": 63118080 }, { "step": 87, "value": 63118080 }, { "step": 88, "value": 63118080 }, { "step": 89, "value": 63118080 }, { "step": 90, "value": 63118080 }, { "step": 91, "value": 63118080 }, { "step": 92, "value": 63118080 }, { "step": 93, "value": 62675712 }, { "step": 94, "value": 62675712 }, { "step": 95, "value": 62233344 }, { "step": 96, "value": 62233344 }, { "step": 97, "value": 62233344 }, { "step": 98, "value": 62233344 }, { "step": 99, "value": 62233344 }, { "step": 100, "value": 62233344 } ], "submission_sha256": "d760974e7c825df6d076f2b466d1198c0a69769c927556ac372ebacf61e1a14e", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/gear/flops/submission.json", "comparison_value": 62233344, "comparison_rank_key": [ 62233344.0, 913833984.0, -0.0003695269950472202 ], "rank": 4 }, { "run_id": "beam-search--flops", "method_id": "beam-search", "task_id": "flops", "metric": "flops_per_token_measured", "unit": "FLOPs/token", "direction": "minimize", "best_value": 69847872, "reference_value": 239078400, "relative_improvement": 0.7078453260520399, "rank_key": [ 69847872.0, 798490624.0, -0.001963588439452657 ], "best_launch_seq": 97, "best_candidate_id": "g13-08-graded-mlp-blocks123-896-double-dose-on-depth5-256", "best_is_reference": false, "budget": { "spent": 97, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 4, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 4 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": null }, "evaluation_gpu_hours": 18.02574361111111, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 239078400, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "g01-01-gqa2-ve-all-layers", "value": 226495488, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "g01-02-mlp3x-x0-token-table", "value": 213912576, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "g02-01-gqa2-ve-all-on-mlp3x", "value": 201329664, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "g02-03-dmodel384-ve-all-layers", "value": 150999552, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "g02-04-short-window-512", "value": 195038208, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "g02-05-two-query-heads", "value": 182455296, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "g02-06-depth6-fixed-width", "value": 188745984, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "g03-01-dmodel384-two-query-heads", "value": 125832192, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "g03-02-mlp3x-x0-table-on-two-query-heads", "value": 157289472, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "g03-03-mlp3x-ve-all-six-layers", "value": 169873920, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "g03-04-short-window-256", "value": 185601024, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "g03-05-dmodel384-depth6", "value": 120327552, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "g03-06-one-query-head", "value": 154142208, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "g03-07-depth5-fixed-width", "value": 163580160, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "g03-08-two-query-heads-short-window", "value": 147850752, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "g04-01-mlp3x-x0-table-on-depth6", "value": 109710720, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "g04-02-two-query-heads-depth6", "value": 100665600, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "g04-03-low-rank-lm-head", "value": 136054272, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "g04-04-dmodel384-depth7", "value": 135663552, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "g04-05-x0-token-table-on-depth6", "value": 120327552, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "g04-06-mlp3x-on-two-query-heads-384", "value": 111676416, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "g04-07-one-query-head-short-window", "value": 124257024, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "g04-08-dmodel384-short-window-512", "value": 136843776, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "g05-01-short-window-512-on-leader", "value": 94374144, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "g05-02-short-window-512-on-mlp3x-depth6", "value": 100273536, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "g05-03-one-query-head-on-mlp3x-depth8", "value": 86509056, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "g05-04-span-floor-on-x0-table-depth6", "value": 103812480, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "g05-05-x0-token-table-on-leader", "value": 100665600, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "g05-06-depth5-on-mlp3x-x0-table", "value": 96144192, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "g05-07-mlp2x-on-two-query-heads-384", "value": 97520640, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "g05-08-depth5-on-x0-table", "value": 104991552, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "g06-01-span-floor-on-leader", "value": 79431168, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "g06-02-mlp3x-on-short-window-leader", "value": 83757312, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "g06-03-span-floor-on-depth5", "value": 81988416, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "g06-04-span-floor-on-mlp2x", "value": 83364864, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "g06-05-span-floor-256-on-leader", "value": 76678656, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "g06-07-span-floor-256-on-depth5", "value": 77269824, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "g06-08-one-kv-head-on-mlp2x", "value": 92800512, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "g07-07-drop-forced-long-layer-on-leader-512", "value": 77071872, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "g07-08-lm-head-rank320-on-depth5-512", "value": 79579968, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "g08-01-lm-head-rank360-on-leader", "value": 76328448, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "g08-02-qk-channel-scales-on-no-hop-member", "value": 77071872, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "g08-03-lm-head-rank320-on-depth5-256", "value": 74861376, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "g08-04-mlp-hidden-1024-on-window512-leader", "value": 74712576, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "g08-05-interior-hop-on-leader", "value": 76678656, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "g08-06-ve-gate-uncounted-contraction", "value": 77070336, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "g08-07-window128-on-depth5", "value": 74910528, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "g09-02-mlp-hidden-1024-on-depth5-256", "value": 71912256, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "g09-03-lm-head-rank320-on-window128-depth5", "value": 72502080, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "g09-07-window64-on-depth5", "value": 73730880, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "g10-05-single-microbatch-updates-on-depth5-256", "value": 71912256, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "g11-01-graded-mlp-block0-960-on-depth5-256", "value": 71617344, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "g11-02-graded-mlp-block0-1088-single-microbatch-on-window128-depth5", "value": 72207168, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "g11-03-single-microbatch-updates-on-window64-depth5", "value": 73730880, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "g11-04-single-microbatch-updates-on-window512-leader", "value": 74712576, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "g11-06-graded-window-block0-32-with-x0-lambda-on-window128-depth5", "value": 72059712, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "g11-07-graded-window-block0-32-single-microbatch-on-window64-depth5", "value": 73583424, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "g11-08-graded-mlp-blocks01-896-single-microbatch-on-window512-leader", "value": 73532928, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "g12-01-graded-mlp-block0-832-second-rung-on-depth5-256", "value": 71027520, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "g12-02-graded-mlp-block0-960-unmatched-lr-on-depth5-256", "value": 71617344, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "g12-03-single-microbatch-updates-on-graded-window128-depth5", "value": 72059712, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "g12-04-graded-mlp-blocks01-1088-on-window128-depth5", "value": 71912256, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "g12-05-graded-window-block0-128-stacked-on-graded-mlp-depth5-256", "value": 71027520, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "g12-06-graded-window-block0-128-on-depth5-256", "value": 71322432, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "g12-07-graded-window-blocks01-32-single-microbatch-on-window128-depth5", "value": 71617344, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "g12-08-graded-mlp-last-short-block-1088-on-window128-depth5", "value": 72207168, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "g13-01-graded-mlp-blocks12-960-on-832-block0-depth5-256", "value": 70437696, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "g13-02-graded-mlp-blocks12-960-on-window128-graded-depth5-256", "value": 70437696, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "g13-03-graded-mlp-blocks01-960-on-window128-uniform-depth5-256", "value": 70732608, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "g13-04-graded-mlp-blocks123-960-unmatched-lr-on-depth5-256", "value": 70732608, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "g13-05-graded-mlp-block1-960-on-832-block0-depth5-256", "value": 70732608, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "g13-06-graded-mlp-blocks123-960-matched-lr-on-window128-depth5-256", "value": 70142784, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "g13-07-graded-mlp-blocks0123-960-unmatched-lr-on-window128-depth5-256", "value": 70142784, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "g13-08-graded-mlp-blocks123-896-double-dose-on-depth5-256", "value": 69847872, "is_reference": false } ], "curve": [ { "step": 1, "value": 239078400 }, { "step": 2, "value": 226495488 }, { "step": 3, "value": 213912576 }, { "step": 4, "value": 201329664 }, { "step": 5, "value": 201329664 }, { "step": 6, "value": 150999552 }, { "step": 7, "value": 150999552 }, { "step": 8, "value": 150999552 }, { "step": 9, "value": 150999552 }, { "step": 10, "value": 125832192 }, { "step": 11, "value": 125832192 }, { "step": 12, "value": 125832192 }, { "step": 13, "value": 125832192 }, { "step": 14, "value": 120327552 }, { "step": 15, "value": 120327552 }, { "step": 16, "value": 120327552 }, { "step": 17, "value": 120327552 }, { "step": 18, "value": 109710720 }, { "step": 19, "value": 100665600 }, { "step": 20, "value": 100665600 }, { "step": 21, "value": 100665600 }, { "step": 22, "value": 100665600 }, { "step": 23, "value": 100665600 }, { "step": 24, "value": 100665600 }, { "step": 25, "value": 100665600 }, { "step": 26, "value": 94374144 }, { "step": 27, "value": 94374144 }, { "step": 28, "value": 86509056 }, { "step": 29, "value": 86509056 }, { "step": 30, "value": 86509056 }, { "step": 31, "value": 86509056 }, { "step": 32, "value": 86509056 }, { "step": 33, "value": 86509056 }, { "step": 34, "value": 79431168 }, { "step": 35, "value": 79431168 }, { "step": 36, "value": 79431168 }, { "step": 37, "value": 79431168 }, { "step": 38, "value": 76678656 }, { "step": 39, "value": 76678656 }, { "step": 40, "value": 76678656 }, { "step": 41, "value": 76678656 }, { "step": 42, "value": 76678656 }, { "step": 43, "value": 76678656 }, { "step": 44, "value": 76678656 }, { "step": 45, "value": 76678656 }, { "step": 46, "value": 76678656 }, { "step": 47, "value": 76678656 }, { "step": 48, "value": 76678656 }, { "step": 49, "value": 76678656 }, { "step": 50, "value": 76328448 }, { "step": 51, "value": 76328448 }, { "step": 52, "value": 74861376 }, { "step": 53, "value": 74712576 }, { "step": 54, "value": 74712576 }, { "step": 55, "value": 74712576 }, { "step": 56, "value": 74712576 }, { "step": 57, "value": 74712576 }, { "step": 58, "value": 74712576 }, { "step": 59, "value": 71912256 }, { "step": 60, "value": 71912256 }, { "step": 61, "value": 71912256 }, { "step": 62, "value": 71912256 }, { "step": 63, "value": 71912256 }, { "step": 64, "value": 71912256 }, { "step": 65, "value": 71912256 }, { "step": 66, "value": 71912256 }, { "step": 67, "value": 71912256 }, { "step": 68, "value": 71912256 }, { "step": 69, "value": 71912256 }, { "step": 70, "value": 71912256 }, { "step": 71, "value": 71912256 }, { "step": 72, "value": 71912256 }, { "step": 73, "value": 71912256 }, { "step": 74, "value": 71617344 }, { "step": 75, "value": 71617344 }, { "step": 76, "value": 71617344 }, { "step": 77, "value": 71617344 }, { "step": 78, "value": 71617344 }, { "step": 79, "value": 71617344 }, { "step": 80, "value": 71617344 }, { "step": 81, "value": 71617344 }, { "step": 82, "value": 71027520 }, { "step": 83, "value": 71027520 }, { "step": 84, "value": 71027520 }, { "step": 85, "value": 71027520 }, { "step": 86, "value": 71027520 }, { "step": 87, "value": 71027520 }, { "step": 88, "value": 71027520 }, { "step": 89, "value": 71027520 }, { "step": 90, "value": 70437696 }, { "step": 91, "value": 70437696 }, { "step": 92, "value": 70437696 }, { "step": 93, "value": 70437696 }, { "step": 94, "value": 70437696 }, { "step": 95, "value": 70142784 }, { "step": 96, "value": 70142784 }, { "step": 97, "value": 69847872 } ], "submission_sha256": "1cc074900c7b347fe0e929d8e533edbe3537d3555bde78291285ff8a2f35d6ae", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/beam-search/flops/submission.json", "comparison_value": 69847872, "comparison_rank_key": [ 69847872.0, 798490624.0, -0.001963588439452657 ], "rank": 5 }, { "run_id": "heuresis-map-elites--flops", "method_id": "heuresis-map-elites", "task_id": "flops", "metric": "flops_per_token_measured", "unit": "FLOPs/token", "direction": "minimize", "best_value": 84154368, "reference_value": 239078400, "relative_improvement": 0.6480051397365885, "rank_key": [ 84154368.0, 687472640.0, -0.002042581136498356 ], "best_launch_seq": 66, "best_candidate_id": "exp-cc28f0fdf0e9952f7019872c", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 18.877430833333328, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 239078400, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-c7cccefdb1b5d28badfed16f", "value": 179308800, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp-23a6e0c619fe910ab6290046", "value": 165153024, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-3bbfbd229b40d8738bb1c275", "value": 158861568, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-ce2b1ac5d0bf03bda0443059", "value": 172428288, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp-0abf5571f7ec527a5355d6c8", "value": 159154176, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp-34cc107b0f688e6aaedb5d8d", "value": 163580928, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-f00156ccc316a3949e7fa5f6", "value": 150210048, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-f5801edc43d09520cffd1b8d", "value": 151004160, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-1c16a7750be414ad5eb211af", "value": 114040320, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-e8b171c3b02fdb5f15558015", "value": 154144512, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-f831a59dfe9cf4c7f10b66f6", "value": 130551552, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-9eefe448e4baad9314895d27", "value": 170234880, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-0d0e0672e53d754971c7e1f1", "value": 157811712, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-e35ffad19e6fbe328195d60c", "value": 157762560, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-2a92bac8d9d3637abc1079bc", "value": 154380288, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-c2cc06e1c0cd2d74eda8a45b", "value": 193468416, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-7887b0e69ac2d306ced02aa9", "value": 188749824, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp-899a1a554533ae6a423ac321", "value": 180096000, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp-15c8b8ace309d94081abc87d", "value": 176950272, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-d70f91cada8eb2cac0f5e72c", "value": 173803008, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp-2c5c7a2d7d47265e24f9e1c1", "value": 182848512, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp-fe13a4b3c03c7f47a6a34c6f", "value": 155323392, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp-deabaa67cd0a9105aed1a0ca", "value": 229936128, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp-1eb3c843f5834778c1c0490c", "value": 162797568, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-9a0f4b7595518ec841ea06d3", "value": 156506112, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-ada30693f1e0c9b4f9fc52af", "value": 131340288, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-9ebe7ca8ee142f06e67b5274", "value": 118757376, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-da30c1025ad6a849a4c9a32d", "value": 106174464, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp-bb3b7dc43ef88163f6ce3d15", "value": 99883008, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp-5913b4f39c675a3a7c716a38", "value": 93591552, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp-93fa5dced26cb7e1094ad9cd", "value": 90839040, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp-d0369f681e6b76d73f867fe2", "value": 210769920, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp-29cb41c9553c7d2601c2e37a", "value": 137631744, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp-b1cafb6eb601b35325abfe49", "value": 106174464, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp-085e8e8717355e084697d11b", "value": 93591552, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "exp-94587e03229027b51adb6022", "value": 90445824, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp-cbc17d6f80aded2d64125189", "value": 89659392, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "exp-876965868df9e57268f3deda", "value": 89266176, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp-54ec38c89ced87ab7c420e2e", "value": 201332736, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp-12d757aa2fb27f513fdcd5dd", "value": 163584000, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp-0a7009e36ed733e80d746d39", "value": 151001088, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp-2cda63a327ce1d378b9587b5", "value": 144709632, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "exp-531189e0c40f50bc67a077b6", "value": 210769920, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "exp-dfbcdd010dfdea0e995de832", "value": 150214656, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "exp-cab00f536e30299d05d80b3c", "value": 123475968, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "exp-2f66a86652a7b9d48137bfcb", "value": 106174464, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp-432b583b63bd9d334cc782ff", "value": 87300096, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "exp-12244c71b78d4b2595281776", "value": 84154368, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp-cc28f0fdf0e9952f7019872c", "value": 84154368, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "exp-9483bc09a9e38cac096ada29", "value": 198187008, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "exp-fd081d6f6749930ef1a0c6a6", "value": 162011136, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp-d5c9a8b508299cf1051832f5", "value": 190321152, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "exp-b9d09e9cb56b5b1ab68d1227", "value": 145492224, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "exp-60c735a11b1b1d8bcfc00763", "value": 134482176, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "exp-11a7ffa229ae497de03ac85b", "value": 125831424, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "exp-af16fc62532feed8a6148079", "value": 102779136, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "exp-681eaecef9204c9d95dac090", "value": 101796096, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "exp-ea43ee23d43002bb723619c1", "value": 100813056, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "exp-00809f0d83a40b0e9ddd0ab1", "value": 99830016, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "exp-f821b6e30691fb2d335b40fe", "value": 176166144, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "exp-257f2d8719c6b3344eb17e82", "value": 134090496, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "exp-04edebe9097a9ec6de54c36a", "value": 125831808, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "exp-921360cfb43894bdbefaab16", "value": 117573120, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "exp-a56222bdea837fd86debb232", "value": 198205440, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp-33133e4deab7b94e0e9a5dc4", "value": 193486848, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "exp-b4c7ce94a382e548c0cfbffc", "value": 181690368, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "exp-b6ca0feab73be6726ca9faef", "value": 176185344, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "exp-b6a64a34efe1943fbf29bce1", "value": 169881600, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "exp-f735f55f4ee1a676c8be7dcd", "value": 193468416, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "exp-2c16e16c4442e8322f15fe2b", "value": 187963392, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "exp-ffb70c1e7f2073d60cddc242", "value": 176166912, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "exp-189ea25bf06bcd6d134be6bd", "value": 163584000, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "exp-2da9a2e4582e11ccbf65d3e8", "value": 163584000, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "exp-5019433e8198c2d32142d094", "value": 191109120, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "exp-0fdad7f4fda291aa95d43ac2", "value": 159651840, "is_reference": false }, { "step": 98, "launch_seq": 98, "candidate_id": "exp-2e12f908a09a56e15b35a55e", "value": 147068928, "is_reference": false }, { "step": 99, "launch_seq": 99, "candidate_id": "exp-2871a7e6f2730ca8eac5cd59", "value": 134486016, "is_reference": false } ], "curve": [ { "step": 1, "value": 239078400 }, { "step": 2, "value": 179308800 }, { "step": 3, "value": 165153024 }, { "step": 4, "value": 158861568 }, { "step": 5, "value": 158861568 }, { "step": 6, "value": 158861568 }, { "step": 7, "value": 158861568 }, { "step": 8, "value": 158861568 }, { "step": 9, "value": 158861568 }, { "step": 10, "value": 150210048 }, { "step": 11, "value": 150210048 }, { "step": 12, "value": 114040320 }, { "step": 13, "value": 114040320 }, { "step": 14, "value": 114040320 }, { "step": 15, "value": 114040320 }, { "step": 16, "value": 114040320 }, { "step": 17, "value": 114040320 }, { "step": 18, "value": 114040320 }, { "step": 19, "value": 114040320 }, { "step": 20, "value": 114040320 }, { "step": 21, "value": 114040320 }, { "step": 22, "value": 114040320 }, { "step": 23, "value": 114040320 }, { "step": 24, "value": 114040320 }, { "step": 25, "value": 114040320 }, { "step": 26, "value": 114040320 }, { "step": 27, "value": 114040320 }, { "step": 28, "value": 114040320 }, { "step": 29, "value": 114040320 }, { "step": 30, "value": 114040320 }, { "step": 31, "value": 114040320 }, { "step": 32, "value": 114040320 }, { "step": 33, "value": 114040320 }, { "step": 34, "value": 114040320 }, { "step": 35, "value": 114040320 }, { "step": 36, "value": 106174464 }, { "step": 37, "value": 99883008 }, { "step": 38, "value": 93591552 }, { "step": 39, "value": 93591552 }, { "step": 40, "value": 93591552 }, { "step": 41, "value": 90839040 }, { "step": 42, "value": 90839040 }, { "step": 43, "value": 90839040 }, { "step": 44, "value": 90839040 }, { "step": 45, "value": 90839040 }, { "step": 46, "value": 90839040 }, { "step": 47, "value": 90839040 }, { "step": 48, "value": 90839040 }, { "step": 49, "value": 90445824 }, { "step": 50, "value": 90445824 }, { "step": 51, "value": 90445824 }, { "step": 52, "value": 89659392 }, { "step": 53, "value": 89266176 }, { "step": 54, "value": 89266176 }, { "step": 55, "value": 89266176 }, { "step": 56, "value": 89266176 }, { "step": 57, "value": 89266176 }, { "step": 58, "value": 89266176 }, { "step": 59, "value": 89266176 }, { "step": 60, "value": 89266176 }, { "step": 61, "value": 89266176 }, { "step": 62, "value": 89266176 }, { "step": 63, "value": 87300096 }, { "step": 64, "value": 87300096 }, { "step": 65, "value": 84154368 }, { "step": 66, "value": 84154368 }, { "step": 67, "value": 84154368 }, { "step": 68, "value": 84154368 }, { "step": 69, "value": 84154368 }, { "step": 70, "value": 84154368 }, { "step": 71, "value": 84154368 }, { "step": 72, "value": 84154368 }, { "step": 73, "value": 84154368 }, { "step": 74, "value": 84154368 }, { "step": 75, "value": 84154368 }, { "step": 76, "value": 84154368 }, { "step": 77, "value": 84154368 }, { "step": 78, "value": 84154368 }, { "step": 79, "value": 84154368 }, { "step": 80, "value": 84154368 }, { "step": 81, "value": 84154368 }, { "step": 82, "value": 84154368 }, { "step": 83, "value": 84154368 }, { "step": 84, "value": 84154368 }, { "step": 85, "value": 84154368 }, { "step": 86, "value": 84154368 }, { "step": 87, "value": 84154368 }, { "step": 88, "value": 84154368 }, { "step": 89, "value": 84154368 }, { "step": 90, "value": 84154368 }, { "step": 91, "value": 84154368 }, { "step": 92, "value": 84154368 }, { "step": 93, "value": 84154368 }, { "step": 94, "value": 84154368 }, { "step": 95, "value": 84154368 }, { "step": 96, "value": 84154368 }, { "step": 97, "value": 84154368 }, { "step": 98, "value": 84154368 }, { "step": 99, "value": 84154368 }, { "step": 100, "value": 84154368 } ], "submission_sha256": "51a11cfb88d8b07eee730b25fc85d60239d8967a5a799b2609bc36383a01bff9", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/heuresis-map-elites/flops/submission.json", "comparison_value": 84154368, "comparison_rank_key": [ 84154368.0, 687472640.0, -0.002042581136498356 ], "rank": 6 }, { "run_id": "tpe--flops", "method_id": "tpe", "task_id": "flops", "metric": "flops_per_token_measured", "unit": "FLOPs/token", "direction": "minimize", "best_value": 100271808, "reference_value": 239078400, "relative_improvement": 0.5805902666238355, "rank_key": [ 100271808.0, 793772032.0, -0.009305019012407545 ], "best_launch_seq": 78, "best_candidate_id": "tpe-trial-0077", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 15.649004166666662, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 239078400, "is_reference": true }, { "step": 14, "launch_seq": 14, "candidate_id": "tpe-trial-0013", "value": 161614656, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "tpe-trial-0014", "value": 176950080, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "tpe-trial-0015", "value": 192289536, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "tpe-trial-0016", "value": 176952960, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "tpe-trial-0017", "value": 161614656, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "tpe-trial-0021", "value": 176950080, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "tpe-trial-0024", "value": 115607232, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "tpe-trial-0026", "value": 100271808, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "tpe-trial-0028", "value": 119146176, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "tpe-trial-0034", "value": 100271808, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "tpe-trial-0035", "value": 104990400, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "tpe-trial-0036", "value": 104990400, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "tpe-trial-0038", "value": 163579008, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "tpe-trial-0039", "value": 135661824, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "tpe-trial-0042", "value": 220204032, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "tpe-trial-0046", "value": 160434432, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "tpe-trial-0048", "value": 213912576, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "tpe-trial-0051", "value": 182454528, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "tpe-trial-0053", "value": 109708992, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "tpe-trial-0059", "value": 207621120, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "tpe-trial-0061", "value": 169871616, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "tpe-trial-0064", "value": 213911808, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "tpe-trial-0068", "value": 207621120, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "tpe-trial-0070", "value": 115607232, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "tpe-trial-0071", "value": 115607232, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "tpe-trial-0072", "value": 115607232, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "tpe-trial-0074", "value": 207621120, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "tpe-trial-0075", "value": 115607232, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "tpe-trial-0077", "value": 100271808, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "tpe-trial-0078", "value": 100271808, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "tpe-trial-0079", "value": 104990400, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "tpe-trial-0082", "value": 104990400, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "tpe-trial-0083", "value": 104990400, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "tpe-trial-0085", "value": 213912576, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "tpe-trial-0087", "value": 104990400, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "tpe-trial-0088", "value": 239078400, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "tpe-trial-0090", "value": 104990400, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "tpe-trial-0091", "value": 104990400, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "tpe-trial-0092-r1", "value": 104990400, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "tpe-trial-0094", "value": 120325824, "is_reference": false } ], "curve": [ { "step": 1, "value": 239078400 }, { "step": 2, "value": 239078400 }, { "step": 3, "value": 239078400 }, { "step": 4, "value": 239078400 }, { "step": 5, "value": 239078400 }, { "step": 6, "value": 239078400 }, { "step": 7, "value": 239078400 }, { "step": 8, "value": 239078400 }, { "step": 9, "value": 239078400 }, { "step": 10, "value": 239078400 }, { "step": 11, "value": 239078400 }, { "step": 12, "value": 239078400 }, { "step": 13, "value": 239078400 }, { "step": 14, "value": 161614656 }, { "step": 15, "value": 161614656 }, { "step": 16, "value": 161614656 }, { "step": 17, "value": 161614656 }, { "step": 18, "value": 161614656 }, { "step": 19, "value": 161614656 }, { "step": 20, "value": 161614656 }, { "step": 21, "value": 161614656 }, { "step": 22, "value": 161614656 }, { "step": 23, "value": 161614656 }, { "step": 24, "value": 161614656 }, { "step": 25, "value": 115607232 }, { "step": 26, "value": 115607232 }, { "step": 27, "value": 100271808 }, { "step": 28, "value": 100271808 }, { "step": 29, "value": 100271808 }, { "step": 30, "value": 100271808 }, { "step": 31, "value": 100271808 }, { "step": 32, "value": 100271808 }, { "step": 33, "value": 100271808 }, { "step": 34, "value": 100271808 }, { "step": 35, "value": 100271808 }, { "step": 36, "value": 100271808 }, { "step": 37, "value": 100271808 }, { "step": 38, "value": 100271808 }, { "step": 39, "value": 100271808 }, { "step": 40, "value": 100271808 }, { "step": 41, "value": 100271808 }, { "step": 42, "value": 100271808 }, { "step": 43, "value": 100271808 }, { "step": 44, "value": 100271808 }, { "step": 45, "value": 100271808 }, { "step": 46, "value": 100271808 }, { "step": 47, "value": 100271808 }, { "step": 48, "value": 100271808 }, { "step": 49, "value": 100271808 }, { "step": 50, "value": 100271808 }, { "step": 51, "value": 100271808 }, { "step": 52, "value": 100271808 }, { "step": 53, "value": 100271808 }, { "step": 54, "value": 100271808 }, { "step": 55, "value": 100271808 }, { "step": 56, "value": 100271808 }, { "step": 57, "value": 100271808 }, { "step": 58, "value": 100271808 }, { "step": 59, "value": 100271808 }, { "step": 60, "value": 100271808 }, { "step": 61, "value": 100271808 }, { "step": 62, "value": 100271808 }, { "step": 63, "value": 100271808 }, { "step": 64, "value": 100271808 }, { "step": 65, "value": 100271808 }, { "step": 66, "value": 100271808 }, { "step": 67, "value": 100271808 }, { "step": 68, "value": 100271808 }, { "step": 69, "value": 100271808 }, { "step": 70, "value": 100271808 }, { "step": 71, "value": 100271808 }, { "step": 72, "value": 100271808 }, { "step": 73, "value": 100271808 }, { "step": 74, "value": 100271808 }, { "step": 75, "value": 100271808 }, { "step": 76, "value": 100271808 }, { "step": 77, "value": 100271808 }, { "step": 78, "value": 100271808 }, { "step": 79, "value": 100271808 }, { "step": 80, "value": 100271808 }, { "step": 81, "value": 100271808 }, { "step": 82, "value": 100271808 }, { "step": 83, "value": 100271808 }, { "step": 84, "value": 100271808 }, { "step": 85, "value": 100271808 }, { "step": 86, "value": 100271808 }, { "step": 87, "value": 100271808 }, { "step": 88, "value": 100271808 }, { "step": 89, "value": 100271808 }, { "step": 90, "value": 100271808 }, { "step": 91, "value": 100271808 }, { "step": 92, "value": 100271808 }, { "step": 93, "value": 100271808 }, { "step": 94, "value": 100271808 }, { "step": 95, "value": 100271808 }, { "step": 96, "value": 100271808 }, { "step": 97, "value": 100271808 }, { "step": 98, "value": 100271808 }, { "step": 99, "value": 100271808 }, { "step": 100, "value": 100271808 } ], "submission_sha256": "61c65193b7d3554e5f0365e0c885e2db1066a53da7a080fa26cc5e75b2ad4ab3", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/tpe/flops/submission.json", "comparison_value": 100271808, "comparison_rank_key": [ 100271808.0, 793772032.0, -0.009305019012407545 ], "rank": 7 } ] }, { "id": "kvcache", "title": "GPU-resident KV-cache allocation at validation BPB < 1.05", "metric": "kv_cache_bytes", "unit": "bytes", "direction": "minimize", "budget": 100, "comparison_step": null, "quality_gate": { "metric": "val_bpb", "operator": "<", "value": 1.05 }, "tiebreaks": [ { "metric": "nopref_kv_cache_bytes", "direction": "minimize" } ], "runs": [ { "run_id": "sequential-search--kvcache", "method_id": "sequential-search", "task_id": "kvcache", "metric": "kv_cache_bytes", "unit": "bytes", "direction": "minimize", "best_value": 1024, "reference_value": 34603520, "relative_improvement": 0.9999704076348302, "rank_key": [ 1024.0, 1024.0, -0.00251824443428883 ], "best_launch_seq": 58, "best_candidate_id": "kvbook625b", "best_is_reference": false, "budget": { "spent": 62, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": null }, "evaluation_gpu_hours": 12.56002, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 34603520, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "winext", "value": 26214912, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "kvgroup2", "value": 10492416, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "kvgroup4", "value": 5249536, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "kvfuse", "value": 5246464, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "kvids0", "value": 4726272, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "kvspan1024", "value": 3678720, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "kvspan512", "value": 1843712, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "kvve7", "value": 1843712, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "kvhist512", "value": 1840128, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "batch262k", "value": 1840128, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "kvwin256c", "value": 922624, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "kvhd64", "value": 463872, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "kvint8b", "value": 241664, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "batch131k", "value": 241664, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "kvshare4b", "value": 140288, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "kvrebuild12", "value": 107520, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "kvrebuild1234b", "value": 74752, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "kvidsbook", "value": 71168, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "kvk6", "value": 62976, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "kvkrms", "value": 61952, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "kvv6merge", "value": 53760, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "kvidpack", "value": 53248, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "kvcode5", "value": 45056, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "kvidpack5b", "value": 44544, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "kvstream3ve6", "value": 27648, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "kvmixk5", "value": 25600, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "kvmerge5", "value": 23552, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "kvw743", "value": 23040, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "kvw429", "value": 22528, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "kvtop192", "value": 17408, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "kvtop166", "value": 15360, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "kvfold384", "value": 1536, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "kvbook625", "value": 1024, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "kvbook625b", "value": 1024, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "x0lr005", "value": 1024, "is_reference": false } ], "curve": [ { "step": 1, "value": 34603520 }, { "step": 2, "value": 26214912 }, { "step": 3, "value": 10492416 }, { "step": 4, "value": 10492416 }, { "step": 5, "value": 5249536 }, { "step": 6, "value": 5246464 }, { "step": 7, "value": 4726272 }, { "step": 8, "value": 4726272 }, { "step": 9, "value": 4726272 }, { "step": 10, "value": 3678720 }, { "step": 11, "value": 1843712 }, { "step": 12, "value": 1843712 }, { "step": 13, "value": 1843712 }, { "step": 14, "value": 1843712 }, { "step": 15, "value": 1840128 }, { "step": 16, "value": 1840128 }, { "step": 17, "value": 1840128 }, { "step": 18, "value": 1840128 }, { "step": 19, "value": 1840128 }, { "step": 20, "value": 1840128 }, { "step": 21, "value": 922624 }, { "step": 22, "value": 463872 }, { "step": 23, "value": 241664 }, { "step": 24, "value": 241664 }, { "step": 25, "value": 241664 }, { "step": 26, "value": 140288 }, { "step": 27, "value": 107520 }, { "step": 28, "value": 107520 }, { "step": 29, "value": 74752 }, { "step": 30, "value": 71168 }, { "step": 31, "value": 62976 }, { "step": 32, "value": 61952 }, { "step": 33, "value": 53760 }, { "step": 34, "value": 53248 }, { "step": 35, "value": 45056 }, { "step": 36, "value": 45056 }, { "step": 37, "value": 44544 }, { "step": 38, "value": 44544 }, { "step": 39, "value": 44544 }, { "step": 40, "value": 44544 }, { "step": 41, "value": 44544 }, { "step": 42, "value": 44544 }, { "step": 43, "value": 44544 }, { "step": 44, "value": 44544 }, { "step": 45, "value": 27648 }, { "step": 46, "value": 27648 }, { "step": 47, "value": 25600 }, { "step": 48, "value": 23552 }, { "step": 49, "value": 23040 }, { "step": 50, "value": 22528 }, { "step": 51, "value": 17408 }, { "step": 52, "value": 17408 }, { "step": 53, "value": 15360 }, { "step": 54, "value": 15360 }, { "step": 55, "value": 15360 }, { "step": 56, "value": 1536 }, { "step": 57, "value": 1024 }, { "step": 58, "value": 1024 }, { "step": 59, "value": 1024 }, { "step": 60, "value": 1024 }, { "step": 61, "value": 1024 }, { "step": 62, "value": 1024 } ], "submission_sha256": "e1427627897ffb0e51f81aeb3e8e0dfc68be3e0529c09cfdfad9ebab932588ec", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/sequential-search/kvcache/submission.json", "comparison_value": 1024, "comparison_rank_key": [ 1024.0, 1024.0, -0.00251824443428883 ], "rank": 1 }, { "run_id": "autoscientists--kvcache", "method_id": "autoscientists", "task_id": "kvcache", "metric": "kv_cache_bytes", "unit": "bytes", "direction": "minimize", "best_value": 59392, "reference_value": 34603520, "relative_improvement": 0.9982836428201524, "rank_key": [ 59392.0, 59392.0, -0.0001468756818452377 ], "best_launch_seq": 56, "best_candidate_id": "exp-fc7c6a8b9340706a67b16a08", "best_is_reference": false, "budget": { "spent": 65, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 2, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 2 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 13.708323888888891, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 34603520, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-4bffbeaa595e99212b687fa1", "value": 26215424, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp-58ba98b3ced7a2291b95e1c0", "value": 18088448, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-9a4352a9d3d00ce3b9c2823c", "value": 21378048, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-90d672afe764eeaf5738cd48", "value": 8520192, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-f5731f98b22e8c24df58b3a5", "value": 10856960, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp-d103f28767d3c97fde5c62ff", "value": 6376448, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-0dd6800a9a67236238307c5d", "value": 5430272, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp-cdbed35486a3a9c3cea01e99", "value": 5344256, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-fd71d2cc5345b9be14433475", "value": 3832832, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-8186ada45c6b98eafc97b49c", "value": 1920000, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-d644b92ec8887125ceeb88c5", "value": 1876992, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-80bb2342f00b9c3be26fd9bc", "value": 1082368, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp-a2c1c7fd4414c96c739fe29e", "value": 1477632, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp-bdd23e3dda64958ed25a648c", "value": 549888, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp-302e6fc8f1a0907e0312942f", "value": 683520, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-445b0c3c453a01cf86b4ee24", "value": 275456, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp-433a26f34f4f9f0b70179fcb", "value": 272384, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp-9e725ddbf5fc26177d2aed16", "value": 136704, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp-655bd25a328f2a3d929fa5ae", "value": 269312, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp-ba337b51c506c4779330a933", "value": 136192, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-f65cf9385ed273655376ec6d", "value": 135168, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp-dcec5cdff62e691622a8d1b7", "value": 134144, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp-2fe44f3827a26b2d8a1a379e", "value": 116736, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "exp-a082aa9d42dfb4f117c84f78", "value": 134144, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp-06b1d395c6fe1d62f84c433c", "value": 91648, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp-c6af780506dd03f58a1141fe", "value": 83968, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp-3e6d8b72c9aaef92c07458ac", "value": 78336, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp-a22f46759681ddd17fe6d7c3", "value": 77824, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp-e18765d5dc7036ab55cd5b9d", "value": 73216, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "exp-c6fd45b51ef81c2a4044e75d", "value": 71168, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp-94b42299e221bf8ae0f02533", "value": 70656, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp-336504a131c6ad71289f1e5d", "value": 69632, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp-c820c9c41353c703a91f8b44", "value": 65024, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp-a6332f5fa963d0d8444dcd8a", "value": 64512, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp-fc4e1e4d38ffe95fabca1c82", "value": 60928, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp-fc7c6a8b9340706a67b16a08", "value": 59392, "is_reference": false } ], "curve": [ { "step": 1, "value": 34603520 }, { "step": 2, "value": 26215424 }, { "step": 3, "value": 18088448 }, { "step": 4, "value": 18088448 }, { "step": 5, "value": 8520192 }, { "step": 6, "value": 8520192 }, { "step": 7, "value": 8520192 }, { "step": 8, "value": 6376448 }, { "step": 9, "value": 6376448 }, { "step": 10, "value": 5430272 }, { "step": 11, "value": 5430272 }, { "step": 12, "value": 5430272 }, { "step": 13, "value": 5430272 }, { "step": 14, "value": 5430272 }, { "step": 15, "value": 5430272 }, { "step": 16, "value": 5344256 }, { "step": 17, "value": 3832832 }, { "step": 18, "value": 3832832 }, { "step": 19, "value": 1920000 }, { "step": 20, "value": 1876992 }, { "step": 21, "value": 1082368 }, { "step": 22, "value": 1082368 }, { "step": 23, "value": 1082368 }, { "step": 24, "value": 549888 }, { "step": 25, "value": 549888 }, { "step": 26, "value": 275456 }, { "step": 27, "value": 275456 }, { "step": 28, "value": 272384 }, { "step": 29, "value": 136704 }, { "step": 30, "value": 136704 }, { "step": 31, "value": 136192 }, { "step": 32, "value": 136192 }, { "step": 33, "value": 136192 }, { "step": 34, "value": 135168 }, { "step": 35, "value": 135168 }, { "step": 36, "value": 135168 }, { "step": 37, "value": 135168 }, { "step": 38, "value": 134144 }, { "step": 39, "value": 134144 }, { "step": 40, "value": 134144 }, { "step": 41, "value": 116736 }, { "step": 42, "value": 116736 }, { "step": 43, "value": 91648 }, { "step": 44, "value": 83968 }, { "step": 45, "value": 83968 }, { "step": 46, "value": 78336 }, { "step": 47, "value": 77824 }, { "step": 48, "value": 73216 }, { "step": 49, "value": 71168 }, { "step": 50, "value": 70656 }, { "step": 51, "value": 69632 }, { "step": 52, "value": 65024 }, { "step": 53, "value": 65024 }, { "step": 54, "value": 64512 }, { "step": 55, "value": 60928 }, { "step": 56, "value": 59392 }, { "step": 57, "value": 59392 }, { "step": 58, "value": 59392 }, { "step": 59, "value": 59392 }, { "step": 60, "value": 59392 }, { "step": 61, "value": 59392 }, { "step": 62, "value": 59392 }, { "step": 63, "value": 59392 }, { "step": 64, "value": 59392 }, { "step": 65, "value": 59392 } ], "submission_sha256": "7d08b4c88845a710bd6039a9c0bd71a14d8cc18b6a613774523e4c77f09dea1f", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/autoscientists/kvcache/submission.json", "comparison_value": 59392, "comparison_rank_key": [ 59392.0, 59392.0, -0.0001468756818452377 ], "rank": 2 }, { "run_id": "beam-search--kvcache", "method_id": "beam-search", "task_id": "kvcache", "metric": "kv_cache_bytes", "unit": "bytes", "direction": "minimize", "best_value": 69632, "reference_value": 34603520, "relative_improvement": 0.9979877191684545, "rank_key": [ 69632.0, 69632.0, -0.0004540638239545203 ], "best_launch_seq": 93, "best_candidate_id": "g14-c03-sixty-four-positions-off-each-pair-of-the-injected-member", "best_is_reference": false, "budget": { "spent": 99, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 4, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 4 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": null }, "evaluation_gpu_hours": 19.686913611111105, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 34603520, "is_reference": true }, { "step": 4, "launch_seq": 4, "candidate_id": "g02-c01-window-extent-ring-retention", "value": 26215424, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "g02-c02-grouped-query-kv-heads", "value": 16777728, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "g03-c01-ring-retention-on-grouped-kv", "value": 10492928, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "g03-c03-quarter-span-short-layers", "value": 15741952, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "g03-c05-cross-layer-kv-sharing", "value": 22021120, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "g03-c02-single-key-value-head", "value": 8389120, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "g03-c04-six-layer-depth-at-fixed-width", "value": 19923968, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "g04-c05-single-kv-head-on-quarter-span", "value": 3677184, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "g04-c07-six-layer-depth-on-grouped-kv", "value": 12583424, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "g04-c04-no-full-context-layer", "value": 4203520, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "g04-c08-token-shift-mixer-layers", "value": 8393216, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "g05-c03-single-cached-kv-head-on-uniform-quarter-span", "value": 2106368, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "g05-c05-per-query-head-attention-temperature", "value": 8389120, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "g05-c02-mixer-for-the-full-context-layers", "value": 1582080, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "g05-c04-paired-cache-sharing-with-a-causal-follower-read", "value": 2627584, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "g05-c06-value-residual-on-every-layer", "value": 8389120, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "g05-c08-quarter-width-cached-row-on-the-mixer-parent", "value": 2101760, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "g06-c01-quarter-width-cached-row-on-the-mixer-leader", "value": 795648, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "g06-c03-window-bounded-ring-on-a-uniform-quarter-span", "value": 533504, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "g06-c05-int8-cached-store-with-per-position-scales", "value": 1082368, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "g06-c07-single-cached-kv-head-under-follower-sharing", "value": 1316864, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "g06-c02-four-cache-free-mixer-layers", "value": 1057792, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "g06-c06-eighth-context-retention-span", "value": 1057792, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "g07-c01-paired-leader-follower-sharing-across-the-mixers", "value": 269312, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "g08-c01-block-exact-retention-extent", "value": 267264, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "g08-c03-single-pair-leader-follower-on-the-ring", "value": 401408, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "g08-c05-half-row-int8-store-at-thirty-two-channel-scales", "value": 427520, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "g08-c07-lower-pair-eighth-extent-on-the-wide-row", "value": 664576, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "g08-c02-token-value-injection-on-the-cache-free-layers", "value": 269312, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "g08-c04-lower-half-retention-halving", "value": 402432, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "g08-c06-seven-layer-stack-at-fixed-width", "value": 663552, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "g08-c08-paired-sharing-on-the-wide-row", "value": 531456, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "g09-c01-one-allocation-decode-state", "value": 266752, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "g09-c03-low-rank-previous-position-state", "value": 267264, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "g09-c05-cached-layer-becomes-a-cache-free-mixer", "value": 270336, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "g09-c07-two-leader-follower-pairs-on-the-halved-stack", "value": 203776, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "g09-c02-int8-ring-payload-at-sixteen-channel-scales", "value": 152576, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "g09-c04-half-extent-on-the-shared-lower-pair", "value": 202752, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "g09-c06-eighth-extent-on-the-unpaired-caches-funded-by-injection", "value": 202752, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "g10-c01-one-allocation-for-the-int8-ring-state", "value": 152064, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "g10-c05-int8-ring-payload-on-the-shared-pair-lineage", "value": 115712, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "g10-c07-block-exact-extents-on-both-pairs", "value": 201728, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "g10-c02-token-indexed-injection-on-the-int8-leader", "value": 152576, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "g10-c08-int8-ring-payload-on-the-two-pair-stack", "value": 117760, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "g11-c03-one-allocation-for-the-two-pair-state", "value": 115200, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "g11-c02-intermediate-extent-rung-on-the-upper-pair", "value": 97280, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "g11-c04-token-indexed-injection-on-the-two-pair-stack", "value": 117760, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "g11-c06-one-absmax-per-cached-row", "value": 139776, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "g12-c03-quarter-rung-on-the-lower-pair", "value": 106496, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "g12-c07-three-eighths-off-the-upper-pair", "value": 90112, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "g12-c02-token-indexed-injection-at-the-remaining-mixers", "value": 97280, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "g12-c06-token-indexed-injection-on-the-fused-two-pair-state", "value": 115200, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "g12-c08-one-allocation-for-the-injection-funded-state", "value": 115200, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "g13-c03-quarter-rung-on-the-lower-pair-of-the-injected-member", "value": 88064, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "g13-c05-three-eighths-off-the-widest-slot", "value": 78848, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "g13-c07-quarter-rung-on-the-widest-slot-of-the-fused-state", "value": 96768, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "g13-c02-token-indexed-injection-at-the-last-two-mixers", "value": 90112, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "g13-c08-one-absmax-per-cached-row-on-the-fused-state", "value": 105984, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "g14-c03-sixty-four-positions-off-each-pair-of-the-injected-member", "value": 69632, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "g14-c07-power-of-two-cached-scale-exponents-on-the-retained-member", "value": 86016, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "g14-c05-one-absmax-per-cached-row-on-the-fused-state", "value": 89088, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "g14-c04-power-of-two-cached-scale-exponents-on-the-injected-member", "value": 83968, "is_reference": false }, { "step": 98, "launch_seq": 98, "candidate_id": "g14-c08-int8-packed-retained-previous-position", "value": 88576, "is_reference": false }, { "step": 99, "launch_seq": 99, "candidate_id": "g14-c06-sixty-four-positions-off-each-slot-of-the-fused-state", "value": 78336, "is_reference": false } ], "curve": [ { "step": 1, "value": 34603520 }, { "step": 2, "value": 34603520 }, { "step": 3, "value": 34603520 }, { "step": 4, "value": 26215424 }, { "step": 5, "value": 16777728 }, { "step": 6, "value": 10492928 }, { "step": 7, "value": 10492928 }, { "step": 8, "value": 10492928 }, { "step": 9, "value": 8389120 }, { "step": 10, "value": 8389120 }, { "step": 11, "value": 8389120 }, { "step": 12, "value": 8389120 }, { "step": 13, "value": 8389120 }, { "step": 14, "value": 3677184 }, { "step": 15, "value": 3677184 }, { "step": 16, "value": 3677184 }, { "step": 17, "value": 3677184 }, { "step": 18, "value": 3677184 }, { "step": 19, "value": 3677184 }, { "step": 20, "value": 3677184 }, { "step": 21, "value": 2106368 }, { "step": 22, "value": 2106368 }, { "step": 23, "value": 2106368 }, { "step": 24, "value": 1582080 }, { "step": 25, "value": 1582080 }, { "step": 26, "value": 1582080 }, { "step": 27, "value": 1582080 }, { "step": 28, "value": 795648 }, { "step": 29, "value": 533504 }, { "step": 30, "value": 533504 }, { "step": 31, "value": 533504 }, { "step": 32, "value": 533504 }, { "step": 33, "value": 533504 }, { "step": 34, "value": 533504 }, { "step": 35, "value": 533504 }, { "step": 36, "value": 269312 }, { "step": 37, "value": 269312 }, { "step": 38, "value": 269312 }, { "step": 39, "value": 269312 }, { "step": 40, "value": 269312 }, { "step": 41, "value": 269312 }, { "step": 42, "value": 269312 }, { "step": 43, "value": 269312 }, { "step": 44, "value": 267264 }, { "step": 45, "value": 267264 }, { "step": 46, "value": 267264 }, { "step": 47, "value": 267264 }, { "step": 48, "value": 267264 }, { "step": 49, "value": 267264 }, { "step": 50, "value": 267264 }, { "step": 51, "value": 267264 }, { "step": 52, "value": 266752 }, { "step": 53, "value": 266752 }, { "step": 54, "value": 266752 }, { "step": 55, "value": 203776 }, { "step": 56, "value": 152576 }, { "step": 57, "value": 152576 }, { "step": 58, "value": 152576 }, { "step": 59, "value": 152576 }, { "step": 60, "value": 152064 }, { "step": 61, "value": 152064 }, { "step": 62, "value": 115712 }, { "step": 63, "value": 115712 }, { "step": 64, "value": 115712 }, { "step": 65, "value": 115712 }, { "step": 66, "value": 115712 }, { "step": 67, "value": 115712 }, { "step": 68, "value": 115712 }, { "step": 69, "value": 115200 }, { "step": 70, "value": 115200 }, { "step": 71, "value": 115200 }, { "step": 72, "value": 97280 }, { "step": 73, "value": 97280 }, { "step": 74, "value": 97280 }, { "step": 75, "value": 97280 }, { "step": 76, "value": 97280 }, { "step": 77, "value": 97280 }, { "step": 78, "value": 97280 }, { "step": 79, "value": 90112 }, { "step": 80, "value": 90112 }, { "step": 81, "value": 90112 }, { "step": 82, "value": 90112 }, { "step": 83, "value": 90112 }, { "step": 84, "value": 90112 }, { "step": 85, "value": 88064 }, { "step": 86, "value": 78848 }, { "step": 87, "value": 78848 }, { "step": 88, "value": 78848 }, { "step": 89, "value": 78848 }, { "step": 90, "value": 78848 }, { "step": 91, "value": 78848 }, { "step": 92, "value": 78848 }, { "step": 93, "value": 69632 }, { "step": 94, "value": 69632 }, { "step": 95, "value": 69632 }, { "step": 96, "value": 69632 }, { "step": 97, "value": 69632 }, { "step": 98, "value": 69632 }, { "step": 99, "value": 69632 } ], "submission_sha256": "149ee20700a3845a876aea4e6a476885ec42f8ab2de11d8cd1b14bb298d4a770", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/beam-search/kvcache/submission.json", "comparison_value": 69632, "comparison_rank_key": [ 69632.0, 69632.0, -0.0004540638239545203 ], "rank": 3 }, { "run_id": "arbor--kvcache", "method_id": "arbor", "task_id": "kvcache", "metric": "kv_cache_bytes", "unit": "bytes", "direction": "minimize", "best_value": 96768, "reference_value": 34603520, "relative_improvement": 0.9972035214914552, "rank_key": [ 96768.0, 96768.0, -0.0006193217644312465 ], "best_launch_seq": 50, "best_candidate_id": "exp-8e909e2b77e47952bc92f941", "best_is_reference": false, "budget": { "spent": 50, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 9.839154166666669, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 34603520, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-cbad60380fc15e9fa2241257", "value": 8389120, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp-6d041deee265e50c907d848e", "value": 26215424, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-b9a6e13b6c9c084a37f003bc", "value": 21758976, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-30000526e38602993ab3f54d", "value": 6161408, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-db8b2d8756aa212d294cfed1", "value": 4194816, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-f595fd2919933908abf5a6b8", "value": 2179584, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp-381e25a7dfe07ad2a9fb930c", "value": 2647552, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-49494568fa3f3fcc604333d3", "value": 1581568, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-fe4f778adefa278d1267797e", "value": 820736, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-7fff38bf55ae541d2ec89ee8", "value": 893952, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-0133fab4972b7304e7f76ac5", "value": 298496, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-151c00db757e8c79956f274a", "value": 225280, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-64f70bb4fea08b9ed4c55486", "value": 187904, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-343c262772372195105fb8f4", "value": 141312, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp-4226334bb5e1e768cb954092", "value": 113152, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp-c28e9a2f9c126b8722645d1d", "value": 130048, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp-3168b68d8d39249442c92295", "value": 113152, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp-ab4ddb312873acfb3378b204", "value": 103936, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp-c25bac7d2ce8e9ab93b41907", "value": 100864, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp-63106637e40fc92c6ad7d778", "value": 100864, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp-4ee192ae5c188f4eebc07f27", "value": 97792, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp-a0bfdae9669622ebe07dc95c", "value": 97792, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp-9088c3d177ab7938a1154c7a", "value": 97792, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp-8e909e2b77e47952bc92f941", "value": 96768, "is_reference": false } ], "curve": [ { "step": 1, "value": 34603520 }, { "step": 2, "value": 8389120 }, { "step": 3, "value": 8389120 }, { "step": 4, "value": 8389120 }, { "step": 5, "value": 6161408 }, { "step": 6, "value": 4194816 }, { "step": 7, "value": 2179584 }, { "step": 8, "value": 2179584 }, { "step": 9, "value": 2179584 }, { "step": 10, "value": 2179584 }, { "step": 11, "value": 1581568 }, { "step": 12, "value": 820736 }, { "step": 13, "value": 820736 }, { "step": 14, "value": 820736 }, { "step": 15, "value": 298496 }, { "step": 16, "value": 298496 }, { "step": 17, "value": 225280 }, { "step": 18, "value": 187904 }, { "step": 19, "value": 187904 }, { "step": 20, "value": 141312 }, { "step": 21, "value": 141312 }, { "step": 22, "value": 141312 }, { "step": 23, "value": 113152 }, { "step": 24, "value": 113152 }, { "step": 25, "value": 113152 }, { "step": 26, "value": 113152 }, { "step": 27, "value": 113152 }, { "step": 28, "value": 113152 }, { "step": 29, "value": 113152 }, { "step": 30, "value": 103936 }, { "step": 31, "value": 103936 }, { "step": 32, "value": 100864 }, { "step": 33, "value": 100864 }, { "step": 34, "value": 100864 }, { "step": 35, "value": 100864 }, { "step": 36, "value": 100864 }, { "step": 37, "value": 100864 }, { "step": 38, "value": 100864 }, { "step": 39, "value": 100864 }, { "step": 40, "value": 100864 }, { "step": 41, "value": 100864 }, { "step": 42, "value": 100864 }, { "step": 43, "value": 100864 }, { "step": 44, "value": 100864 }, { "step": 45, "value": 100864 }, { "step": 46, "value": 97792 }, { "step": 47, "value": 97792 }, { "step": 48, "value": 97792 }, { "step": 49, "value": 97792 }, { "step": 50, "value": 96768 } ], "submission_sha256": "fc7e3f05eb59f2b9e758dd27c3fc8d4f8fe7f6a44353d0cedd43baed8e19f851", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/arbor/kvcache/submission.json", "comparison_value": 96768, "comparison_rank_key": [ 96768.0, 96768.0, -0.0006193217644312465 ], "rank": 4 }, { "run_id": "gear--kvcache", "method_id": "gear", "task_id": "kvcache", "metric": "kv_cache_bytes", "unit": "bytes", "direction": "minimize", "best_value": 124416, "reference_value": 34603520, "relative_improvement": 0.9964045276318709, "rank_key": [ 124416.0, 124416.0, -0.0004253857742240186 ], "best_launch_seq": 100, "best_candidate_id": "exp0097", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": null }, "evaluation_gpu_hours": 19.684501388888886, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 34603520, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp0002", "value": 33554944, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp0003", "value": 17039872, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp0005", "value": 17039872, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp0006", "value": 34603520, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp0007", "value": 33554944, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp0008", "value": 17039872, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp0011", "value": 8520192, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp0012", "value": 17826304, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp0013", "value": 10649088, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp0014", "value": 6241792, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp0017", "value": 8520192, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp0018", "value": 8520192, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp0019", "value": 2662912, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp0024b", "value": 9568768, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp0025", "value": 2662912, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp0027", "value": 2130432, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp0028", "value": 2130432, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp0029", "value": 2130432, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp0031", "value": 1065472, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp0032", "value": 1065472, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp0033", "value": 1065472, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp0035", "value": 532992, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp0036", "value": 8520192, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp0038", "value": 270848, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "exp0039", "value": 541184, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp0040", "value": 2130432, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp0044", "value": 203264, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp0045", "value": 186880, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "exp0048", "value": 338432, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp0050", "value": 2130432, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp0051", "value": 180736, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp0052b", "value": 1065472, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp0053", "value": 666112, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp0054", "value": 169472, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp0055", "value": 333312, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp0056", "value": 338432, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "exp0057", "value": 159232, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp0060c", "value": 158208, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp0063", "value": 159232, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "exp0064", "value": 158208, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "exp0067", "value": 143872, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp0068", "value": 143872, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "exp0069", "value": 143872, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "exp0070", "value": 143872, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "exp0071", "value": 135680, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "exp0073", "value": 132608, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "exp0074", "value": 143872, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "exp0077", "value": 143872, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "exp0080", "value": 142848, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "exp0082", "value": 127488, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp0083", "value": 138752, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "exp0085", "value": 126464, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "exp0091", "value": 125952, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "exp0093", "value": 124928, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "exp0094", "value": 124416, "is_reference": false }, { "step": 100, "launch_seq": 100, "candidate_id": "exp0097", "value": 124416, "is_reference": false } ], "curve": [ { "step": 1, "value": 34603520 }, { "step": 2, "value": 33554944 }, { "step": 3, "value": 17039872 }, { "step": 4, "value": 17039872 }, { "step": 5, "value": 17039872 }, { "step": 6, "value": 17039872 }, { "step": 7, "value": 17039872 }, { "step": 8, "value": 17039872 }, { "step": 9, "value": 17039872 }, { "step": 10, "value": 17039872 }, { "step": 11, "value": 8520192 }, { "step": 12, "value": 8520192 }, { "step": 13, "value": 8520192 }, { "step": 14, "value": 6241792 }, { "step": 15, "value": 6241792 }, { "step": 16, "value": 6241792 }, { "step": 17, "value": 6241792 }, { "step": 18, "value": 6241792 }, { "step": 19, "value": 2662912 }, { "step": 20, "value": 2662912 }, { "step": 21, "value": 2662912 }, { "step": 22, "value": 2662912 }, { "step": 23, "value": 2662912 }, { "step": 24, "value": 2662912 }, { "step": 25, "value": 2662912 }, { "step": 26, "value": 2662912 }, { "step": 27, "value": 2662912 }, { "step": 28, "value": 2130432 }, { "step": 29, "value": 2130432 }, { "step": 30, "value": 2130432 }, { "step": 31, "value": 2130432 }, { "step": 32, "value": 1065472 }, { "step": 33, "value": 1065472 }, { "step": 34, "value": 1065472 }, { "step": 35, "value": 1065472 }, { "step": 36, "value": 532992 }, { "step": 37, "value": 532992 }, { "step": 38, "value": 532992 }, { "step": 39, "value": 270848 }, { "step": 40, "value": 270848 }, { "step": 41, "value": 270848 }, { "step": 42, "value": 270848 }, { "step": 43, "value": 270848 }, { "step": 44, "value": 270848 }, { "step": 45, "value": 203264 }, { "step": 46, "value": 186880 }, { "step": 47, "value": 186880 }, { "step": 48, "value": 186880 }, { "step": 49, "value": 186880 }, { "step": 50, "value": 186880 }, { "step": 51, "value": 186880 }, { "step": 52, "value": 180736 }, { "step": 53, "value": 180736 }, { "step": 54, "value": 180736 }, { "step": 55, "value": 180736 }, { "step": 56, "value": 169472 }, { "step": 57, "value": 169472 }, { "step": 58, "value": 169472 }, { "step": 59, "value": 159232 }, { "step": 60, "value": 159232 }, { "step": 61, "value": 159232 }, { "step": 62, "value": 159232 }, { "step": 63, "value": 158208 }, { "step": 64, "value": 158208 }, { "step": 65, "value": 158208 }, { "step": 66, "value": 158208 }, { "step": 67, "value": 158208 }, { "step": 68, "value": 158208 }, { "step": 69, "value": 158208 }, { "step": 70, "value": 143872 }, { "step": 71, "value": 143872 }, { "step": 72, "value": 143872 }, { "step": 73, "value": 143872 }, { "step": 74, "value": 135680 }, { "step": 75, "value": 135680 }, { "step": 76, "value": 132608 }, { "step": 77, "value": 132608 }, { "step": 78, "value": 132608 }, { "step": 79, "value": 132608 }, { "step": 80, "value": 132608 }, { "step": 81, "value": 132608 }, { "step": 82, "value": 132608 }, { "step": 83, "value": 132608 }, { "step": 84, "value": 132608 }, { "step": 85, "value": 127488 }, { "step": 86, "value": 127488 }, { "step": 87, "value": 127488 }, { "step": 88, "value": 126464 }, { "step": 89, "value": 126464 }, { "step": 90, "value": 126464 }, { "step": 91, "value": 126464 }, { "step": 92, "value": 126464 }, { "step": 93, "value": 126464 }, { "step": 94, "value": 125952 }, { "step": 95, "value": 125952 }, { "step": 96, "value": 124928 }, { "step": 97, "value": 124416 }, { "step": 98, "value": 124416 }, { "step": 99, "value": 124416 }, { "step": 100, "value": 124416 } ], "submission_sha256": "ae56f6b3c4c483a8d1a45fa9f6c1f09289864f005b417df9ed57301ce206329a", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/gear/kvcache/submission.json", "comparison_value": 124416, "comparison_rank_key": [ 124416.0, 124416.0, -0.0004253857742240186 ], "rank": 5 }, { "run_id": "heuresis-map-elites--kvcache", "method_id": "heuresis-map-elites", "task_id": "kvcache", "metric": "kv_cache_bytes", "unit": "bytes", "direction": "minimize", "best_value": 500224, "reference_value": 34603520, "relative_improvement": 0.9855441296145594, "rank_key": [ 500224.0, 500224.0, -0.002630464649238551 ], "best_launch_seq": 67, "best_candidate_id": "exp-ddf41160d1c3cb8b1611c494", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 17.534113055555558, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 34603520, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-7f283a9e4a33640d35e6f96b", "value": 8797696, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp-afc32817e81e470b23ba24ce", "value": 3951104, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-7ea09b1897510556f104b301", "value": 20081152, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-85d3c04f14a914e502893a11", "value": 9901568, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-27dae400ebd6877ecf1fd639", "value": 6182400, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-6d8345287c9ed5bb046060a2", "value": 17826304, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-86eb13cf1dd8521ac37675ea", "value": 16777728, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-9f31b69d6906951048c62a67", "value": 4194816, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-e92ac970c398f78740784741", "value": 16908800, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-92ec13b1bb63706b537e5a4d", "value": 12714496, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp-3c6d448450e7b68ab18f2f1b", "value": 10492416, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp-38b206359577ca7f069e30ca", "value": 7346688, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp-98fa1a466066b7d079d1a5e5", "value": 5773824, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp-7d153d9b2468a1fd070c123b", "value": 16777728, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp-afa01013ccc664fc40485705", "value": 8389120, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp-2ab683c10795091b517b5194", "value": 4194816, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-f0fe2ba88f8844eb698d875e", "value": 10492416, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-67849ca288ea7879ff87e293", "value": 7346688, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-56259e66512dc468e8f45b8a", "value": 4726272, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp-506d3d8aac426cb293abb5cb", "value": 2368000, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp-2a0487e92b24ae9555bf245f", "value": 598528, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp-ba77d9ee0363bf67898f1531", "value": 5249536, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp-ab644182db0d3b680bea8f12", "value": 3676672, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp-2b476f5f74005e5959ef74e4", "value": 1841664, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp-246cc1a4cdd4e0153a24b7de", "value": 15741440, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "exp-2a96323aed2e5200b7752e0d", "value": 13427200, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp-5a26a4c7439486b21db44bfb", "value": 11989504, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp-e4f3a020d1e7cd8bbcac68fc", "value": 5995008, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp-28bfcd57ca18c7ec444a6dc9", "value": 5454336, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "exp-8c8ab39504032ed65e6249a7", "value": 26085888, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "exp-44a5b5ea1b34eeb7e115a7f4", "value": 10492416, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "exp-bef28ad4fa843c9ea7be4e73", "value": 5249536, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp-63133fbe7bbc61c8920cba53", "value": 3676672, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp-0ea462ebec1f4de50674c827", "value": 1843712, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "exp-bb7069a95cf4f3e6cd43ff01", "value": 664064, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "exp-ddf41160d1c3cb8b1611c494", "value": 500224, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "exp-88e464a9edc41e969d33d36c", "value": 2107904, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "exp-8be98ad5d4564c332441bc90", "value": 10492416, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "exp-ccb0c00a5ce97bff5fb11118", "value": 7346688, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "exp-c10c6acc194dbbac5b0fe7bf", "value": 3676672, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "exp-88b63dc949d3fcfdbfd06954", "value": 10492416, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "exp-f044cf108780de5fbe5c2158", "value": 5249536, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "exp-a1475a3769ee64757ba7fcb8", "value": 3676672, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "exp-4dc7a4550104dddba3dc8ea5", "value": 17834496, "is_reference": false }, { "step": 99, "launch_seq": 99, "candidate_id": "exp-a72041f5ea113baceaa8376c", "value": 7346688, "is_reference": false } ], "curve": [ { "step": 1, "value": 34603520 }, { "step": 2, "value": 8797696 }, { "step": 3, "value": 3951104 }, { "step": 4, "value": 3951104 }, { "step": 5, "value": 3951104 }, { "step": 6, "value": 3951104 }, { "step": 7, "value": 3951104 }, { "step": 8, "value": 3951104 }, { "step": 9, "value": 3951104 }, { "step": 10, "value": 3951104 }, { "step": 11, "value": 3951104 }, { "step": 12, "value": 3951104 }, { "step": 13, "value": 3951104 }, { "step": 14, "value": 3951104 }, { "step": 15, "value": 3951104 }, { "step": 16, "value": 3951104 }, { "step": 17, "value": 3951104 }, { "step": 18, "value": 3951104 }, { "step": 19, "value": 3951104 }, { "step": 20, "value": 3951104 }, { "step": 21, "value": 3951104 }, { "step": 22, "value": 3951104 }, { "step": 23, "value": 3951104 }, { "step": 24, "value": 3951104 }, { "step": 25, "value": 3951104 }, { "step": 26, "value": 3951104 }, { "step": 27, "value": 3951104 }, { "step": 28, "value": 3951104 }, { "step": 29, "value": 3951104 }, { "step": 30, "value": 3951104 }, { "step": 31, "value": 3951104 }, { "step": 32, "value": 3951104 }, { "step": 33, "value": 3951104 }, { "step": 34, "value": 3951104 }, { "step": 35, "value": 3951104 }, { "step": 36, "value": 3951104 }, { "step": 37, "value": 2368000 }, { "step": 38, "value": 598528 }, { "step": 39, "value": 598528 }, { "step": 40, "value": 598528 }, { "step": 41, "value": 598528 }, { "step": 42, "value": 598528 }, { "step": 43, "value": 598528 }, { "step": 44, "value": 598528 }, { "step": 45, "value": 598528 }, { "step": 46, "value": 598528 }, { "step": 47, "value": 598528 }, { "step": 48, "value": 598528 }, { "step": 49, "value": 598528 }, { "step": 50, "value": 598528 }, { "step": 51, "value": 598528 }, { "step": 52, "value": 598528 }, { "step": 53, "value": 598528 }, { "step": 54, "value": 598528 }, { "step": 55, "value": 598528 }, { "step": 56, "value": 598528 }, { "step": 57, "value": 598528 }, { "step": 58, "value": 598528 }, { "step": 59, "value": 598528 }, { "step": 60, "value": 598528 }, { "step": 61, "value": 598528 }, { "step": 62, "value": 598528 }, { "step": 63, "value": 598528 }, { "step": 64, "value": 598528 }, { "step": 65, "value": 598528 }, { "step": 66, "value": 598528 }, { "step": 67, "value": 500224 }, { "step": 68, "value": 500224 }, { "step": 69, "value": 500224 }, { "step": 70, "value": 500224 }, { "step": 71, "value": 500224 }, { "step": 72, "value": 500224 }, { "step": 73, "value": 500224 }, { "step": 74, "value": 500224 }, { "step": 75, "value": 500224 }, { "step": 76, "value": 500224 }, { "step": 77, "value": 500224 }, { "step": 78, "value": 500224 }, { "step": 79, "value": 500224 }, { "step": 80, "value": 500224 }, { "step": 81, "value": 500224 }, { "step": 82, "value": 500224 }, { "step": 83, "value": 500224 }, { "step": 84, "value": 500224 }, { "step": 85, "value": 500224 }, { "step": 86, "value": 500224 }, { "step": 87, "value": 500224 }, { "step": 88, "value": 500224 }, { "step": 89, "value": 500224 }, { "step": 90, "value": 500224 }, { "step": 91, "value": 500224 }, { "step": 92, "value": 500224 }, { "step": 93, "value": 500224 }, { "step": 94, "value": 500224 }, { "step": 95, "value": 500224 }, { "step": 96, "value": 500224 }, { "step": 97, "value": 500224 }, { "step": 98, "value": 500224 }, { "step": 99, "value": 500224 }, { "step": 100, "value": 500224 } ], "submission_sha256": "7f39e6771f0c9aaad2c42f1e36e3d45bb2c6e569e388a01cce8721aee9506191", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/heuresis-map-elites/kvcache/submission.json", "comparison_value": 500224, "comparison_rank_key": [ 500224.0, 500224.0, -0.002630464649238551 ], "rank": 6 }, { "run_id": "tpe--kvcache", "method_id": "tpe", "task_id": "kvcache", "metric": "kv_cache_bytes", "unit": "bytes", "direction": "minimize", "best_value": 18219520, "reference_value": 34603520, "relative_improvement": 0.4734778427165791, "rank_key": [ 18219520.0, 3942912.0, -0.015373863657836662 ], "best_launch_seq": 53, "best_candidate_id": "tpe-trial-0052", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 17.953643333333332, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 34603520, "is_reference": true }, { "step": 14, "launch_seq": 14, "candidate_id": "tpe-trial-0013", "value": 34734592, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "tpe-trial-0015", "value": 32899584, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "tpe-trial-0021", "value": 34734592, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "tpe-trial-0022", "value": 26608128, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "tpe-trial-0023", "value": 23986688, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "tpe-trial-0024", "value": 21889536, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "tpe-trial-0029", "value": 34603520, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "tpe-trial-0034", "value": 26608128, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "tpe-trial-0035", "value": 18219520, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "tpe-trial-0036", "value": 18219520, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "tpe-trial-0037", "value": 18612736, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "tpe-trial-0039", "value": 18612736, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "tpe-trial-0042", "value": 18219520, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "tpe-trial-0043", "value": 18219520, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "tpe-trial-0044", "value": 18219520, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "tpe-trial-0045", "value": 25166336, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "tpe-trial-0049", "value": 20972032, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "tpe-trial-0050", "value": 19399168, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "tpe-trial-0051", "value": 18612736, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "tpe-trial-0052", "value": 18219520, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "tpe-trial-0055", "value": 25166336, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "tpe-trial-0057", "value": 18612736, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "tpe-trial-0060", "value": 26214912, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "tpe-trial-0061", "value": 18612736, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "tpe-trial-0064", "value": 20972032, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "tpe-trial-0065", "value": 29360640, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "tpe-trial-0069", "value": 22938112, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "tpe-trial-0075", "value": 29360640, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "tpe-trial-0078", "value": 29360640, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "tpe-trial-0079", "value": 18612736, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "tpe-trial-0080", "value": 21889536, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "tpe-trial-0081", "value": 18219520, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "tpe-trial-0084", "value": 20972032, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "tpe-trial-0085", "value": 18219520, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "tpe-trial-0088", "value": 23986688, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "tpe-trial-0089", "value": 20972032, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "tpe-trial-0091", "value": 18612736, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "tpe-trial-0092", "value": 20972032, "is_reference": false }, { "step": 98, "launch_seq": 98, "candidate_id": "tpe-trial-0096", "value": 19399168, "is_reference": false } ], "curve": [ { "step": 1, "value": 34603520 }, { "step": 2, "value": 34603520 }, { "step": 3, "value": 34603520 }, { "step": 4, "value": 34603520 }, { "step": 5, "value": 34603520 }, { "step": 6, "value": 34603520 }, { "step": 7, "value": 34603520 }, { "step": 8, "value": 34603520 }, { "step": 9, "value": 34603520 }, { "step": 10, "value": 34603520 }, { "step": 11, "value": 34603520 }, { "step": 12, "value": 34603520 }, { "step": 13, "value": 34603520 }, { "step": 14, "value": 34603520 }, { "step": 15, "value": 34603520 }, { "step": 16, "value": 32899584 }, { "step": 17, "value": 32899584 }, { "step": 18, "value": 32899584 }, { "step": 19, "value": 32899584 }, { "step": 20, "value": 32899584 }, { "step": 21, "value": 32899584 }, { "step": 22, "value": 32899584 }, { "step": 23, "value": 26608128 }, { "step": 24, "value": 23986688 }, { "step": 25, "value": 21889536 }, { "step": 26, "value": 21889536 }, { "step": 27, "value": 21889536 }, { "step": 28, "value": 21889536 }, { "step": 29, "value": 21889536 }, { "step": 30, "value": 21889536 }, { "step": 31, "value": 21889536 }, { "step": 32, "value": 21889536 }, { "step": 33, "value": 21889536 }, { "step": 34, "value": 21889536 }, { "step": 35, "value": 21889536 }, { "step": 36, "value": 18219520 }, { "step": 37, "value": 18219520 }, { "step": 38, "value": 18219520 }, { "step": 39, "value": 18219520 }, { "step": 40, "value": 18219520 }, { "step": 41, "value": 18219520 }, { "step": 42, "value": 18219520 }, { "step": 43, "value": 18219520 }, { "step": 44, "value": 18219520 }, { "step": 45, "value": 18219520 }, { "step": 46, "value": 18219520 }, { "step": 47, "value": 18219520 }, { "step": 48, "value": 18219520 }, { "step": 49, "value": 18219520 }, { "step": 50, "value": 18219520 }, { "step": 51, "value": 18219520 }, { "step": 52, "value": 18219520 }, { "step": 53, "value": 18219520 }, { "step": 54, "value": 18219520 }, { "step": 55, "value": 18219520 }, { "step": 56, "value": 18219520 }, { "step": 57, "value": 18219520 }, { "step": 58, "value": 18219520 }, { "step": 59, "value": 18219520 }, { "step": 60, "value": 18219520 }, { "step": 61, "value": 18219520 }, { "step": 62, "value": 18219520 }, { "step": 63, "value": 18219520 }, { "step": 64, "value": 18219520 }, { "step": 65, "value": 18219520 }, { "step": 66, "value": 18219520 }, { "step": 67, "value": 18219520 }, { "step": 68, "value": 18219520 }, { "step": 69, "value": 18219520 }, { "step": 70, "value": 18219520 }, { "step": 71, "value": 18219520 }, { "step": 72, "value": 18219520 }, { "step": 73, "value": 18219520 }, { "step": 74, "value": 18219520 }, { "step": 75, "value": 18219520 }, { "step": 76, "value": 18219520 }, { "step": 77, "value": 18219520 }, { "step": 78, "value": 18219520 }, { "step": 79, "value": 18219520 }, { "step": 80, "value": 18219520 }, { "step": 81, "value": 18219520 }, { "step": 82, "value": 18219520 }, { "step": 83, "value": 18219520 }, { "step": 84, "value": 18219520 }, { "step": 85, "value": 18219520 }, { "step": 86, "value": 18219520 }, { "step": 87, "value": 18219520 }, { "step": 88, "value": 18219520 }, { "step": 89, "value": 18219520 }, { "step": 90, "value": 18219520 }, { "step": 91, "value": 18219520 }, { "step": 92, "value": 18219520 }, { "step": 93, "value": 18219520 }, { "step": 94, "value": 18219520 }, { "step": 95, "value": 18219520 }, { "step": 96, "value": 18219520 }, { "step": 97, "value": 18219520 }, { "step": 98, "value": 18219520 }, { "step": 99, "value": 18219520 }, { "step": 100, "value": 18219520 } ], "submission_sha256": "cf712d11a6b22380cd6f2a89a4f1fae16ead5a6a13208b16ce4ce45f7dd348d8", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/tpe/kvcache/submission.json", "comparison_value": 18219520, "comparison_rank_key": [ 18219520.0, 3942912.0, -0.015373863657836662 ], "rank": 7 } ] }, { "id": "memory", "title": "Peak allocated memory at validation BPB < 1.05", "metric": "peak_vram_bytes", "unit": "bytes", "direction": "minimize", "budget": 100, "comparison_step": null, "quality_gate": { "metric": "val_bpb", "operator": "<", "value": 1.05 }, "tiebreaks": [ { "metric": "total_tokens", "direction": "minimize" } ], "runs": [ { "run_id": "arbor--memory", "method_id": "arbor", "task_id": "memory", "metric": "peak_vram_bytes", "unit": "bytes", "direction": "minimize", "best_value": 400084992, "reference_value": 47198976512, "relative_improvement": 0.9915234392445293, "rank_key": [ 400084992.0, 196591616.0, -0.0012713717345373965 ], "best_launch_seq": 48, "best_candidate_id": "exp-190223ffdcaf8347643f19db", "best_is_reference": false, "budget": { "spent": 49, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 9.555408055555555, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 47198976512, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-0d3ce48ed94909b3576f4d33", "value": 45319928320, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp-2b659c096ff8426aa82d616d", "value": 43709315584, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-cfdc7273419486aec2014477", "value": 44246186496, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-3f67283a863801fe6897569d", "value": 5597830144, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-36820cd80fe37831fe56edc8", "value": 4702246912, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-4d76c43371e63c4b2520de0f", "value": 2582018048, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-167f0f9c39c156d72ecb99e7", "value": 3226802176, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-2d68af9a1ffe53a4f489aec8", "value": 2582018048, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-c1ef66e9b0a3ebd9cf6d4fa7", "value": 2582018048, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-09d6153f8da0b365575a4c41", "value": 2582018048, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-4c12d421f0451e9196b13e80", "value": 1753454592, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp-4f34750944652a2ffcd0d256", "value": 2296608768, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp-76fa7ac442c75bd54c29e3b1", "value": 1692678656, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-680e0c11cae9bf5b24107e85", "value": 1592043520, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp-1eca24665908d2b3d26f9d65", "value": 1505274368, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp-71646107e7bdcc43cb787e14", "value": 1102654464, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp-5d680a9e58a91d5835f61ddf", "value": 1505274368, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp-be5111889e556a73d3d388d6", "value": 945827840, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp-27d16e7ac521afdc172b3a45", "value": 945827840, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp-da41d16f05f8302bc664cb99", "value": 874000384, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-ecba9f4265f82247b7990710", "value": 789163008, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-8cff90228128534d5fb08e61", "value": 1102654464, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-1a6bf5e32ba0ce58c93299d9", "value": 756226560, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-33a99aed74d9d25780a74295", "value": 756226560, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp-f8d9e5d3ec9256e3f5da047f", "value": 680204800, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp-6b0822caf38adad55d210647", "value": 680598016, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp-026135a1e519357d0381d34d", "value": 563745280, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp-88cc9732db5b4b3434db01df", "value": 435660288, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp-2fb6c14ce0b1549aee284c78", "value": 459276288, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp-d81892fe4012bfd3b8e0dd46", "value": 513775104, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp-190223ffdcaf8347643f19db", "value": 400084992, "is_reference": false } ], "curve": [ { "step": 1, "value": 47198976512 }, { "step": 2, "value": 45319928320 }, { "step": 3, "value": 43709315584 }, { "step": 4, "value": 43709315584 }, { "step": 5, "value": 5597830144 }, { "step": 6, "value": 5597830144 }, { "step": 7, "value": 5597830144 }, { "step": 8, "value": 5597830144 }, { "step": 9, "value": 5597830144 }, { "step": 10, "value": 5597830144 }, { "step": 11, "value": 5597830144 }, { "step": 12, "value": 4702246912 }, { "step": 13, "value": 2582018048 }, { "step": 14, "value": 2582018048 }, { "step": 15, "value": 2582018048 }, { "step": 16, "value": 2582018048 }, { "step": 17, "value": 2582018048 }, { "step": 18, "value": 2582018048 }, { "step": 19, "value": 2582018048 }, { "step": 20, "value": 1753454592 }, { "step": 21, "value": 1753454592 }, { "step": 22, "value": 1753454592 }, { "step": 23, "value": 1753454592 }, { "step": 24, "value": 1753454592 }, { "step": 25, "value": 1692678656 }, { "step": 26, "value": 1592043520 }, { "step": 27, "value": 1505274368 }, { "step": 28, "value": 1102654464 }, { "step": 29, "value": 1102654464 }, { "step": 30, "value": 945827840 }, { "step": 31, "value": 945827840 }, { "step": 32, "value": 874000384 }, { "step": 33, "value": 789163008 }, { "step": 34, "value": 789163008 }, { "step": 35, "value": 756226560 }, { "step": 36, "value": 756226560 }, { "step": 37, "value": 680204800 }, { "step": 38, "value": 680204800 }, { "step": 39, "value": 680204800 }, { "step": 40, "value": 680204800 }, { "step": 41, "value": 680204800 }, { "step": 42, "value": 680204800 }, { "step": 43, "value": 563745280 }, { "step": 44, "value": 435660288 }, { "step": 45, "value": 435660288 }, { "step": 46, "value": 435660288 }, { "step": 47, "value": 435660288 }, { "step": 48, "value": 400084992 }, { "step": 49, "value": 400084992 } ], "submission_sha256": "18e6cbe8dad849f3db49a8847bf2cdb2852c4ffa3c39c2b03d715c86ac7544c9", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/arbor/memory/submission.json", "comparison_value": 400084992, "comparison_rank_key": [ 400084992.0, 196591616.0, -0.0012713717345373965 ], "rank": 1 }, { "run_id": "gear--memory", "method_id": "gear", "task_id": "memory", "metric": "peak_vram_bytes", "unit": "bytes", "direction": "minimize", "best_value": 466297856, "reference_value": 47198976512, "relative_improvement": 0.9901205939098817, "rank_key": [ 466297856.0, 247726080.0, -2.3452188527617324e-05 ], "best_launch_seq": 81, "best_candidate_id": "exp0081", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 22.802099444444437, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 47198976512, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-88d94e3b1af2add8d9419201", "value": 38877477376, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp0004", "value": 23913249280, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp0005", "value": 12270385664, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp0006", "value": 6448953856, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp0007", "value": 19752499712, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp0008", "value": 47198976512, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp0009", "value": 5564536832, "is_reference": false }, { "step": 10, "launch_seq": 11, "candidate_id": "exp0010-b", "value": 5027403776, "is_reference": false }, { "step": 11, "launch_seq": 12, "candidate_id": "exp0011", "value": 5564274688, "is_reference": false }, { "step": 12, "launch_seq": 13, "candidate_id": "exp0012", "value": 4978614784, "is_reference": false }, { "step": 13, "launch_seq": 14, "candidate_id": "exp0013", "value": 4322830848, "is_reference": false }, { "step": 15, "launch_seq": 16, "candidate_id": "exp0015", "value": 4020961792, "is_reference": false }, { "step": 16, "launch_seq": 17, "candidate_id": "exp0016", "value": 3922270208, "is_reference": false }, { "step": 18, "launch_seq": 19, "candidate_id": "exp0018", "value": 2288886272, "is_reference": false }, { "step": 19, "launch_seq": 20, "candidate_id": "exp0019", "value": 2095426048, "is_reference": false }, { "step": 21, "launch_seq": 22, "candidate_id": "exp0021", "value": 2028317184, "is_reference": false }, { "step": 22, "launch_seq": 23, "candidate_id": "exp0022", "value": 2028317184, "is_reference": false }, { "step": 23, "launch_seq": 24, "candidate_id": "exp0023", "value": 1883730944, "is_reference": false }, { "step": 24, "launch_seq": 25, "candidate_id": "exp0024", "value": 1883730944, "is_reference": false }, { "step": 25, "launch_seq": 26, "candidate_id": "exp0025", "value": 1849141248, "is_reference": false }, { "step": 26, "launch_seq": 27, "candidate_id": "exp0026", "value": 1790420992, "is_reference": false }, { "step": 27, "launch_seq": 28, "candidate_id": "exp0027", "value": 1636208640, "is_reference": false }, { "step": 28, "launch_seq": 29, "candidate_id": "exp0028", "value": 1609008640, "is_reference": false }, { "step": 29, "launch_seq": 30, "candidate_id": "exp0029", "value": 1603241472, "is_reference": false }, { "step": 31, "launch_seq": 32, "candidate_id": "exp0031", "value": 1603241472, "is_reference": false }, { "step": 32, "launch_seq": 33, "candidate_id": "exp0032", "value": 1444447744, "is_reference": false }, { "step": 33, "launch_seq": 34, "candidate_id": "exp0033", "value": 1268745216, "is_reference": false }, { "step": 35, "launch_seq": 36, "candidate_id": "exp0035", "value": 1109951488, "is_reference": false }, { "step": 37, "launch_seq": 38, "candidate_id": "exp0037", "value": 1109951488, "is_reference": false }, { "step": 38, "launch_seq": 39, "candidate_id": "exp0038", "value": 1109951488, "is_reference": false }, { "step": 39, "launch_seq": 40, "candidate_id": "exp0039", "value": 770148352, "is_reference": false }, { "step": 40, "launch_seq": 41, "candidate_id": "exp0040", "value": 770148352, "is_reference": false }, { "step": 41, "launch_seq": 42, "candidate_id": "exp0041", "value": 770148352, "is_reference": false }, { "step": 42, "launch_seq": 43, "candidate_id": "exp0042", "value": 770148352, "is_reference": false }, { "step": 43, "launch_seq": 44, "candidate_id": "exp0043", "value": 703039488, "is_reference": false }, { "step": 47, "launch_seq": 48, "candidate_id": "exp0047", "value": 703038464, "is_reference": false }, { "step": 51, "launch_seq": 52, "candidate_id": "exp0051", "value": 677872640, "is_reference": false }, { "step": 55, "launch_seq": 56, "candidate_id": "exp0055", "value": 677872640, "is_reference": false }, { "step": 57, "launch_seq": 58, "candidate_id": "exp0057", "value": 678907904, "is_reference": false }, { "step": 58, "launch_seq": 59, "candidate_id": "exp0058", "value": 636964864, "is_reference": false }, { "step": 59, "launch_seq": 60, "candidate_id": "exp0059", "value": 645228544, "is_reference": false }, { "step": 60, "launch_seq": 61, "candidate_id": "exp0060", "value": 590228992, "is_reference": false }, { "step": 61, "launch_seq": 62, "candidate_id": "exp0061", "value": 562871296, "is_reference": false }, { "step": 65, "launch_seq": 66, "candidate_id": "exp0065", "value": 487105536, "is_reference": false }, { "step": 77, "launch_seq": 78, "candidate_id": "exp0078", "value": 478717952, "is_reference": false }, { "step": 80, "launch_seq": 81, "candidate_id": "exp0081", "value": 466297856, "is_reference": false } ], "curve": [ { "step": 1, "value": 47198976512 }, { "step": 2, "value": 38877477376 }, { "step": 3, "value": 38877477376 }, { "step": 4, "value": 23913249280 }, { "step": 5, "value": 12270385664 }, { "step": 6, "value": 6448953856 }, { "step": 7, "value": 6448953856 }, { "step": 8, "value": 6448953856 }, { "step": 9, "value": 5564536832 }, { "step": 10, "value": 5027403776 }, { "step": 11, "value": 5027403776 }, { "step": 12, "value": 4978614784 }, { "step": 13, "value": 4322830848 }, { "step": 14, "value": 4322830848 }, { "step": 15, "value": 4020961792 }, { "step": 16, "value": 3922270208 }, { "step": 17, "value": 3922270208 }, { "step": 18, "value": 2288886272 }, { "step": 19, "value": 2095426048 }, { "step": 20, "value": 2095426048 }, { "step": 21, "value": 2028317184 }, { "step": 22, "value": 2028317184 }, { "step": 23, "value": 1883730944 }, { "step": 24, "value": 1883730944 }, { "step": 25, "value": 1849141248 }, { "step": 26, "value": 1790420992 }, { "step": 27, "value": 1636208640 }, { "step": 28, "value": 1609008640 }, { "step": 29, "value": 1603241472 }, { "step": 30, "value": 1603241472 }, { "step": 31, "value": 1603241472 }, { "step": 32, "value": 1444447744 }, { "step": 33, "value": 1268745216 }, { "step": 34, "value": 1268745216 }, { "step": 35, "value": 1109951488 }, { "step": 36, "value": 1109951488 }, { "step": 37, "value": 1109951488 }, { "step": 38, "value": 1109951488 }, { "step": 39, "value": 770148352 }, { "step": 40, "value": 770148352 }, { "step": 41, "value": 770148352 }, { "step": 42, "value": 770148352 }, { "step": 43, "value": 703039488 }, { "step": 44, "value": 703039488 }, { "step": 45, "value": 703039488 }, { "step": 46, "value": 703039488 }, { "step": 47, "value": 703038464 }, { "step": 48, "value": 703038464 }, { "step": 49, "value": 703038464 }, { "step": 50, "value": 703038464 }, { "step": 51, "value": 677872640 }, { "step": 52, "value": 677872640 }, { "step": 53, "value": 677872640 }, { "step": 54, "value": 677872640 }, { "step": 55, "value": 677872640 }, { "step": 56, "value": 677872640 }, { "step": 57, "value": 677872640 }, { "step": 58, "value": 636964864 }, { "step": 59, "value": 636964864 }, { "step": 60, "value": 590228992 }, { "step": 61, "value": 562871296 }, { "step": 62, "value": 562871296 }, { "step": 63, "value": 562871296 }, { "step": 64, "value": 562871296 }, { "step": 65, "value": 487105536 }, { "step": 66, "value": 487105536 }, { "step": 67, "value": 487105536 }, { "step": 68, "value": 487105536 }, { "step": 69, "value": 487105536 }, { "step": 70, "value": 487105536 }, { "step": 71, "value": 487105536 }, { "step": 72, "value": 487105536 }, { "step": 73, "value": 487105536 }, { "step": 74, "value": 487105536 }, { "step": 75, "value": 487105536 }, { "step": 76, "value": 487105536 }, { "step": 77, "value": 478717952 }, { "step": 78, "value": 478717952 }, { "step": 79, "value": 478717952 }, { "step": 80, "value": 466297856 }, { "step": 81, "value": 466297856 }, { "step": 82, "value": 466297856 }, { "step": 83, "value": 466297856 }, { "step": 84, "value": 466297856 }, { "step": 85, "value": 466297856 }, { "step": 86, "value": 466297856 }, { "step": 87, "value": 466297856 }, { "step": 88, "value": 466297856 }, { "step": 89, "value": 466297856 }, { "step": 90, "value": 466297856 }, { "step": 91, "value": 466297856 }, { "step": 92, "value": 466297856 }, { "step": 93, "value": 466297856 }, { "step": 94, "value": 466297856 }, { "step": 95, "value": 466297856 }, { "step": 96, "value": 466297856 }, { "step": 97, "value": 466297856 }, { "step": 98, "value": 466297856 }, { "step": 99, "value": 466297856 }, { "step": 100, "value": 466297856 } ], "submission_sha256": "294cb7c4f0ebf4c153b30376319d2475ae962cc6e0be381b966c2b7b3e5a1630", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/gear/memory/submission.json", "comparison_value": 466297856, "comparison_rank_key": [ 466297856.0, 247726080.0, -2.3452188527617324e-05 ], "rank": 2 }, { "run_id": "autoscientists--memory", "method_id": "autoscientists", "task_id": "memory", "metric": "peak_vram_bytes", "unit": "bytes", "direction": "minimize", "best_value": 559877120, "reference_value": 47198976512, "relative_improvement": 0.9881379393924431, "rank_key": [ 559877120.0, 177012736.0, -0.0011416618389190614 ], "best_launch_seq": 54, "best_candidate_id": "exp-7d60738495c689b8f0d9aac5", "best_is_reference": false, "budget": { "spent": 65, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 2, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 2 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 13.787280555555558, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 47198976512, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-cd08f0a734f080978548ce51", "value": 6134700032, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp-df4f9c73d4eb49743df68f68", "value": 4978614784, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-ff62af370f1f104302dd611c", "value": 3184859136, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-a8fd44201a80dba543b30e20", "value": 4524087296, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-37b08a30480d9a20058b5c83", "value": 2379552768, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-2faa1d0b836a29e0903b2d4b", "value": 2379552768, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp-da79ea4ad45dc768099cd9d1", "value": 2373893120, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp-40ee82e66090e375cd8a94fa", "value": 2305735680, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-d8f6f7fc2fee62f4f0f8423d", "value": 2238625792, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-2106b2dac71432bbf9236c9b", "value": 2097065984, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-c46aabb63d6b4066baa48a7b", "value": 2122231808, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-2814391e7dfc5073a1dee135", "value": 1963110400, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-ee603079aac07b5f8c2b4964", "value": 2097065984, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp-b6dd38ac91b1a93984547254", "value": 1828892672, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-2fcd1c0e82a203318d8cd269", "value": 1465700352, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-f73b62780f965ee95544d462", "value": 1102508032, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-01a041d02b7ce7644d4eb2f5", "value": 1102507520, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp-53cd2850a2091d48cf13ba40", "value": 1027969024, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp-bc7c87eb64a619642fd450e2", "value": 910528512, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp-346311620b4563c474326b4e", "value": 910528512, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-e5f2dd2f5f6663d4715b0b28", "value": 910528512, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp-79ba6b919da0acf84941b174", "value": 905809920, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp-ee4477e9b1516191de3e288e", "value": 847089664, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp-32716ac7fa05f0f8a28325d5", "value": 839544832, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp-6bedcbcfab9279e64087feaa", "value": 928402944, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-378ff3e8f2400f7fd7d44709", "value": 779984896, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-340a0f852a0edf24abdc077a", "value": 779984896, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-00b57328e2fa758c2709a3dc", "value": 763207680, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-b6c5498e304f7798620193a6", "value": 738025472, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp-1802db57ca67b2f51f56adf1", "value": 721248256, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp-c76e73263e118c404a86a46b", "value": 721248256, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp-02a6b99db0ef55559010bb85", "value": 671125504, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "exp-93f9ff7c7b46db824eb968d0", "value": 853753856, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp-c9009f41199c57ddcb90875c", "value": 666931200, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "exp-b09c95f6a8940d21ffa258d2", "value": 632525824, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp-52a2da923f7c92226d1f5fa8", "value": 627084288, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp-77ab72081ab5161053671c55", "value": 622889984, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "exp-de4908d39dbffd1e231dceb5", "value": 589335552, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp-f921adde94faa7ed62dc7b69", "value": 572558336, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp-e1cba187c488671e48833149", "value": 589987840, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp-af729590a56a1e72e997312d", "value": 572176896, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "exp-0b42400a553758118e8bc6fa", "value": 572176896, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp-7d60738495c689b8f0d9aac5", "value": 559877120, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp-ead96cf9dbc8a89f45dca65d", "value": 572558336, "is_reference": false } ], "curve": [ { "step": 1, "value": 47198976512 }, { "step": 2, "value": 6134700032 }, { "step": 3, "value": 4978614784 }, { "step": 4, "value": 3184859136 }, { "step": 5, "value": 3184859136 }, { "step": 6, "value": 2379552768 }, { "step": 7, "value": 2379552768 }, { "step": 8, "value": 2373893120 }, { "step": 9, "value": 2305735680 }, { "step": 10, "value": 2305735680 }, { "step": 11, "value": 2238625792 }, { "step": 12, "value": 2097065984 }, { "step": 13, "value": 2097065984 }, { "step": 14, "value": 1963110400 }, { "step": 15, "value": 1963110400 }, { "step": 16, "value": 1828892672 }, { "step": 17, "value": 1465700352 }, { "step": 18, "value": 1465700352 }, { "step": 19, "value": 1102508032 }, { "step": 20, "value": 1102508032 }, { "step": 21, "value": 1102507520 }, { "step": 22, "value": 1027969024 }, { "step": 23, "value": 910528512 }, { "step": 24, "value": 910528512 }, { "step": 25, "value": 910528512 }, { "step": 26, "value": 910528512 }, { "step": 27, "value": 905809920 }, { "step": 28, "value": 905809920 }, { "step": 29, "value": 847089664 }, { "step": 30, "value": 847089664 }, { "step": 31, "value": 839544832 }, { "step": 32, "value": 839544832 }, { "step": 33, "value": 779984896 }, { "step": 34, "value": 779984896 }, { "step": 35, "value": 763207680 }, { "step": 36, "value": 738025472 }, { "step": 37, "value": 721248256 }, { "step": 38, "value": 721248256 }, { "step": 39, "value": 671125504 }, { "step": 40, "value": 671125504 }, { "step": 41, "value": 666931200 }, { "step": 42, "value": 632525824 }, { "step": 43, "value": 632525824 }, { "step": 44, "value": 627084288 }, { "step": 45, "value": 622889984 }, { "step": 46, "value": 622889984 }, { "step": 47, "value": 622889984 }, { "step": 48, "value": 622889984 }, { "step": 49, "value": 589335552 }, { "step": 50, "value": 572558336 }, { "step": 51, "value": 572558336 }, { "step": 52, "value": 572176896 }, { "step": 53, "value": 572176896 }, { "step": 54, "value": 559877120 }, { "step": 55, "value": 559877120 }, { "step": 56, "value": 559877120 }, { "step": 57, "value": 559877120 }, { "step": 58, "value": 559877120 }, { "step": 59, "value": 559877120 }, { "step": 60, "value": 559877120 }, { "step": 61, "value": 559877120 }, { "step": 62, "value": 559877120 }, { "step": 63, "value": 559877120 }, { "step": 64, "value": 559877120 }, { "step": 65, "value": 559877120 } ], "submission_sha256": "7ef643e9295e2e73f9c7644fdfdfff00dcc038aa4dc68fa9783396a6a0f57939", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/autoscientists/memory/submission.json", "comparison_value": 559877120, "comparison_rank_key": [ 559877120.0, 177012736.0, -0.0011416618389190614 ], "rank": 3 }, { "run_id": "sequential-search--memory", "method_id": "sequential-search", "task_id": "memory", "metric": "peak_vram_bytes", "unit": "bytes", "direction": "minimize", "best_value": 608924672, "reference_value": 47198976512, "relative_improvement": 0.9870987738082586, "rank_key": [ 608924672.0, 463470592.0, -0.0005135860325595765 ], "best_launch_seq": 87, "best_candidate_id": "c0100", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "evaluation_capacity_gpus": 1, "gpu_instance_type": null, "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "gpus_per_evaluation": 1, "workers": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 20.995725833333335, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 47198976512, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "c0001", "value": 12270385664, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "c0002", "value": 6134700032, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "c0004", "value": 5060958720, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "c0005", "value": 5060958720, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "c0006", "value": 4978614784, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "c0007", "value": 4335352320, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "c0010", "value": 2896781824, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "c0011", "value": 2897211904, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "c0012", "value": 2896781824, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "c0013", "value": 2632849920, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "c0014", "value": 2632849920, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "c0015", "value": 1954813440, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "c0016", "value": 1627522560, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "c0018", "value": 1545149440, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "c0021", "value": 1545149440, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "c0022", "value": 1545280512, "is_reference": false }, { "step": 26, "launch_seq": 27, "candidate_id": "c0026", "value": 1545149440, "is_reference": false }, { "step": 28, "launch_seq": 29, "candidate_id": "c0028", "value": 1515445248, "is_reference": false }, { "step": 29, "launch_seq": 30, "candidate_id": "c0029", "value": 1487805440, "is_reference": false }, { "step": 33, "launch_seq": 34, "candidate_id": "c0033", "value": 1477139456, "is_reference": false }, { "step": 34, "launch_seq": 35, "candidate_id": "c0034", "value": 1465588736, "is_reference": false }, { "step": 35, "launch_seq": 36, "candidate_id": "c0035", "value": 1424815616, "is_reference": false }, { "step": 36, "launch_seq": 37, "candidate_id": "c0036", "value": 1410459136, "is_reference": false }, { "step": 38, "launch_seq": 39, "candidate_id": "c0038", "value": 1029445120, "is_reference": false }, { "step": 39, "launch_seq": 40, "candidate_id": "c0039", "value": 952047104, "is_reference": false }, { "step": 40, "launch_seq": 41, "candidate_id": "c0040", "value": 952047104, "is_reference": false }, { "step": 41, "launch_seq": 42, "candidate_id": "c0041", "value": 971331072, "is_reference": false }, { "step": 44, "launch_seq": 45, "candidate_id": "c0044", "value": 1026381312, "is_reference": false }, { "step": 47, "launch_seq": 48, "candidate_id": "c0047", "value": 957650432, "is_reference": false }, { "step": 48, "launch_seq": 49, "candidate_id": "c0048", "value": 855053824, "is_reference": false }, { "step": 49, "launch_seq": 50, "candidate_id": "c0049", "value": 851550720, "is_reference": false }, { "step": 50, "launch_seq": 51, "candidate_id": "c0050", "value": 851550720, "is_reference": false }, { "step": 51, "launch_seq": 52, "candidate_id": "c0051", "value": 851550720, "is_reference": false }, { "step": 52, "launch_seq": 53, "candidate_id": "c0052", "value": 851550720, "is_reference": false }, { "step": 53, "launch_seq": 54, "candidate_id": "c0053", "value": 902439424, "is_reference": false }, { "step": 57, "launch_seq": 58, "candidate_id": "c0057", "value": 846045696, "is_reference": false }, { "step": 58, "launch_seq": 59, "candidate_id": "c0058", "value": 846045696, "is_reference": false }, { "step": 59, "launch_seq": 60, "candidate_id": "c0059", "value": 846045696, "is_reference": false }, { "step": 64, "launch_seq": 65, "candidate_id": "c0064", "value": 647693312, "is_reference": false }, { "step": 65, "launch_seq": 66, "candidate_id": "c0066", "value": 628315648, "is_reference": false }, { "step": 68, "launch_seq": 69, "candidate_id": "c0069", "value": 628315648, "is_reference": false }, { "step": 69, "launch_seq": 70, "candidate_id": "c0070", "value": 628315648, "is_reference": false }, { "step": 73, "launch_seq": 74, "candidate_id": "c0075", "value": 628315648, "is_reference": false }, { "step": 75, "launch_seq": 76, "candidate_id": "c0074", "value": 628315648, "is_reference": false }, { "step": 76, "launch_seq": 77, "candidate_id": "c0079", "value": 623297536, "is_reference": false }, { "step": 77, "launch_seq": 78, "candidate_id": "c0080", "value": 623297536, "is_reference": false }, { "step": 78, "launch_seq": 79, "candidate_id": "c0082", "value": 623297536, "is_reference": false }, { "step": 81, "launch_seq": 82, "candidate_id": "c0085", "value": 623297536, "is_reference": false }, { "step": 82, "launch_seq": 83, "candidate_id": "c0086", "value": 610104320, "is_reference": false }, { "step": 85, "launch_seq": 86, "candidate_id": "c0099", "value": 610132992, "is_reference": false }, { "step": 86, "launch_seq": 87, "candidate_id": "c0100", "value": 608924672, "is_reference": false }, { "step": 88, "launch_seq": 89, "candidate_id": "c0102", "value": 610104320, "is_reference": false }, { "step": 89, "launch_seq": 90, "candidate_id": "c0103", "value": 608924672, "is_reference": false }, { "step": 90, "launch_seq": 91, "candidate_id": "c0106", "value": 608924672, "is_reference": false }, { "step": 91, "launch_seq": 92, "candidate_id": "c0107", "value": 608924672, "is_reference": false }, { "step": 92, "launch_seq": 93, "candidate_id": "c0108", "value": 608924672, "is_reference": false }, { "step": 93, "launch_seq": 94, "candidate_id": "c0110", "value": 647515136, "is_reference": false }, { "step": 94, "launch_seq": 95, "candidate_id": "c0111", "value": 648275968, "is_reference": false }, { "step": 95, "launch_seq": 96, "candidate_id": "c0113", "value": 656453632, "is_reference": false }, { "step": 96, "launch_seq": 97, "candidate_id": "c0114", "value": 651095040, "is_reference": false }, { "step": 97, "launch_seq": 98, "candidate_id": "c0115", "value": 698078720, "is_reference": false }, { "step": 98, "launch_seq": 99, "candidate_id": "c0116", "value": 692196864, "is_reference": false }, { "step": 99, "launch_seq": 100, "candidate_id": "c0117", "value": 685807104, "is_reference": false }, { "step": 100, "launch_seq": 101, "candidate_id": "c0118", "value": 685282816, "is_reference": false } ], "curve": [ { "step": 1, "value": 47198976512 }, { "step": 2, "value": 12270385664 }, { "step": 3, "value": 6134700032 }, { "step": 4, "value": 6134700032 }, { "step": 5, "value": 5060958720 }, { "step": 6, "value": 5060958720 }, { "step": 7, "value": 4978614784 }, { "step": 8, "value": 4335352320 }, { "step": 9, "value": 4335352320 }, { "step": 10, "value": 4335352320 }, { "step": 11, "value": 2896781824 }, { "step": 12, "value": 2896781824 }, { "step": 13, "value": 2896781824 }, { "step": 14, "value": 2632849920 }, { "step": 15, "value": 2632849920 }, { "step": 16, "value": 1954813440 }, { "step": 17, "value": 1627522560 }, { "step": 18, "value": 1545149440 }, { "step": 19, "value": 1545149440 }, { "step": 20, "value": 1545149440 }, { "step": 21, "value": 1545149440 }, { "step": 22, "value": 1545149440 }, { "step": 23, "value": 1545149440 }, { "step": 24, "value": 1545149440 }, { "step": 25, "value": 1545149440 }, { "step": 26, "value": 1545149440 }, { "step": 27, "value": 1545149440 }, { "step": 28, "value": 1515445248 }, { "step": 29, "value": 1487805440 }, { "step": 30, "value": 1487805440 }, { "step": 31, "value": 1487805440 }, { "step": 32, "value": 1487805440 }, { "step": 33, "value": 1477139456 }, { "step": 34, "value": 1465588736 }, { "step": 35, "value": 1424815616 }, { "step": 36, "value": 1410459136 }, { "step": 37, "value": 1410459136 }, { "step": 38, "value": 1029445120 }, { "step": 39, "value": 952047104 }, { "step": 40, "value": 952047104 }, { "step": 41, "value": 952047104 }, { "step": 42, "value": 952047104 }, { "step": 43, "value": 952047104 }, { "step": 44, "value": 952047104 }, { "step": 45, "value": 952047104 }, { "step": 46, "value": 952047104 }, { "step": 47, "value": 952047104 }, { "step": 48, "value": 855053824 }, { "step": 49, "value": 851550720 }, { "step": 50, "value": 851550720 }, { "step": 51, "value": 851550720 }, { "step": 52, "value": 851550720 }, { "step": 53, "value": 851550720 }, { "step": 54, "value": 851550720 }, { "step": 55, "value": 851550720 }, { "step": 56, "value": 851550720 }, { "step": 57, "value": 846045696 }, { "step": 58, "value": 846045696 }, { "step": 59, "value": 846045696 }, { "step": 60, "value": 846045696 }, { "step": 61, "value": 846045696 }, { "step": 62, "value": 846045696 }, { "step": 63, "value": 846045696 }, { "step": 64, "value": 647693312 }, { "step": 65, "value": 628315648 }, { "step": 66, "value": 628315648 }, { "step": 67, "value": 628315648 }, { "step": 68, "value": 628315648 }, { "step": 69, "value": 628315648 }, { "step": 70, "value": 628315648 }, { "step": 71, "value": 628315648 }, { "step": 72, "value": 628315648 }, { "step": 73, "value": 628315648 }, { "step": 74, "value": 628315648 }, { "step": 75, "value": 628315648 }, { "step": 76, "value": 623297536 }, { "step": 77, "value": 623297536 }, { "step": 78, "value": 623297536 }, { "step": 79, "value": 623297536 }, { "step": 80, "value": 623297536 }, { "step": 81, "value": 623297536 }, { "step": 82, "value": 610104320 }, { "step": 83, "value": 610104320 }, { "step": 84, "value": 610104320 }, { "step": 85, "value": 610104320 }, { "step": 86, "value": 608924672 }, { "step": 87, "value": 608924672 }, { "step": 88, "value": 608924672 }, { "step": 89, "value": 608924672 }, { "step": 90, "value": 608924672 }, { "step": 91, "value": 608924672 }, { "step": 92, "value": 608924672 }, { "step": 93, "value": 608924672 }, { "step": 94, "value": 608924672 }, { "step": 95, "value": 608924672 }, { "step": 96, "value": 608924672 }, { "step": 97, "value": 608924672 }, { "step": 98, "value": 608924672 }, { "step": 99, "value": 608924672 }, { "step": 100, "value": 608924672 } ], "submission_sha256": "46f25a13d2cdf6ec554d0ed8706e39e4fe37e969df455f7038f6e625d7da1d49", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/sequential-search/memory/submission.json", "comparison_value": 608924672, "comparison_rank_key": [ 608924672.0, 463470592.0, -0.0005135860325595765 ], "rank": 4 }, { "run_id": "beam-search--memory", "method_id": "beam-search", "task_id": "memory", "metric": "peak_vram_bytes", "unit": "bytes", "direction": "minimize", "best_value": 1020860928, "reference_value": 47198976512, "relative_improvement": 0.9783711215064069, "rank_key": [ 1020860928.0, 238813184.0, -0.014643390739156992 ], "best_launch_seq": 90, "best_candidate_id": "g13-normstore-on-qkfold-recomp", "best_is_reference": false, "budget": { "spent": 97, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 4, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 4 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": null }, "evaluation_gpu_hours": 18.645728055555555, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 47198976512, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "g1-chunked-head", "value": 47337650688, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "g1-mlp-recompute", "value": 39668103680, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "g2-microbatch-64", "value": 20140472832, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "g2-norm-folded-proj", "value": 42876744192, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "g2-mlp-recompute-on-head", "value": 39806777856, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "g2-qk-rotary-norm-fold", "value": 35373136384, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "g2-microbatch-32-on-head", "value": 11955918336, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "g3-mlp-recompute-on-mb32-head", "value": 10062190080, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "g3-microbatch-16-chunked-head", "value": 5238965760, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "g3-qk-fold-32-chunked-head", "value": 8988448256, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "g3-resid-mix-norm-recompute", "value": 33466823168, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "g3-microbatch-16-on-head", "value": 6193169920, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "g3-train-seqlen-1024-reshape", "value": 20140472832, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "g3-norm-fold-on-qk-fold", "value": 31050904064, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "g3-half-step-token-granularity", "value": 39508717056, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "g4-microbatch-4-chunked-head", "value": 4978483712, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "g4-norm-fold-16-on-head", "value": 5629033984, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "g4-qk-fold-16-chunked-head", "value": 4978876928, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "g4-mix-norm-fold-mb32-head", "value": 8491421184, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "g4-half-batch-tokens-mb16", "value": 5238965760, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "g4-microbatch-8-on-head", "value": 4978614784, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "g4-head-chunk-1024-on-qk-fold", "value": 9337584128, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "g5-eval-rowblock-mb4", "value": 1913043456, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "g5-eval-rowblock-mb8", "value": 4459141120, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "g5-mix-norm-fold-evalblock-16", "value": 3903077888, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "g5-qk-fold-evalblock-halfbatch", "value": 4702094848, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "g5-half-update-tokens-mb4", "value": 4978483712, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "g5-mlp-fold-evalblock-mb8", "value": 3016097280, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "g5-rows-8-evalblock-qk16", "value": 2747661824, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "g5-rows-8-evalblock-halfbatch", "value": 3016097280, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "g6-normfold-qk16-mb8", "value": 2451961344, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "g6-mixnorm-fold-mlpfold-mb8", "value": 2602956288, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "g6-fc-recompute-span-refund", "value": 2600861184, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "g6-halfupdate-evalblock-mb4", "value": 1913043456, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "g6-int8-relu-retention-qk16", "value": 2412379648, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "g6-muon-ramp-token-basis-mb8", "value": 3016097280, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "g7-headchunk1024-mb4", "value": 1807115776, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "g7-normfold-int8-qk16", "value": 2115630592, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "g7-halfupdate-normfold-qk16", "value": 2451961344, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "g7-qkfold-fcrecompute", "value": 2332425728, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "g7-int8-relu-mb4", "value": 1745402368, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "g7-fcrecompute-int8-qk16", "value": 2198273536, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "g7-mixfold-normfold-qk16", "value": 2334520832, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "g7-bf16-matrix-params", "value": 2426696192, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "g8-bf16-matrix-params-mb4", "value": 1558654464, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "g8-int8-headchunk1024-mb4", "value": 1639474688, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "g8-fcrecompute-span-mb4", "value": 1699133952, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "g8-int4-relu-qk16", "value": 1981412864, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "g8-qkfold-int8-mb4", "value": 1611184640, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "g8-head-inplace-grads-1024", "value": 1807115776, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "g8-normfold-mb4", "value": 1751560704, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "g8-mixfold-normfold-int8", "value": 1998190080, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "g9-qkfold-bf16-mb4", "value": 1424436736, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "g9-normfold-qkfold-int8", "value": 1449701888, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "g9-bf16-params-hc1024", "value": 1452726784, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "g9-bf16-params-fcrecomp", "value": 1524968960, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "g9-spanclamp-only-bf16", "value": 1558654464, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "g9-headchunk1024-qkfold", "value": 1505256960, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "g9-headchunk256-int8", "value": 1808039424, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "g9-qkfold-fcrecomp-mb4", "value": 1564916224, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "g10-normfold-qkfold-bf16", "value": 1290219008, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "g10-hc1024-normfold-qkfold", "value": 1343774208, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "g10-normfold-bf16-hc1024", "value": 1318509056, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "g10-bf16-qkfold-hc1024", "value": 1318509056, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "g10-halfupdate-qkfold-bf16", "value": 1424436736, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "g10-mixfold-normfold-mb4", "value": 1390981632, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "g10-fcrecomp-bf16-hc1024", "value": 1351965184, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "g10-int4-relu-qkfold-hc1024", "value": 1438148096, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "g11-hc1024-on-leader", "value": 1184291328, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "g11-normfc-recompute-clamp", "value": 1217747456, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "g11-bf16-unembedding", "value": 1248949760, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "g11-int8-normstore-fp32", "value": 1281105408, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "g11-int8-normstore-bf16", "value": 1227550208, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "g11-mixfold-bf16-hc1024", "value": 1259788800, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "g11-muon-state-bf16", "value": 1293441536, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "g12-mixfold-on-leader", "value": 1125571072, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "g12-qkfold-on-normfc-recompute", "value": 1083529728, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "g12-hc1024-on-normstore", "value": 1121622528, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "g12-normfold-on-bf16-unembed", "value": 1123120640, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "g12-int4-relu-on-leader", "value": 1117182464, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "g12-normstore-on-normfc", "value": 1155078656, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "g12-bf16-unembed-on-normstore", "value": 1185607168, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "g12-head-bwd-inplace-scale", "value": 1248949760, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "g13-normstore-on-qkfold-recomp", "value": 1020860928, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "g13-mixfold-on-int4-relu", "value": 1058462208, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "g13-int4-relu-on-normstore", "value": 1054513664, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "g13-int4-relu-on-bf16-unembed", "value": 1064400384, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "g13-bf16-unembed-on-recompute", "value": 1022359040, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "g13-fcrecompute-on-int4-relu", "value": 1066752512, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "g13-bf16-unembed-normstore-hc1024", "value": 1038740992, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "g13-mixfold-on-bf16-unembed", "value": 1064400384, "is_reference": false } ], "curve": [ { "step": 1, "value": 47198976512 }, { "step": 2, "value": 47198976512 }, { "step": 3, "value": 39668103680 }, { "step": 4, "value": 20140472832 }, { "step": 5, "value": 20140472832 }, { "step": 6, "value": 20140472832 }, { "step": 7, "value": 20140472832 }, { "step": 8, "value": 20140472832 }, { "step": 9, "value": 11955918336 }, { "step": 10, "value": 10062190080 }, { "step": 11, "value": 5238965760 }, { "step": 12, "value": 5238965760 }, { "step": 13, "value": 5238965760 }, { "step": 14, "value": 5238965760 }, { "step": 15, "value": 5238965760 }, { "step": 16, "value": 5238965760 }, { "step": 17, "value": 5238965760 }, { "step": 18, "value": 4978483712 }, { "step": 19, "value": 4978483712 }, { "step": 20, "value": 4978483712 }, { "step": 21, "value": 4978483712 }, { "step": 22, "value": 4978483712 }, { "step": 23, "value": 4978483712 }, { "step": 24, "value": 4978483712 }, { "step": 25, "value": 4978483712 }, { "step": 26, "value": 1913043456 }, { "step": 27, "value": 1913043456 }, { "step": 28, "value": 1913043456 }, { "step": 29, "value": 1913043456 }, { "step": 30, "value": 1913043456 }, { "step": 31, "value": 1913043456 }, { "step": 32, "value": 1913043456 }, { "step": 33, "value": 1913043456 }, { "step": 34, "value": 1913043456 }, { "step": 35, "value": 1913043456 }, { "step": 36, "value": 1913043456 }, { "step": 37, "value": 1913043456 }, { "step": 38, "value": 1913043456 }, { "step": 39, "value": 1913043456 }, { "step": 40, "value": 1913043456 }, { "step": 41, "value": 1913043456 }, { "step": 42, "value": 1807115776 }, { "step": 43, "value": 1807115776 }, { "step": 44, "value": 1807115776 }, { "step": 45, "value": 1807115776 }, { "step": 46, "value": 1745402368 }, { "step": 47, "value": 1745402368 }, { "step": 48, "value": 1745402368 }, { "step": 49, "value": 1745402368 }, { "step": 50, "value": 1558654464 }, { "step": 51, "value": 1558654464 }, { "step": 52, "value": 1558654464 }, { "step": 53, "value": 1558654464 }, { "step": 54, "value": 1558654464 }, { "step": 55, "value": 1558654464 }, { "step": 56, "value": 1558654464 }, { "step": 57, "value": 1558654464 }, { "step": 58, "value": 1424436736 }, { "step": 59, "value": 1424436736 }, { "step": 60, "value": 1424436736 }, { "step": 61, "value": 1424436736 }, { "step": 62, "value": 1424436736 }, { "step": 63, "value": 1424436736 }, { "step": 64, "value": 1424436736 }, { "step": 65, "value": 1424436736 }, { "step": 66, "value": 1290219008 }, { "step": 67, "value": 1290219008 }, { "step": 68, "value": 1290219008 }, { "step": 69, "value": 1290219008 }, { "step": 70, "value": 1290219008 }, { "step": 71, "value": 1290219008 }, { "step": 72, "value": 1290219008 }, { "step": 73, "value": 1290219008 }, { "step": 74, "value": 1184291328 }, { "step": 75, "value": 1184291328 }, { "step": 76, "value": 1184291328 }, { "step": 77, "value": 1184291328 }, { "step": 78, "value": 1184291328 }, { "step": 79, "value": 1184291328 }, { "step": 80, "value": 1184291328 }, { "step": 81, "value": 1184291328 }, { "step": 82, "value": 1125571072 }, { "step": 83, "value": 1083529728 }, { "step": 84, "value": 1083529728 }, { "step": 85, "value": 1083529728 }, { "step": 86, "value": 1083529728 }, { "step": 87, "value": 1083529728 }, { "step": 88, "value": 1083529728 }, { "step": 89, "value": 1083529728 }, { "step": 90, "value": 1020860928 }, { "step": 91, "value": 1020860928 }, { "step": 92, "value": 1020860928 }, { "step": 93, "value": 1020860928 }, { "step": 94, "value": 1020860928 }, { "step": 95, "value": 1020860928 }, { "step": 96, "value": 1020860928 }, { "step": 97, "value": 1020860928 } ], "submission_sha256": "cf937e7db14f365df591509e9a998ea515d8e6de1bc7c5f670e7f034b74227b7", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/beam-search/memory/submission.json", "comparison_value": 1020860928, "comparison_rank_key": [ 1020860928.0, 238813184.0, -0.014643390739156992 ], "rank": 5 }, { "run_id": "heuresis-map-elites--memory", "method_id": "heuresis-map-elites", "task_id": "memory", "metric": "peak_vram_bytes", "unit": "bytes", "direction": "minimize", "best_value": 1698522112, "reference_value": 47198976512, "relative_improvement": 0.9640135817019642, "rank_key": [ 1698522112.0, 289931264.0, -0.0061748443354883165 ], "best_launch_seq": 54, "best_candidate_id": "exp-f0b7d00f6b9cbac0ef36cfb1", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 17.462985833333338, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 47198976512, "is_reference": true }, { "step": 3, "launch_seq": 3, "candidate_id": "exp-d613f783e93e4dd2e56a4bda", "value": 13464197120, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-01f9a1cd6064cdcd1cd754c0", "value": 7105890304, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-1efd31f589e55bfdd04ac3c7", "value": 4988082688, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-94d10d7f77b8704634f8eb03", "value": 2844698112, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp-bed28ebcec00415b77e483d3", "value": 2846627840, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-26040eae46d06ec1038ffad5", "value": 2104252928, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-34955f7360d73df350e8027d", "value": 2070721024, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-a034b32718291f4bca463398", "value": 3429177344, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-b5f8484f5e7636cbae477abd", "value": 4299438080, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp-0c8b2a46dfa0d6a8f8bfcc2f", "value": 3429177344, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-b7556861a369c75d1f9da051", "value": 3131389952, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-03d6d668308b543e0166a28e", "value": 2835046912, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp-936dbe201d51398c04a43166", "value": 2505793024, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-fee4caaa4b1525df98eb35ce", "value": 6128932864, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-32ed9b6f46465c2adf6e886a", "value": 11988155904, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-5a8afc0c1f64de16a7d21cf7", "value": 6680362496, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp-ab69d59b67f37156bd77a13b", "value": 3411023872, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp-d5b8f1c68ea96ee6953851ea", "value": 6112155648, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp-c50fdd69a5654787fb8ab8bb", "value": 3163363328, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp-378d1b44730a7f6c88e5095f", "value": 6061824000, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp-70810bcb0871ea8b7693b6bc", "value": 3113031680, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "exp-0ba2cddb945971dd7db97b74", "value": 3062697984, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp-bcf81fcf9925f754f17fcb76", "value": 2374859264, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp-12b2f822f4494693671996fd", "value": 2068658688, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp-f0b7d00f6b9cbac0ef36cfb1", "value": 1698522112, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp-466d6cde57f5096778c77b39", "value": 5060171776, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp-378e183bb915c7a5c8ed9c6f", "value": 2644580352, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "exp-7109b2e53adff9f964935323", "value": 6128932864, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "exp-5839397f11b1b858de2751c1", "value": 3180140544, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp-32fef585e72987bf68f93bab", "value": 2525827072, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp-26f375c7bad793559c2b742f", "value": 6128932864, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp-7b46fb86eaa0f97bbbf19b79", "value": 3450050560, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "exp-768ff20e40cb5feabe6bd706", "value": 3181451264, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "exp-365352f3d9c27c553dc31acd", "value": 3956546560, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp-445e2296ff85932e5424f169", "value": 2467217920, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "exp-2c74aea23cf0f8c24a01dcb4", "value": 2307199488, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "exp-9a15d16d2b242f9010d7c5e5", "value": 2318409216, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "exp-1ce97eb32363ef9fea106873", "value": 2307199488, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "exp-45c53f0a309297ac69515abb", "value": 6128932864, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "exp-e8fc98b012c6fb106a24f30a", "value": 3181451264, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "exp-705aadc905027fe10997c514", "value": 6128932864, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "exp-917484b3af848d66663098f6", "value": 3431864320, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "exp-3c875ebffe7cf435fbe9cf4a", "value": 3331198976, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "exp-053054a52add7badc850a7dc", "value": 6128932864, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp-f0169d272757b8ec7cd8fd4e", "value": 3150908928, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "exp-e8e8a23d3b949f31e2422c19", "value": 2861501952, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "exp-d86de6f8660a07454ce264c5", "value": 6128932864, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "exp-0d8ee926361c42ffa764cd34", "value": 3449952256, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "exp-6d11012d5606902c1ee45a84", "value": 2778945536, "is_reference": false }, { "step": 98, "launch_seq": 98, "candidate_id": "exp-374fb66af8aa934221df12b9", "value": 6134700032, "is_reference": false }, { "step": 100, "launch_seq": 100, "candidate_id": "exp-f4affd41e7a8ff18242ca940", "value": 3183810560, "is_reference": false } ], "curve": [ { "step": 1, "value": 47198976512 }, { "step": 2, "value": 47198976512 }, { "step": 3, "value": 13464197120 }, { "step": 4, "value": 7105890304 }, { "step": 5, "value": 4988082688 }, { "step": 6, "value": 4988082688 }, { "step": 7, "value": 2844698112 }, { "step": 8, "value": 2844698112 }, { "step": 9, "value": 2844698112 }, { "step": 10, "value": 2844698112 }, { "step": 11, "value": 2844698112 }, { "step": 12, "value": 2104252928 }, { "step": 13, "value": 2070721024 }, { "step": 14, "value": 2070721024 }, { "step": 15, "value": 2070721024 }, { "step": 16, "value": 2070721024 }, { "step": 17, "value": 2070721024 }, { "step": 18, "value": 2070721024 }, { "step": 19, "value": 2070721024 }, { "step": 20, "value": 2070721024 }, { "step": 21, "value": 2070721024 }, { "step": 22, "value": 2070721024 }, { "step": 23, "value": 2070721024 }, { "step": 24, "value": 2070721024 }, { "step": 25, "value": 2070721024 }, { "step": 26, "value": 2070721024 }, { "step": 27, "value": 2070721024 }, { "step": 28, "value": 2070721024 }, { "step": 29, "value": 2070721024 }, { "step": 30, "value": 2070721024 }, { "step": 31, "value": 2070721024 }, { "step": 32, "value": 2070721024 }, { "step": 33, "value": 2070721024 }, { "step": 34, "value": 2070721024 }, { "step": 35, "value": 2070721024 }, { "step": 36, "value": 2070721024 }, { "step": 37, "value": 2070721024 }, { "step": 38, "value": 2070721024 }, { "step": 39, "value": 2070721024 }, { "step": 40, "value": 2070721024 }, { "step": 41, "value": 2070721024 }, { "step": 42, "value": 2070721024 }, { "step": 43, "value": 2070721024 }, { "step": 44, "value": 2070721024 }, { "step": 45, "value": 2070721024 }, { "step": 46, "value": 2070721024 }, { "step": 47, "value": 2070721024 }, { "step": 48, "value": 2070721024 }, { "step": 49, "value": 2070721024 }, { "step": 50, "value": 2070721024 }, { "step": 51, "value": 2068658688 }, { "step": 52, "value": 2068658688 }, { "step": 53, "value": 2068658688 }, { "step": 54, "value": 1698522112 }, { "step": 55, "value": 1698522112 }, { "step": 56, "value": 1698522112 }, { "step": 57, "value": 1698522112 }, { "step": 58, "value": 1698522112 }, { "step": 59, "value": 1698522112 }, { "step": 60, "value": 1698522112 }, { "step": 61, "value": 1698522112 }, { "step": 62, "value": 1698522112 }, { "step": 63, "value": 1698522112 }, { "step": 64, "value": 1698522112 }, { "step": 65, "value": 1698522112 }, { "step": 66, "value": 1698522112 }, { "step": 67, "value": 1698522112 }, { "step": 68, "value": 1698522112 }, { "step": 69, "value": 1698522112 }, { "step": 70, "value": 1698522112 }, { "step": 71, "value": 1698522112 }, { "step": 72, "value": 1698522112 }, { "step": 73, "value": 1698522112 }, { "step": 74, "value": 1698522112 }, { "step": 75, "value": 1698522112 }, { "step": 76, "value": 1698522112 }, { "step": 77, "value": 1698522112 }, { "step": 78, "value": 1698522112 }, { "step": 79, "value": 1698522112 }, { "step": 80, "value": 1698522112 }, { "step": 81, "value": 1698522112 }, { "step": 82, "value": 1698522112 }, { "step": 83, "value": 1698522112 }, { "step": 84, "value": 1698522112 }, { "step": 85, "value": 1698522112 }, { "step": 86, "value": 1698522112 }, { "step": 87, "value": 1698522112 }, { "step": 88, "value": 1698522112 }, { "step": 89, "value": 1698522112 }, { "step": 90, "value": 1698522112 }, { "step": 91, "value": 1698522112 }, { "step": 92, "value": 1698522112 }, { "step": 93, "value": 1698522112 }, { "step": 94, "value": 1698522112 }, { "step": 95, "value": 1698522112 }, { "step": 96, "value": 1698522112 }, { "step": 97, "value": 1698522112 }, { "step": 98, "value": 1698522112 }, { "step": 99, "value": 1698522112 }, { "step": 100, "value": 1698522112 } ], "submission_sha256": "b7df5c17e4bf7911582f1b8b626bf7fa22e5b473654c1cb15cd0797fa083fc1e", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/heuresis-map-elites/memory/submission.json", "comparison_value": 1698522112, "comparison_rank_key": [ 1698522112.0, 289931264.0, -0.0061748443354883165 ], "rank": 6 }, { "run_id": "tpe--memory", "method_id": "tpe", "task_id": "memory", "metric": "peak_vram_bytes", "unit": "bytes", "direction": "minimize", "best_value": 6387505664, "reference_value": 47198976512, "relative_improvement": 0.8646685556332768, "rank_key": [ 6387505664.0, 746848256.0, -0.01050141625733092 ], "best_launch_seq": 54, "best_candidate_id": "tpe-trial-0053", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 17.025823611111107, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 47198976512, "is_reference": true }, { "step": 14, "launch_seq": 14, "candidate_id": "tpe-trial-0013", "value": 20374928384, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "tpe-trial-0014", "value": 22433042944, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "tpe-trial-0015", "value": 24562205184, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "tpe-trial-0016", "value": 22359905792, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "tpe-trial-0017", "value": 17357070336, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "tpe-trial-0021", "value": 18968845312, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "tpe-trial-0024", "value": 14323275776, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "tpe-trial-0025", "value": 6387505664, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "tpe-trial-0026", "value": 6387505664, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "tpe-trial-0027", "value": 6387505664, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "tpe-trial-0030", "value": 10946533888, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "tpe-trial-0031", "value": 10946533888, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "tpe-trial-0038", "value": 9523589632, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "tpe-trial-0039", "value": 8200786432, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "tpe-trial-0040", "value": 13171534336, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "tpe-trial-0043", "value": 9523589632, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "tpe-trial-0051", "value": 9523589632, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "tpe-trial-0052", "value": 9523589632, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "tpe-trial-0053", "value": 6387505664, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "tpe-trial-0060", "value": 18419447808, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "tpe-trial-0061", "value": 9523589632, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "tpe-trial-0063", "value": 9523589632, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "tpe-trial-0065", "value": 10946533888, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "tpe-trial-0070", "value": 7373447680, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "tpe-trial-0071", "value": 7373447680, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "tpe-trial-0072", "value": 8430956544, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "tpe-trial-0073", "value": 8430956544, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "tpe-trial-0075", "value": 9418733568, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "tpe-trial-0077", "value": 24541106176, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "tpe-trial-0080", "value": 8430956544, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "tpe-trial-0081", "value": 10939717120, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "tpe-trial-0082", "value": 7373447680, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "tpe-trial-0083", "value": 7373447680, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "tpe-trial-0084", "value": 7373447680, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "tpe-trial-0085", "value": 14323275776, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "tpe-trial-0087", "value": 7373447680, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "tpe-trial-0088-r1", "value": 9418733568, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "tpe-trial-0091", "value": 7373447680, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "tpe-trial-0092", "value": 7373447680, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "tpe-trial-0094", "value": 7373447680, "is_reference": false }, { "step": 98, "launch_seq": 98, "candidate_id": "tpe-trial-0096", "value": 12270385664, "is_reference": false } ], "curve": [ { "step": 1, "value": 47198976512 }, { "step": 2, "value": 47198976512 }, { "step": 3, "value": 47198976512 }, { "step": 4, "value": 47198976512 }, { "step": 5, "value": 47198976512 }, { "step": 6, "value": 47198976512 }, { "step": 7, "value": 47198976512 }, { "step": 8, "value": 47198976512 }, { "step": 9, "value": 47198976512 }, { "step": 10, "value": 47198976512 }, { "step": 11, "value": 47198976512 }, { "step": 12, "value": 47198976512 }, { "step": 13, "value": 47198976512 }, { "step": 14, "value": 20374928384 }, { "step": 15, "value": 20374928384 }, { "step": 16, "value": 20374928384 }, { "step": 17, "value": 20374928384 }, { "step": 18, "value": 17357070336 }, { "step": 19, "value": 17357070336 }, { "step": 20, "value": 17357070336 }, { "step": 21, "value": 17357070336 }, { "step": 22, "value": 17357070336 }, { "step": 23, "value": 17357070336 }, { "step": 24, "value": 17357070336 }, { "step": 25, "value": 14323275776 }, { "step": 26, "value": 6387505664 }, { "step": 27, "value": 6387505664 }, { "step": 28, "value": 6387505664 }, { "step": 29, "value": 6387505664 }, { "step": 30, "value": 6387505664 }, { "step": 31, "value": 6387505664 }, { "step": 32, "value": 6387505664 }, { "step": 33, "value": 6387505664 }, { "step": 34, "value": 6387505664 }, { "step": 35, "value": 6387505664 }, { "step": 36, "value": 6387505664 }, { "step": 37, "value": 6387505664 }, { "step": 38, "value": 6387505664 }, { "step": 39, "value": 6387505664 }, { "step": 40, "value": 6387505664 }, { "step": 41, "value": 6387505664 }, { "step": 42, "value": 6387505664 }, { "step": 43, "value": 6387505664 }, { "step": 44, "value": 6387505664 }, { "step": 45, "value": 6387505664 }, { "step": 46, "value": 6387505664 }, { "step": 47, "value": 6387505664 }, { "step": 48, "value": 6387505664 }, { "step": 49, "value": 6387505664 }, { "step": 50, "value": 6387505664 }, { "step": 51, "value": 6387505664 }, { "step": 52, "value": 6387505664 }, { "step": 53, "value": 6387505664 }, { "step": 54, "value": 6387505664 }, { "step": 55, "value": 6387505664 }, { "step": 56, "value": 6387505664 }, { "step": 57, "value": 6387505664 }, { "step": 58, "value": 6387505664 }, { "step": 59, "value": 6387505664 }, { "step": 60, "value": 6387505664 }, { "step": 61, "value": 6387505664 }, { "step": 62, "value": 6387505664 }, { "step": 63, "value": 6387505664 }, { "step": 64, "value": 6387505664 }, { "step": 65, "value": 6387505664 }, { "step": 66, "value": 6387505664 }, { "step": 67, "value": 6387505664 }, { "step": 68, "value": 6387505664 }, { "step": 69, "value": 6387505664 }, { "step": 70, "value": 6387505664 }, { "step": 71, "value": 6387505664 }, { "step": 72, "value": 6387505664 }, { "step": 73, "value": 6387505664 }, { "step": 74, "value": 6387505664 }, { "step": 75, "value": 6387505664 }, { "step": 76, "value": 6387505664 }, { "step": 77, "value": 6387505664 }, { "step": 78, "value": 6387505664 }, { "step": 79, "value": 6387505664 }, { "step": 80, "value": 6387505664 }, { "step": 81, "value": 6387505664 }, { "step": 82, "value": 6387505664 }, { "step": 83, "value": 6387505664 }, { "step": 84, "value": 6387505664 }, { "step": 85, "value": 6387505664 }, { "step": 86, "value": 6387505664 }, { "step": 87, "value": 6387505664 }, { "step": 88, "value": 6387505664 }, { "step": 89, "value": 6387505664 }, { "step": 90, "value": 6387505664 }, { "step": 91, "value": 6387505664 }, { "step": 92, "value": 6387505664 }, { "step": 93, "value": 6387505664 }, { "step": 94, "value": 6387505664 }, { "step": 95, "value": 6387505664 }, { "step": 96, "value": 6387505664 }, { "step": 97, "value": 6387505664 }, { "step": 98, "value": 6387505664 }, { "step": 99, "value": 6387505664 }, { "step": 100, "value": 6387505664 } ], "submission_sha256": "2ae6fde1d3fd03ed7b33742c22c5f6b4cd18aa05d63c7054641829eb7200b354", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/tpe/memory/submission.json", "comparison_value": 6387505664, "comparison_rank_key": [ 6387505664.0, 746848256.0, -0.01050141625733092 ], "rank": 7 } ] }, { "id": "params", "title": "Parameter count at validation BPB < 1.05", "metric": "num_params_total", "unit": "parameters", "direction": "minimize", "budget": 100, "comparison_step": null, "quality_gate": { "metric": "val_bpb", "operator": "<", "value": 1.05 }, "tiebreaks": [ { "metric": "total_tokens", "direction": "minimize" } ], "runs": [ { "run_id": "gear--params", "method_id": "gear", "task_id": "params", "metric": "num_params_total", "unit": "parameters", "direction": "minimize", "best_value": 13197712, "reference_value": 50332176, "relative_improvement": 0.7377877721797683, "rank_key": [ 13197712.0, 694943744.0, -0.00010600394986104433 ], "best_launch_seq": 95, "best_candidate_id": "exp0095", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": null }, "evaluation_gpu_hours": 20.008956388888894, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 50332176, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp0002", "value": 26345772, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp0006", "value": 33030544, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp0007", "value": 50332176, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp0008", "value": 41288208, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp0009", "value": 23593360, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp0010", "value": 37749264, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp0012", "value": 20447632, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp0013", "value": 23593360, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp0017", "value": 33030544, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp0018", "value": 20447632, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp0019", "value": 23593360, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp0020", "value": 20447632, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp0021", "value": 18678160, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp0022", "value": 18284944, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp0023", "value": 20447632, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp0024", "value": 20447632, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp0025", "value": 15139216, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp0026", "value": 20447632, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp0027", "value": 20447632, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp0028", "value": 15139216, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp0029", "value": 18088336, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp0030", "value": 18088336, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp0031", "value": 14401936, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp0032", "value": 14156176, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp0033", "value": 15729040, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp0034", "value": 14942608, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp0036", "value": 15729040, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp0037", "value": 14942608, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp0038", "value": 13910416, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp0039", "value": 14942608, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp0041", "value": 14942608, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp0043", "value": 14942608, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp0044", "value": 13910416, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp0045", "value": 14942608, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp0046", "value": 14942608, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp0047", "value": 13910416, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp0048", "value": 13664656, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp0050", "value": 13664656, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp0051", "value": 13664656, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp0052", "value": 13418896, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp0056", "value": 13910416, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp0057", "value": 13418896, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "exp0060", "value": 13357456, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "exp0061", "value": 13664656, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp0064", "value": 13296016, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "exp0065", "value": 13234576, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp0071", "value": 13418896, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "exp0085", "value": 13627792, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp0086", "value": 13222288, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "exp0089", "value": 13627792, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "exp0090", "value": 13418896, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "exp0095", "value": 13197712, "is_reference": false } ], "curve": [ { "step": 1, "value": 50332176 }, { "step": 2, "value": 26345772 }, { "step": 3, "value": 26345772 }, { "step": 4, "value": 26345772 }, { "step": 5, "value": 26345772 }, { "step": 6, "value": 26345772 }, { "step": 7, "value": 26345772 }, { "step": 8, "value": 26345772 }, { "step": 9, "value": 23593360 }, { "step": 10, "value": 23593360 }, { "step": 11, "value": 23593360 }, { "step": 12, "value": 20447632 }, { "step": 13, "value": 20447632 }, { "step": 14, "value": 20447632 }, { "step": 15, "value": 20447632 }, { "step": 16, "value": 20447632 }, { "step": 17, "value": 20447632 }, { "step": 18, "value": 20447632 }, { "step": 19, "value": 20447632 }, { "step": 20, "value": 20447632 }, { "step": 21, "value": 18678160 }, { "step": 22, "value": 18284944 }, { "step": 23, "value": 18284944 }, { "step": 24, "value": 18284944 }, { "step": 25, "value": 15139216 }, { "step": 26, "value": 15139216 }, { "step": 27, "value": 15139216 }, { "step": 28, "value": 15139216 }, { "step": 29, "value": 15139216 }, { "step": 30, "value": 15139216 }, { "step": 31, "value": 14401936 }, { "step": 32, "value": 14156176 }, { "step": 33, "value": 14156176 }, { "step": 34, "value": 14156176 }, { "step": 35, "value": 14156176 }, { "step": 36, "value": 14156176 }, { "step": 37, "value": 14156176 }, { "step": 38, "value": 13910416 }, { "step": 39, "value": 13910416 }, { "step": 40, "value": 13910416 }, { "step": 41, "value": 13910416 }, { "step": 42, "value": 13910416 }, { "step": 43, "value": 13910416 }, { "step": 44, "value": 13910416 }, { "step": 45, "value": 13910416 }, { "step": 46, "value": 13910416 }, { "step": 47, "value": 13910416 }, { "step": 48, "value": 13664656 }, { "step": 49, "value": 13664656 }, { "step": 50, "value": 13664656 }, { "step": 51, "value": 13664656 }, { "step": 52, "value": 13418896 }, { "step": 53, "value": 13418896 }, { "step": 54, "value": 13418896 }, { "step": 55, "value": 13418896 }, { "step": 56, "value": 13418896 }, { "step": 57, "value": 13418896 }, { "step": 58, "value": 13418896 }, { "step": 59, "value": 13418896 }, { "step": 60, "value": 13357456 }, { "step": 61, "value": 13357456 }, { "step": 62, "value": 13357456 }, { "step": 63, "value": 13357456 }, { "step": 64, "value": 13296016 }, { "step": 65, "value": 13234576 }, { "step": 66, "value": 13234576 }, { "step": 67, "value": 13234576 }, { "step": 68, "value": 13234576 }, { "step": 69, "value": 13234576 }, { "step": 70, "value": 13234576 }, { "step": 71, "value": 13234576 }, { "step": 72, "value": 13234576 }, { "step": 73, "value": 13234576 }, { "step": 74, "value": 13234576 }, { "step": 75, "value": 13234576 }, { "step": 76, "value": 13234576 }, { "step": 77, "value": 13234576 }, { "step": 78, "value": 13234576 }, { "step": 79, "value": 13234576 }, { "step": 80, "value": 13234576 }, { "step": 81, "value": 13234576 }, { "step": 82, "value": 13234576 }, { "step": 83, "value": 13234576 }, { "step": 84, "value": 13234576 }, { "step": 85, "value": 13234576 }, { "step": 86, "value": 13222288 }, { "step": 87, "value": 13222288 }, { "step": 88, "value": 13222288 }, { "step": 89, "value": 13222288 }, { "step": 90, "value": 13222288 }, { "step": 91, "value": 13222288 }, { "step": 92, "value": 13222288 }, { "step": 93, "value": 13222288 }, { "step": 94, "value": 13222288 }, { "step": 95, "value": 13197712 }, { "step": 96, "value": 13197712 }, { "step": 97, "value": 13197712 }, { "step": 98, "value": 13197712 }, { "step": 99, "value": 13197712 }, { "step": 100, "value": 13197712 } ], "submission_sha256": "e4a9668477156841fe5b62b4de0d4d06b4e176d40ee4f4b618810912432463b4", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/gear/params/submission.json", "comparison_value": 13197712, "comparison_rank_key": [ 13197712.0, 694943744.0, -0.00010600394986104433 ], "rank": 1 }, { "run_id": "autoscientists--params", "method_id": "autoscientists", "task_id": "params", "metric": "num_params_total", "unit": "parameters", "direction": "minimize", "best_value": 13910060, "reference_value": 50332176, "relative_improvement": 0.7236348374844751, "rank_key": [ 13910060.0, 691011584.0, -0.00016979447973008988 ], "best_launch_seq": 99, "best_candidate_id": "exp-dd1e9d23c582da43cdc3abdc", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 2, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 2 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 18.188301944444436, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 50332176, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-821a64ad46b91be4f3b3ab2d", "value": 47186446, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-76bcdb36df756e94a358d55c", "value": 39846284, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-f5db658b8ae58630f2a49daa", "value": 30409230, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-8a7397745925f497e5b48185", "value": 36962574, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-7d63d3e71b9f3ba06ec0fdce", "value": 23069198, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp-4210dd6ea4539e32cf30d15c", "value": 28573966, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-5b104c67b3420e6fd47b6168", "value": 23069198, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-bb5cecf4b941fca9dc17692e", "value": 21233934, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-d126e65bbbaa672075b4b5fd", "value": 19005966, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-639743d661a69555fea4c9c8", "value": 16908684, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-553c3e6a6baa8a50d4081536", "value": 17170702, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp-e1014d30316037bc9e38e1c9", "value": 16515468, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp-1c94824c20519f681270c0d0", "value": 16122252, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp-e638ec566712f6e1825137c3", "value": 15925644, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp-965694496f80648a46f27a32", "value": 15860108, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp-5caf86167a558ca69a489347", "value": 16122252, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp-27e7fe1b2639b1941e4148d2", "value": 15597964, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-f2a491a7140a83b82f780d5d", "value": 15597964, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-f2cd5df733c9f9a2ef3f689c", "value": 15466892, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-170c0dffba61e2787e7f1f13", "value": 15597964, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-72186131e174d7dc339fe5b4", "value": 15204748, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp-d9e962bfd90bb4e33c765421", "value": 15073676, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp-892db48a1462e2dfffcaaab2", "value": 15073676, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp-d3bb579f6ff82f04289f8320", "value": 15073676, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp-8e6687766fb24f0495b76bd8", "value": 15073388, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp-14569b6ade00c5802526cfec", "value": 14975084, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp-15f64ade431cef8e60f69ae9", "value": 14975084, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp-74137642b51c0340f58a68d0", "value": 14811244, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp-c2b4f6bac667bb623d5adde1", "value": 14975084, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "exp-a640344fa155ad45d29ba943", "value": 14925932, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "exp-ebd1b7c0e519b79915f3f875", "value": 14975084, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "exp-b80eadf51fe7661eaa7f89fe", "value": 14975084, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp-7607edcc482807884bb51a38", "value": 14614636, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp-4b255027b38910474c297559", "value": 14549100, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp-81c1c135d5d7bda0881dc82b", "value": 14499948, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "exp-1a7d7471202004c5e400dfd7", "value": 14352492, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "exp-e569dc262a2408784f2ab29c", "value": 14237804, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "exp-6ab6464dd474b1895476d508", "value": 14303340, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "exp-363f397e9950ebe630d68a3d", "value": 14188652, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp-47d46661bd5b032db8c6065a", "value": 14237740, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "exp-92d03c6db01fd581fac35cb4", "value": 14237804, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "exp-cf01df3ae98a8fa7485f4a3f", "value": 14188588, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "exp-caae46c990d5c7c5a55eee20", "value": 14188652, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "exp-0fe51a142fc4f19cfe1beb0a", "value": 14155820, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "exp-7adeec4843e2634427c5163d", "value": 14139436, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "exp-58eca4e765889b6e33dfaaad", "value": 14123052, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "exp-faf560c2ef7b5a2f12166c4b", "value": 14188652, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "exp-524fd8c46527af6f0d564102", "value": 14090284, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "exp-1c929da433d96907178ca1c4", "value": 14106668, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "exp-c00a99041ff15e5ac9307764", "value": 14155820, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp-f6f20d760e0b21380a0a153d", "value": 14090284, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "exp-32808f610d1f4fb3357719a4", "value": 14073900, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "exp-a21b8d2c53756868e54d1cca", "value": 14090284, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "exp-979126c2b5dc8ba340818fd2", "value": 14057516, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "exp-fd96dc3785980b386d148f3d", "value": 14041132, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "exp-413fb3ba3360db70bb11d868", "value": 14057516, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "exp-c32d39836c648999e60938d1", "value": 14024748, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "exp-e245cb6d0c7d33c57e74c411", "value": 14057516, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "exp-518a9cda556c7f8c86f930b7", "value": 14041132, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "exp-58b58de754686f756b75e33d", "value": 13991980, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "exp-271d5ee73840279a67217d03", "value": 13959212, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "exp-4aa833a03bd5c69eec20a7ab", "value": 13942828, "is_reference": false }, { "step": 98, "launch_seq": 98, "candidate_id": "exp-2d6494f5babbc3c9dbf1c3bf", "value": 13926444, "is_reference": false }, { "step": 99, "launch_seq": 99, "candidate_id": "exp-dd1e9d23c582da43cdc3abdc", "value": 13910060, "is_reference": false } ], "curve": [ { "step": 1, "value": 50332176 }, { "step": 2, "value": 47186446 }, { "step": 3, "value": 47186446 }, { "step": 4, "value": 39846284 }, { "step": 5, "value": 30409230 }, { "step": 6, "value": 30409230 }, { "step": 7, "value": 23069198 }, { "step": 8, "value": 23069198 }, { "step": 9, "value": 23069198 }, { "step": 10, "value": 23069198 }, { "step": 11, "value": 23069198 }, { "step": 12, "value": 21233934 }, { "step": 13, "value": 19005966 }, { "step": 14, "value": 19005966 }, { "step": 15, "value": 16908684 }, { "step": 16, "value": 16908684 }, { "step": 17, "value": 16908684 }, { "step": 18, "value": 16908684 }, { "step": 19, "value": 16908684 }, { "step": 20, "value": 16908684 }, { "step": 21, "value": 16908684 }, { "step": 22, "value": 16515468 }, { "step": 23, "value": 16515468 }, { "step": 24, "value": 16515468 }, { "step": 25, "value": 16122252 }, { "step": 26, "value": 16122252 }, { "step": 27, "value": 16122252 }, { "step": 28, "value": 16122252 }, { "step": 29, "value": 15925644 }, { "step": 30, "value": 15860108 }, { "step": 31, "value": 15860108 }, { "step": 32, "value": 15597964 }, { "step": 33, "value": 15597964 }, { "step": 34, "value": 15466892 }, { "step": 35, "value": 15466892 }, { "step": 36, "value": 15204748 }, { "step": 37, "value": 15073676 }, { "step": 38, "value": 15073676 }, { "step": 39, "value": 15073676 }, { "step": 40, "value": 15073676 }, { "step": 41, "value": 15073676 }, { "step": 42, "value": 15073676 }, { "step": 43, "value": 15073676 }, { "step": 44, "value": 15073676 }, { "step": 45, "value": 15073676 }, { "step": 46, "value": 15073676 }, { "step": 47, "value": 15073676 }, { "step": 48, "value": 15073676 }, { "step": 49, "value": 15073676 }, { "step": 50, "value": 15073388 }, { "step": 51, "value": 15073388 }, { "step": 52, "value": 14975084 }, { "step": 53, "value": 14975084 }, { "step": 54, "value": 14975084 }, { "step": 55, "value": 14975084 }, { "step": 56, "value": 14975084 }, { "step": 57, "value": 14811244 }, { "step": 58, "value": 14811244 }, { "step": 59, "value": 14811244 }, { "step": 60, "value": 14811244 }, { "step": 61, "value": 14811244 }, { "step": 62, "value": 14811244 }, { "step": 63, "value": 14614636 }, { "step": 64, "value": 14549100 }, { "step": 65, "value": 14549100 }, { "step": 66, "value": 14499948 }, { "step": 67, "value": 14352492 }, { "step": 68, "value": 14237804 }, { "step": 69, "value": 14237804 }, { "step": 70, "value": 14188652 }, { "step": 71, "value": 14188652 }, { "step": 72, "value": 14188652 }, { "step": 73, "value": 14188652 }, { "step": 74, "value": 14188652 }, { "step": 75, "value": 14188588 }, { "step": 76, "value": 14188588 }, { "step": 77, "value": 14188588 }, { "step": 78, "value": 14155820 }, { "step": 79, "value": 14155820 }, { "step": 80, "value": 14139436 }, { "step": 81, "value": 14123052 }, { "step": 82, "value": 14123052 }, { "step": 83, "value": 14090284 }, { "step": 84, "value": 14090284 }, { "step": 85, "value": 14090284 }, { "step": 86, "value": 14090284 }, { "step": 87, "value": 14073900 }, { "step": 88, "value": 14073900 }, { "step": 89, "value": 14057516 }, { "step": 90, "value": 14041132 }, { "step": 91, "value": 14041132 }, { "step": 92, "value": 14024748 }, { "step": 93, "value": 14024748 }, { "step": 94, "value": 14024748 }, { "step": 95, "value": 13991980 }, { "step": 96, "value": 13959212 }, { "step": 97, "value": 13942828 }, { "step": 98, "value": 13926444 }, { "step": 99, "value": 13910060 }, { "step": 100, "value": 13910060 } ], "submission_sha256": "e9dbbe82a3a8ed3563b322dff21f5bb6b79ec35312669dd8902632382ffa7412", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/autoscientists/params/submission.json", "comparison_value": 13910060, "comparison_rank_key": [ 13910060.0, 691011584.0, -0.00016979447973008988 ], "rank": 2 }, { "run_id": "beam-search--params", "method_id": "beam-search", "task_id": "params", "metric": "num_params_total", "unit": "parameters", "direction": "minimize", "best_value": 14221456, "reference_value": 50332176, "relative_improvement": 0.7174480197319504, "rank_key": [ 14221456.0, 823132160.0, -0.0003993871397383053 ], "best_launch_seq": 88, "best_candidate_id": "g13-c1-ve1qh2-mlp-edge-hidden-640", "best_is_reference": false, "budget": { "spent": 95, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 4, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 4 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": null }, "evaluation_gpu_hours": 17.70368277777777, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 50332176, "is_reference": true }, { "step": 3, "launch_seq": 3, "candidate_id": "g01-c2-gqa-half-kv-heads", "value": 39846160, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "g02-c3-ve-pair-share", "value": 41943568, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "g02-c4-mlp-expansion-3x", "value": 46137872, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "g03-c3-ve-pair-share-gqa", "value": 35651856, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "g03-c5-mlp-expansion-2x", "value": 41943568, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "g03-c2-gqa-depth-7", "value": 36962574, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "g03-c4-ve-pair-share-update-granularity", "value": 41943568, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "g03-c6-mlp3x-ve-pair-share", "value": 37749264, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "g04-c5-gqa-depth7-ve-head-width", "value": 32768270, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "g04-c3-mlp3x-ve-pair-update-granularity", "value": 37749264, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "g04-c2-ve-pair-gqa-update-granularity", "value": 35651856, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "g04-c4-ve-pair-mlp-expansion-2x", "value": 33554960, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "g04-c8-gqa-ve-stride-3", "value": 37748944, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "g05-c1-vehw-update-granularity", "value": 32768270, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "g05-c5-gqa-mlp-expansion-2x", "value": 27263248, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "g05-c7-gqa-depth7-ve-stride-3", "value": 34865358, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "g05-c2-vehw-depth-6", "value": 28836044, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "g05-c6-gqa-tied-unembedding", "value": 31457552, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "g06-c1-gqa-mlp2x-depth-7", "value": 25428238, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "g06-c3-vehw-depth6-finer-updates", "value": 28836044, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "g06-c7-vehw-ve-single-table", "value": 29622542, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "g06-c2-gqa-mlp2x-mqa", "value": 24117392, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "g06-c4-vehw-depth6-mlp-expansion-3x", "value": 25690316, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "g06-c6-tied-ve-head-width", "value": 29360400, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "g06-c8-vehw-depth-5", "value": 25952458, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "g07-c1-mqa-mlp2x-width-384", "value": 16253072, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "g07-c3-gqa-mlp2x-d7-tied-unembedding", "value": 21233934, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "g07-c5-vehw-d6-mlp3x-ve-single-table", "value": 23593164, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "g07-c7-vehw-d5-tied-unembedding", "value": 21758154, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "g07-c2-mqa-mlp2x-ve-single-table", "value": 23068816, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "g07-c4-gqa-mlp2x-depth-5", "value": 21758154, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "g07-c6-vehw-d6-mlp3x-depth-5", "value": 23331018, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "g07-c8-vehw-d5-mlp-edge-taper", "value": 23855306, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "g08-c2-mqa384-ve-single-table", "value": 15204496, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "g08-c4-tied-d7-attn-pair-share", "value": 18874638, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "g08-c8-gqa-d5-ve-head-width", "value": 19661002, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "g08-c3-tied-d7-ve-head-width", "value": 19136782, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "g08-c5-vehw-d5-tied-depth-4", "value": 17825928, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "g08-c7-gqa-d5-mqa", "value": 19005546, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "g09-c4-mqa384-ve-row-half", "value": 15204496, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "g09-c6-vehw-d4-ve-single-table", "value": 16777352, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "g09-c8-tied-d7-depth-6", "value": 17039564, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "g09-c7-tied-d7-share-ve-head-width", "value": 16777486, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "g10-c5-mqa384-verow-ve-single-table", "value": 14680208, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "g10-c4-mqa384-ve1-query-heads-2", "value": 14418064, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "g11-c10-verow1-gate-per-copy", "value": 14680208, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "g12-c7-verow-gate-per-copy", "value": 15204496, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "g12-c4-ve1qh2-gate-split-value", "value": 14418064, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "g12-c8-verow-mlp-hidden-by-window", "value": 14909584, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "g12-c2-ve1-mlp-edge-hidden-512", "value": 14811280, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "g13-c1-ve1qh2-mlp-edge-hidden-640", "value": 14221456, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "g13-c3-verow1-mlp-hidden-by-ve-site", "value": 14483600, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "g13-c5-ve1-mlp-edge-hidden-384", "value": 14614672, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "g13-c7-verow-ve-single-table-gate-per-copy", "value": 14385296, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "g13-c8-verow-mlp-hidden-short-688", "value": 14835856, "is_reference": false } ], "curve": [ { "step": 1, "value": 50332176 }, { "step": 2, "value": 50332176 }, { "step": 3, "value": 39846160 }, { "step": 4, "value": 39846160 }, { "step": 5, "value": 39846160 }, { "step": 6, "value": 39846160 }, { "step": 7, "value": 39846160 }, { "step": 8, "value": 39846160 }, { "step": 9, "value": 35651856 }, { "step": 10, "value": 35651856 }, { "step": 11, "value": 35651856 }, { "step": 12, "value": 35651856 }, { "step": 13, "value": 35651856 }, { "step": 14, "value": 35651856 }, { "step": 15, "value": 35651856 }, { "step": 16, "value": 35651856 }, { "step": 17, "value": 32768270 }, { "step": 18, "value": 32768270 }, { "step": 19, "value": 32768270 }, { "step": 20, "value": 32768270 }, { "step": 21, "value": 32768270 }, { "step": 22, "value": 32768270 }, { "step": 23, "value": 32768270 }, { "step": 24, "value": 32768270 }, { "step": 25, "value": 32768270 }, { "step": 26, "value": 27263248 }, { "step": 27, "value": 27263248 }, { "step": 28, "value": 27263248 }, { "step": 29, "value": 27263248 }, { "step": 30, "value": 27263248 }, { "step": 31, "value": 27263248 }, { "step": 32, "value": 25428238 }, { "step": 33, "value": 25428238 }, { "step": 34, "value": 25428238 }, { "step": 35, "value": 25428238 }, { "step": 36, "value": 24117392 }, { "step": 37, "value": 24117392 }, { "step": 38, "value": 24117392 }, { "step": 39, "value": 24117392 }, { "step": 40, "value": 16253072 }, { "step": 41, "value": 16253072 }, { "step": 42, "value": 16253072 }, { "step": 43, "value": 16253072 }, { "step": 44, "value": 16253072 }, { "step": 45, "value": 16253072 }, { "step": 46, "value": 16253072 }, { "step": 47, "value": 16253072 }, { "step": 48, "value": 15204496 }, { "step": 49, "value": 15204496 }, { "step": 50, "value": 15204496 }, { "step": 51, "value": 15204496 }, { "step": 52, "value": 15204496 }, { "step": 53, "value": 15204496 }, { "step": 54, "value": 15204496 }, { "step": 55, "value": 15204496 }, { "step": 56, "value": 15204496 }, { "step": 57, "value": 15204496 }, { "step": 58, "value": 15204496 }, { "step": 59, "value": 15204496 }, { "step": 60, "value": 15204496 }, { "step": 61, "value": 15204496 }, { "step": 62, "value": 15204496 }, { "step": 63, "value": 15204496 }, { "step": 64, "value": 14680208 }, { "step": 65, "value": 14680208 }, { "step": 66, "value": 14680208 }, { "step": 67, "value": 14680208 }, { "step": 68, "value": 14680208 }, { "step": 69, "value": 14418064 }, { "step": 70, "value": 14418064 }, { "step": 71, "value": 14418064 }, { "step": 72, "value": 14418064 }, { "step": 73, "value": 14418064 }, { "step": 74, "value": 14418064 }, { "step": 75, "value": 14418064 }, { "step": 76, "value": 14418064 }, { "step": 77, "value": 14418064 }, { "step": 78, "value": 14418064 }, { "step": 79, "value": 14418064 }, { "step": 80, "value": 14418064 }, { "step": 81, "value": 14418064 }, { "step": 82, "value": 14418064 }, { "step": 83, "value": 14418064 }, { "step": 84, "value": 14418064 }, { "step": 85, "value": 14418064 }, { "step": 86, "value": 14418064 }, { "step": 87, "value": 14418064 }, { "step": 88, "value": 14221456 }, { "step": 89, "value": 14221456 }, { "step": 90, "value": 14221456 }, { "step": 91, "value": 14221456 }, { "step": 92, "value": 14221456 }, { "step": 93, "value": 14221456 }, { "step": 94, "value": 14221456 }, { "step": 95, "value": 14221456 } ], "submission_sha256": "8a144a5a7200b3c2c28d09cb680fb87b447ce931b1256b2380808eca3a9afc03", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/beam-search/params/submission.json", "comparison_value": 14221456, "comparison_rank_key": [ 14221456.0, 823132160.0, -0.0003993871397383053 ], "rank": 3 }, { "run_id": "arbor--params", "method_id": "arbor", "task_id": "params", "metric": "num_params_total", "unit": "parameters", "direction": "minimize", "best_value": 14549136, "reference_value": 50332176, "relative_improvement": 0.7109376713615561, "rank_key": [ 14549136.0, 725614592.0, -0.00029492508856310273 ], "best_launch_seq": 72, "best_candidate_id": "exp-1f6e73ac951333877eedf200", "best_is_reference": false, "budget": { "spent": 71, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 13.230245833333337, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 50332176, "is_reference": true }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-58a5b1e920eb2caf2e300d28", "value": 37749264, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp-c9cfd1e01e3c6cb1782f19bf", "value": 41943568, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp-bda7589dd110622e333ac402", "value": 37749264, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-ffc74e25bf6d8ba32f7d403a", "value": 37749264, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-9fd03ecd1591f8470a52c390", "value": 37749264, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-766a260d86359dcfa6799778", "value": 33554960, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-bbf52a7cb63e73cd3789109d", "value": 37749264, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-8d71605d242a08ee2cc71d8a", "value": 20447632, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-1cf55aa3293773f20f6eb0ee", "value": 25166352, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp-d1466491d5552aa9d6d0d8e9", "value": 20979984, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-a8b738cc4fb90631c22c1288", "value": 22020240, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-57d059f28cdd2c86d2655a19", "value": 17834256, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-1d8ca1eed8bacebfe168965f", "value": 17825936, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-2576898b43d2fdbd4108c28e", "value": 17531700, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp-2590d790946e1cd3122a9a60", "value": 16302898, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp-c2909a16e078fb081daf3999", "value": 16302898, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp-db60f5462fb4fb866805c19f", "value": 16302898, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-6afda924262645c6c6e85686", "value": 16302898, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-aef1979ae98142ae7483f37b", "value": 16302898, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-9937b57f60f713d2b4ae09eb", "value": 16302898, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp-3da65af36d2dc248546df673", "value": 16302898, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp-ef6f0efcb070fba96f549bb9", "value": 16302898, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp-18c3c601be2911284c682b25", "value": 16302898, "is_reference": false }, { "step": 47, "launch_seq": 48, "candidate_id": "exp-a536e81e1a32fe166d516396", "value": 16302898, "is_reference": false }, { "step": 50, "launch_seq": 51, "candidate_id": "exp-1c8579d44f32074155a5c195", "value": 16311090, "is_reference": false }, { "step": 51, "launch_seq": 52, "candidate_id": "exp-5cd098829d735228d1c91bcf", "value": 16311090, "is_reference": false }, { "step": 53, "launch_seq": 54, "candidate_id": "exp-4516462bfa89e4771da2eff1", "value": 19271056, "is_reference": false }, { "step": 54, "launch_seq": 55, "candidate_id": "exp-b64bef50a35e0ee0b82a113b", "value": 15729040, "is_reference": false }, { "step": 57, "launch_seq": 58, "candidate_id": "exp-5451b6225e09e00ee89eaba0", "value": 15040656, "is_reference": false }, { "step": 58, "launch_seq": 59, "candidate_id": "exp-2a6090911eadc32d448a6e5e", "value": 14745744, "is_reference": false }, { "step": 66, "launch_seq": 67, "candidate_id": "exp-81bcd9c40aa7bb46a9b006f1", "value": 14745744, "is_reference": false }, { "step": 71, "launch_seq": 72, "candidate_id": "exp-1f6e73ac951333877eedf200", "value": 14549136, "is_reference": false } ], "curve": [ { "step": 1, "value": 50332176 }, { "step": 2, "value": 50332176 }, { "step": 3, "value": 50332176 }, { "step": 4, "value": 50332176 }, { "step": 5, "value": 50332176 }, { "step": 6, "value": 50332176 }, { "step": 7, "value": 37749264 }, { "step": 8, "value": 37749264 }, { "step": 9, "value": 37749264 }, { "step": 10, "value": 37749264 }, { "step": 11, "value": 37749264 }, { "step": 12, "value": 33554960 }, { "step": 13, "value": 33554960 }, { "step": 14, "value": 20447632 }, { "step": 15, "value": 20447632 }, { "step": 16, "value": 20447632 }, { "step": 17, "value": 20447632 }, { "step": 18, "value": 20447632 }, { "step": 19, "value": 17834256 }, { "step": 20, "value": 17825936 }, { "step": 21, "value": 17531700 }, { "step": 22, "value": 16302898 }, { "step": 23, "value": 16302898 }, { "step": 24, "value": 16302898 }, { "step": 25, "value": 16302898 }, { "step": 26, "value": 16302898 }, { "step": 27, "value": 16302898 }, { "step": 28, "value": 16302898 }, { "step": 29, "value": 16302898 }, { "step": 30, "value": 16302898 }, { "step": 31, "value": 16302898 }, { "step": 32, "value": 16302898 }, { "step": 33, "value": 16302898 }, { "step": 34, "value": 16302898 }, { "step": 35, "value": 16302898 }, { "step": 36, "value": 16302898 }, { "step": 37, "value": 16302898 }, { "step": 38, "value": 16302898 }, { "step": 39, "value": 16302898 }, { "step": 40, "value": 16302898 }, { "step": 41, "value": 16302898 }, { "step": 42, "value": 16302898 }, { "step": 43, "value": 16302898 }, { "step": 44, "value": 16302898 }, { "step": 45, "value": 16302898 }, { "step": 46, "value": 16302898 }, { "step": 47, "value": 16302898 }, { "step": 48, "value": 16302898 }, { "step": 49, "value": 16302898 }, { "step": 50, "value": 16302898 }, { "step": 51, "value": 16302898 }, { "step": 52, "value": 16302898 }, { "step": 53, "value": 16302898 }, { "step": 54, "value": 15729040 }, { "step": 55, "value": 15729040 }, { "step": 56, "value": 15729040 }, { "step": 57, "value": 15040656 }, { "step": 58, "value": 14745744 }, { "step": 59, "value": 14745744 }, { "step": 60, "value": 14745744 }, { "step": 61, "value": 14745744 }, { "step": 62, "value": 14745744 }, { "step": 63, "value": 14745744 }, { "step": 64, "value": 14745744 }, { "step": 65, "value": 14745744 }, { "step": 66, "value": 14745744 }, { "step": 67, "value": 14745744 }, { "step": 68, "value": 14745744 }, { "step": 69, "value": 14745744 }, { "step": 70, "value": 14745744 }, { "step": 71, "value": 14549136 } ], "submission_sha256": "e7e2680ea5df4c95adc0b07895553272d875566908be45fc9b825c9e4f3e1274", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/arbor/params/submission.json", "comparison_value": 14549136, "comparison_rank_key": [ 14549136.0, 725614592.0, -0.00029492508856310273 ], "rank": 4 }, { "run_id": "sequential-search--params", "method_id": "sequential-search", "task_id": "params", "metric": "num_params_total", "unit": "parameters", "direction": "minimize", "best_value": 15151312, "reference_value": 50332176, "relative_improvement": 0.6989736346785405, "rank_key": [ 15151312.0, 601358336.0, -0.00011279194380731106 ], "best_launch_seq": 85, "best_candidate_id": "cand-0083", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "evaluation_capacity_gpus": 1, "gpu_instance_type": null, "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "gpus_per_evaluation": 1, "workers": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 20.87040999999999, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 50332176, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "cand-0001", "value": 50332176, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "cand-0002", "value": 37749264, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "cand-0003", "value": 23593360, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "cand-0004", "value": 23593360, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "cand-0005", "value": 16515280, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "cand-0006", "value": 16515280, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "cand-0007", "value": 16515280, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "cand-0008", "value": 16515280, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "cand-0010", "value": 15925456, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "cand-0011", "value": 15925456, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "cand-0012", "value": 15925456, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "cand-0021", "value": 15728848, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "cand-0023", "value": 15679696, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "cand-0027", "value": 15679696, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "cand-0028", "value": 15532240, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "cand-0029", "value": 15384784, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "cand-0031", "value": 15384784, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "cand-0033", "value": 15384784, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "cand-0037", "value": 15323344, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "cand-0038", "value": 15323344, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "cand-0039", "value": 15323344, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "cand-0040", "value": 15323344, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "cand-0041", "value": 15261904, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "cand-0044", "value": 15261904, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "cand-0045", "value": 15237328, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "cand-0047", "value": 15237328, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "cand-0052", "value": 15261904, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "cand-0053", "value": 15274192, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "cand-0054", "value": 15286480, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "cand-0055", "value": 15298768, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "cand-0056", "value": 15311056, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "cand-0057", "value": 15323344, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "cand-0058", "value": 15323344, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "cand-0059", "value": 15323344, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "cand-0060", "value": 15323344, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "cand-0061", "value": 15323344, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "cand-0064", "value": 15200464, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "cand-0072", "value": 15163600, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "cand-0083", "value": 15151312, "is_reference": false } ], "curve": [ { "step": 1, "value": 50332176 }, { "step": 2, "value": 50332176 }, { "step": 3, "value": 37749264 }, { "step": 4, "value": 23593360 }, { "step": 5, "value": 23593360 }, { "step": 6, "value": 16515280 }, { "step": 7, "value": 16515280 }, { "step": 8, "value": 16515280 }, { "step": 9, "value": 16515280 }, { "step": 10, "value": 16515280 }, { "step": 11, "value": 15925456 }, { "step": 12, "value": 15925456 }, { "step": 13, "value": 15925456 }, { "step": 14, "value": 15925456 }, { "step": 15, "value": 15925456 }, { "step": 16, "value": 15925456 }, { "step": 17, "value": 15925456 }, { "step": 18, "value": 15925456 }, { "step": 19, "value": 15925456 }, { "step": 20, "value": 15925456 }, { "step": 21, "value": 15925456 }, { "step": 22, "value": 15728848 }, { "step": 23, "value": 15728848 }, { "step": 24, "value": 15679696 }, { "step": 25, "value": 15679696 }, { "step": 26, "value": 15679696 }, { "step": 27, "value": 15679696 }, { "step": 28, "value": 15679696 }, { "step": 29, "value": 15532240 }, { "step": 30, "value": 15384784 }, { "step": 31, "value": 15384784 }, { "step": 32, "value": 15384784 }, { "step": 33, "value": 15384784 }, { "step": 34, "value": 15384784 }, { "step": 35, "value": 15384784 }, { "step": 36, "value": 15384784 }, { "step": 37, "value": 15384784 }, { "step": 38, "value": 15384784 }, { "step": 39, "value": 15323344 }, { "step": 40, "value": 15323344 }, { "step": 41, "value": 15323344 }, { "step": 42, "value": 15323344 }, { "step": 43, "value": 15261904 }, { "step": 44, "value": 15261904 }, { "step": 45, "value": 15261904 }, { "step": 46, "value": 15261904 }, { "step": 47, "value": 15237328 }, { "step": 48, "value": 15237328 }, { "step": 49, "value": 15237328 }, { "step": 50, "value": 15237328 }, { "step": 51, "value": 15237328 }, { "step": 52, "value": 15237328 }, { "step": 53, "value": 15237328 }, { "step": 54, "value": 15237328 }, { "step": 55, "value": 15237328 }, { "step": 56, "value": 15237328 }, { "step": 57, "value": 15237328 }, { "step": 58, "value": 15237328 }, { "step": 59, "value": 15237328 }, { "step": 60, "value": 15237328 }, { "step": 61, "value": 15237328 }, { "step": 62, "value": 15237328 }, { "step": 63, "value": 15237328 }, { "step": 64, "value": 15237328 }, { "step": 65, "value": 15237328 }, { "step": 66, "value": 15200464 }, { "step": 67, "value": 15200464 }, { "step": 68, "value": 15200464 }, { "step": 69, "value": 15200464 }, { "step": 70, "value": 15200464 }, { "step": 71, "value": 15200464 }, { "step": 72, "value": 15200464 }, { "step": 73, "value": 15200464 }, { "step": 74, "value": 15163600 }, { "step": 75, "value": 15163600 }, { "step": 76, "value": 15163600 }, { "step": 77, "value": 15163600 }, { "step": 78, "value": 15163600 }, { "step": 79, "value": 15163600 }, { "step": 80, "value": 15163600 }, { "step": 81, "value": 15163600 }, { "step": 82, "value": 15163600 }, { "step": 83, "value": 15163600 }, { "step": 84, "value": 15163600 }, { "step": 85, "value": 15151312 }, { "step": 86, "value": 15151312 }, { "step": 87, "value": 15151312 }, { "step": 88, "value": 15151312 }, { "step": 89, "value": 15151312 }, { "step": 90, "value": 15151312 }, { "step": 91, "value": 15151312 }, { "step": 92, "value": 15151312 }, { "step": 93, "value": 15151312 }, { "step": 94, "value": 15151312 }, { "step": 95, "value": 15151312 }, { "step": 96, "value": 15151312 }, { "step": 97, "value": 15151312 }, { "step": 98, "value": 15151312 }, { "step": 99, "value": 15151312 }, { "step": 100, "value": 15151312 } ], "submission_sha256": "406cb0308957b09616fce3c0ffb4dc378573f69dbb896bb7b54db9357293fdcd", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/sequential-search/params/submission.json", "comparison_value": 15151312, "comparison_rank_key": [ 15151312.0, 601358336.0, -0.00011279194380731106 ], "rank": 5 }, { "run_id": "heuresis-map-elites--params", "method_id": "heuresis-map-elites", "task_id": "params", "metric": "num_params_total", "unit": "parameters", "direction": "minimize", "best_value": 17826840, "reference_value": 50332176, "relative_improvement": 0.6458162269797356, "rank_key": [ 17826840.0, 486277120.0, -0.0007281682164659298 ], "best_launch_seq": 53, "best_candidate_id": "exp-323bf6876216da7bd6700ecb", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 17.54409555555555, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 50332176, "is_reference": true }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-2327219a2547177206ccd8f6", "value": 41943568, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-bc93c1292ba6bddb090df7d0", "value": 37749264, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-22df8d243441fc08c4e4d609", "value": 37487120, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-94e8a322adf0cd39b098e8ae", "value": 37749264, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-071cb1d052ec443b466114ab", "value": 37749264, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-f0215b8eea4053f34929f16f", "value": 27300368, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-80bae2a2277ccffabf2f683e", "value": 26251792, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-b642f5391dd23d23e6849715", "value": 24154640, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp-3f3b17e2e430e17ca173fe79", "value": 33557008, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-af6ae03f025256d69c703f3b", "value": 29363216, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp-955f86110d323c1284557215", "value": 29363216, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp-8b47918e0da7d6029b36407a", "value": 29363216, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp-43b2a0605a31052bd79ef3f3", "value": 29363216, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp-9792d10e1246c44f569bd873", "value": 29363216, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp-90d24f0dd251cc6a5a1193a9", "value": 29363216, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-512077de48ca5acd9ab867af", "value": 37773840, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-0e7877ca5d67a51dd46978d5", "value": 35676688, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp-4b907df2e2c30afdf0927311", "value": 35426832, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp-0aeb074763903133e70f4780", "value": 33554704, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp-c8bfb12ff0dbb982bd936867", "value": 23069328, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp-9359dfaad5c915159adfb6b9", "value": 22577808, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "exp-e60a99f498d58be77389d6e0", "value": 20611728, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp-0387f668b8d9e20770460ca2", "value": 29361176, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp-215cc95955794359dae0fc18", "value": 23069720, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp-313fe89812a622684fdc3825", "value": 18875416, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "exp-323bf6876216da7bd6700ecb", "value": 17826840, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp-6ba6d44bc6c3e2d7962aa39a", "value": 33563672, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "exp-aac7ddc49cb56ce3c6189407", "value": 29369368, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "exp-1b50a3c67d529ec1b107a4c2", "value": 27403288, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "exp-aad1413d2c2bffdb6ead4e54", "value": 26846232, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp-ac42728c53c6b4ca2190cd51", "value": 41976336, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "exp-e0ea76f97833907eba2c2308", "value": 37798416, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "exp-651d5c3de801041e84887bf2", "value": 37274128, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp-ef05592de26be0d2e0cb0af8", "value": 36749840, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "exp-817a647e82d6677505add350", "value": 34603280, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "exp-1ffa7446c16d6b4dba21ff18", "value": 31981712, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "exp-32254e785248275e01eafd60", "value": 30670992, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "exp-5e380d607e0506578edf8d3f", "value": 50070032, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp-4a45914f650c8d7490140966", "value": 29369904, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "exp-89150a2b24f11536b6cdccd8", "value": 25175600, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "exp-4bedca6cdf48ce0897e93e85", "value": 23078448, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "exp-569f8a1228bef8a797ba6b6a", "value": 20981296, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "exp-0eb3e17c42ca5fad965dbba9", "value": 18884144, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "exp-d42650eaa2d0069fd7d25765", "value": 37749264, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "exp-d5ed96de3f754bfe2e9a8a49", "value": 31457424, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "exp-3f7d047b1b3a072700722fbe", "value": 30408848, "is_reference": false }, { "step": 98, "launch_seq": 98, "candidate_id": "exp-4f5e446f083221eb06765afa", "value": 26223248, "is_reference": false } ], "curve": [ { "step": 1, "value": 50332176 }, { "step": 2, "value": 50332176 }, { "step": 3, "value": 50332176 }, { "step": 4, "value": 50332176 }, { "step": 5, "value": 41943568 }, { "step": 6, "value": 41943568 }, { "step": 7, "value": 37749264 }, { "step": 8, "value": 37749264 }, { "step": 9, "value": 37749264 }, { "step": 10, "value": 37749264 }, { "step": 11, "value": 37749264 }, { "step": 12, "value": 37487120 }, { "step": 13, "value": 37487120 }, { "step": 14, "value": 37487120 }, { "step": 15, "value": 37487120 }, { "step": 16, "value": 37487120 }, { "step": 17, "value": 37487120 }, { "step": 18, "value": 37487120 }, { "step": 19, "value": 27300368 }, { "step": 20, "value": 26251792 }, { "step": 21, "value": 24154640 }, { "step": 22, "value": 24154640 }, { "step": 23, "value": 24154640 }, { "step": 24, "value": 24154640 }, { "step": 25, "value": 24154640 }, { "step": 26, "value": 24154640 }, { "step": 27, "value": 24154640 }, { "step": 28, "value": 24154640 }, { "step": 29, "value": 24154640 }, { "step": 30, "value": 24154640 }, { "step": 31, "value": 24154640 }, { "step": 32, "value": 24154640 }, { "step": 33, "value": 24154640 }, { "step": 34, "value": 24154640 }, { "step": 35, "value": 24154640 }, { "step": 36, "value": 24154640 }, { "step": 37, "value": 24154640 }, { "step": 38, "value": 24154640 }, { "step": 39, "value": 24154640 }, { "step": 40, "value": 24154640 }, { "step": 41, "value": 24154640 }, { "step": 42, "value": 24154640 }, { "step": 43, "value": 24154640 }, { "step": 44, "value": 24154640 }, { "step": 45, "value": 24154640 }, { "step": 46, "value": 23069328 }, { "step": 47, "value": 23069328 }, { "step": 48, "value": 22577808 }, { "step": 49, "value": 20611728 }, { "step": 50, "value": 20611728 }, { "step": 51, "value": 20611728 }, { "step": 52, "value": 18875416 }, { "step": 53, "value": 17826840 }, { "step": 54, "value": 17826840 }, { "step": 55, "value": 17826840 }, { "step": 56, "value": 17826840 }, { "step": 57, "value": 17826840 }, { "step": 58, "value": 17826840 }, { "step": 59, "value": 17826840 }, { "step": 60, "value": 17826840 }, { "step": 61, "value": 17826840 }, { "step": 62, "value": 17826840 }, { "step": 63, "value": 17826840 }, { "step": 64, "value": 17826840 }, { "step": 65, "value": 17826840 }, { "step": 66, "value": 17826840 }, { "step": 67, "value": 17826840 }, { "step": 68, "value": 17826840 }, { "step": 69, "value": 17826840 }, { "step": 70, "value": 17826840 }, { "step": 71, "value": 17826840 }, { "step": 72, "value": 17826840 }, { "step": 73, "value": 17826840 }, { "step": 74, "value": 17826840 }, { "step": 75, "value": 17826840 }, { "step": 76, "value": 17826840 }, { "step": 77, "value": 17826840 }, { "step": 78, "value": 17826840 }, { "step": 79, "value": 17826840 }, { "step": 80, "value": 17826840 }, { "step": 81, "value": 17826840 }, { "step": 82, "value": 17826840 }, { "step": 83, "value": 17826840 }, { "step": 84, "value": 17826840 }, { "step": 85, "value": 17826840 }, { "step": 86, "value": 17826840 }, { "step": 87, "value": 17826840 }, { "step": 88, "value": 17826840 }, { "step": 89, "value": 17826840 }, { "step": 90, "value": 17826840 }, { "step": 91, "value": 17826840 }, { "step": 92, "value": 17826840 }, { "step": 93, "value": 17826840 }, { "step": 94, "value": 17826840 }, { "step": 95, "value": 17826840 }, { "step": 96, "value": 17826840 }, { "step": 97, "value": 17826840 }, { "step": 98, "value": 17826840 }, { "step": 99, "value": 17826840 }, { "step": 100, "value": 17826840 } ], "submission_sha256": "c949fa992dcd134fb953ac4569c430f3945a1404b23af09de8b0a4d73cb4357a", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/heuresis-map-elites/params/submission.json", "comparison_value": 17826840, "comparison_rank_key": [ 17826840.0, 486277120.0, -0.0007281682164659298 ], "rank": 6 }, { "run_id": "tpe--params", "method_id": "tpe", "task_id": "params", "metric": "num_params_total", "unit": "parameters", "direction": "minimize", "best_value": 26345772, "reference_value": 50332176, "relative_improvement": 0.4765620306183464, "rank_key": [ 26345772.0, 713555968.0, -0.01559096338452437 ], "best_launch_seq": 25, "best_candidate_id": "tpe-trial-0024", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 16.03298555555556, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 50332176, "is_reference": true }, { "step": 14, "launch_seq": 14, "candidate_id": "tpe-trial-0013", "value": 37945842, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "tpe-trial-0014", "value": 39715316, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "tpe-trial-0016", "value": 39715796, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "tpe-trial-0017", "value": 37945842, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "tpe-trial-0021", "value": 39715316, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "tpe-trial-0024", "value": 26345772, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "tpe-trial-0027", "value": 33030928, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "tpe-trial-0029", "value": 41288592, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "tpe-trial-0030", "value": 33030544, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "tpe-trial-0031", "value": 33030544, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "tpe-trial-0055", "value": 37945842, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "tpe-trial-0059", "value": 33030544, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "tpe-trial-0060", "value": 29360264, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "tpe-trial-0061", "value": 29360264, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "tpe-trial-0063", "value": 29360264, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "tpe-trial-0064", "value": 29360264, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "tpe-trial-0065", "value": 29360264, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "tpe-trial-0066", "value": 29360264, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "tpe-trial-0067", "value": 29360264, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "tpe-trial-0068", "value": 29360264, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "tpe-trial-0069", "value": 29360264, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "tpe-trial-0070", "value": 29360264, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "tpe-trial-0071", "value": 29360264, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "tpe-trial-0073", "value": 29360392, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "tpe-trial-0077", "value": 29360264, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "tpe-trial-0079", "value": 29360264, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "tpe-trial-0081", "value": 29360264, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "tpe-trial-0083", "value": 29360264, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "tpe-trial-0085", "value": 29360264, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "tpe-trial-0089", "value": 29360264, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "tpe-trial-0091", "value": 29360264, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "tpe-trial-0092", "value": 29360264, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "tpe-trial-0094", "value": 29360264, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "tpe-trial-0096", "value": 29360264, "is_reference": false } ], "curve": [ { "step": 1, "value": 50332176 }, { "step": 2, "value": 50332176 }, { "step": 3, "value": 50332176 }, { "step": 4, "value": 50332176 }, { "step": 5, "value": 50332176 }, { "step": 6, "value": 50332176 }, { "step": 7, "value": 50332176 }, { "step": 8, "value": 50332176 }, { "step": 9, "value": 50332176 }, { "step": 10, "value": 50332176 }, { "step": 11, "value": 50332176 }, { "step": 12, "value": 50332176 }, { "step": 13, "value": 50332176 }, { "step": 14, "value": 37945842 }, { "step": 15, "value": 37945842 }, { "step": 16, "value": 37945842 }, { "step": 17, "value": 37945842 }, { "step": 18, "value": 37945842 }, { "step": 19, "value": 37945842 }, { "step": 20, "value": 37945842 }, { "step": 21, "value": 37945842 }, { "step": 22, "value": 37945842 }, { "step": 23, "value": 37945842 }, { "step": 24, "value": 37945842 }, { "step": 25, "value": 26345772 }, { "step": 26, "value": 26345772 }, { "step": 27, "value": 26345772 }, { "step": 28, "value": 26345772 }, { "step": 29, "value": 26345772 }, { "step": 30, "value": 26345772 }, { "step": 31, "value": 26345772 }, { "step": 32, "value": 26345772 }, { "step": 33, "value": 26345772 }, { "step": 34, "value": 26345772 }, { "step": 35, "value": 26345772 }, { "step": 36, "value": 26345772 }, { "step": 37, "value": 26345772 }, { "step": 38, "value": 26345772 }, { "step": 39, "value": 26345772 }, { "step": 40, "value": 26345772 }, { "step": 41, "value": 26345772 }, { "step": 42, "value": 26345772 }, { "step": 43, "value": 26345772 }, { "step": 44, "value": 26345772 }, { "step": 45, "value": 26345772 }, { "step": 46, "value": 26345772 }, { "step": 47, "value": 26345772 }, { "step": 48, "value": 26345772 }, { "step": 49, "value": 26345772 }, { "step": 50, "value": 26345772 }, { "step": 51, "value": 26345772 }, { "step": 52, "value": 26345772 }, { "step": 53, "value": 26345772 }, { "step": 54, "value": 26345772 }, { "step": 55, "value": 26345772 }, { "step": 56, "value": 26345772 }, { "step": 57, "value": 26345772 }, { "step": 58, "value": 26345772 }, { "step": 59, "value": 26345772 }, { "step": 60, "value": 26345772 }, { "step": 61, "value": 26345772 }, { "step": 62, "value": 26345772 }, { "step": 63, "value": 26345772 }, { "step": 64, "value": 26345772 }, { "step": 65, "value": 26345772 }, { "step": 66, "value": 26345772 }, { "step": 67, "value": 26345772 }, { "step": 68, "value": 26345772 }, { "step": 69, "value": 26345772 }, { "step": 70, "value": 26345772 }, { "step": 71, "value": 26345772 }, { "step": 72, "value": 26345772 }, { "step": 73, "value": 26345772 }, { "step": 74, "value": 26345772 }, { "step": 75, "value": 26345772 }, { "step": 76, "value": 26345772 }, { "step": 77, "value": 26345772 }, { "step": 78, "value": 26345772 }, { "step": 79, "value": 26345772 }, { "step": 80, "value": 26345772 }, { "step": 81, "value": 26345772 }, { "step": 82, "value": 26345772 }, { "step": 83, "value": 26345772 }, { "step": 84, "value": 26345772 }, { "step": 85, "value": 26345772 }, { "step": 86, "value": 26345772 }, { "step": 87, "value": 26345772 }, { "step": 88, "value": 26345772 }, { "step": 89, "value": 26345772 }, { "step": 90, "value": 26345772 }, { "step": 91, "value": 26345772 }, { "step": 92, "value": 26345772 }, { "step": 93, "value": 26345772 }, { "step": 94, "value": 26345772 }, { "step": 95, "value": 26345772 }, { "step": 96, "value": 26345772 }, { "step": 97, "value": 26345772 }, { "step": 98, "value": 26345772 }, { "step": 99, "value": 26345772 }, { "step": 100, "value": 26345772 } ], "submission_sha256": "726e863f18c44186aacbd759bd1bec7a268f7d85a81130a24bdaaa5467e2bec1", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/tpe/params/submission.json", "comparison_value": 26345772, "comparison_rank_key": [ 26345772.0, 713555968.0, -0.01559096338452437 ], "rank": 7 } ] }, { "id": "request", "title": "Request latency at validation BPB < 1.05", "metric": "request_ms_median", "unit": "ms", "direction": "minimize", "budget": 100, "comparison_step": null, "quality_gate": { "metric": "val_bpb", "operator": "<", "value": 1.05 }, "tiebreaks": [ { "metric": "nopref_request_ms_median", "direction": "minimize" } ], "runs": [ { "run_id": "autoscientists--request", "method_id": "autoscientists", "task_id": "request", "metric": "request_ms_median", "unit": "ms", "direction": "minimize", "best_value": 29.457807540893555, "reference_value": 642.3138380050659, "relative_improvement": 0.9541379839606363, "rank_key": [ 29.457807540893555, 28.034090995788574, -0.0027847589251912552 ], "best_launch_seq": 88, "best_candidate_id": "exp-bcf27bee966b0803f85b9262", "best_is_reference": false, "budget": { "spent": 87, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 2, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 2 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 17.135465, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 642.3138380050659, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-9112f910ebf809c64cdd2eee", "value": 631.2206983566284, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-e0f352f921e139af920810c2", "value": 612.2527122497559, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-fb070516b9fd8cdda9021499", "value": 471.31896018981934, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-3c70183765547b95071ff33e", "value": 554.51500415802, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-e52bdae62fe51847ba64b7ba", "value": 471.09735012054443, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp-95d33782258b948c9938bc4a", "value": 440.7989978790283, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp-010aa332c5489631e14c48c6", "value": 384.1249942779541, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-a2db77266a15357e03f6016c", "value": 437.53159046173096, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-e6c7ab525eb489ce7e4279b4", "value": 246.20914459228516, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-90808c960d2d2a9f424f6806", "value": 384.0678930282593, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-46342a190eb48bdc97058b0a", "value": 228.26385498046875, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-146fe43a23dd8e5bbc97eb18", "value": 242.82443523406982, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-495f7ec49f3c1f5d75500d71", "value": 221.33123874664307, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp-fdad89f426ab512cd15bc542", "value": 232.98633098602295, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-a11059a152c017ca62f10991", "value": 218.91868114471436, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-df114dc79e90ccd22b9eb2b6", "value": 219.193696975708, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-437269a5eed6c5554f94ae26", "value": 212.25523948669434, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-9a09673c0580e5d94c9ae360", "value": 218.45674514770508, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-594ef63c141d0ee1265abeb5", "value": 202.98945903778076, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp-523b442e135f5727e3180c23", "value": 222.0742702484131, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp-711fa669936f82519dab1299", "value": 205.40833473205566, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp-6b9d188b124de92b6561e8d7", "value": 192.4598217010498, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp-0d8fc783172b70734bb8d870", "value": 193.7037706375122, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-46fd89849cf31e11d6f722da", "value": 190.7409429550171, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp-af72b7833f49ced76b5f6a57", "value": 162.61839866638184, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp-412213f9218a458c37364410", "value": 147.99129962921143, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp-14511114d2ab3e75c0d20843", "value": 155.56859970092773, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp-eff51fe8b8af8b0d4e1df084", "value": 155.10737895965576, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp-824ad014f3018d054078d55f", "value": 150.7202386856079, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp-2ba7958145b99ed72c3b38cc", "value": 143.09155941009521, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-a8b8d216fd87dc81df73d70c", "value": 145.84267139434814, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-48fc59cb07d87ec0f461dd37", "value": 140.7691240310669, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-abfde5398755fec6d6652e5b", "value": 140.26248455047607, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-70566f6de8c53f5075b417a5", "value": 137.54379749298096, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp-c68446ad3c43b869b34b1ea9", "value": 191.10095500946045, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp-d83f245264b5b913ddd895df", "value": 135.0095272064209, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp-86a74fe7196af49ad9761567", "value": 142.95125007629395, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "exp-3282fe9dbdf8dd644ba17daf", "value": 126.87301635742188, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp-ae37daea0330b39e252488ff", "value": 226.02272033691406, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "exp-5b5aea8b1afdb0d98684b812", "value": 120.64695358276367, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp-f7b89c358dfaa09b105fadc0", "value": 115.82529544830322, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp-5e1cc2de4fbd51265d9f8ea5", "value": 117.78450012207031, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp-83b299d80f921a56b53e24ba", "value": 171.53596878051758, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp-a3c4a2c665f7bcb2a334ebe4", "value": 111.29510402679443, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp-6bda1e1fe79207dbbe98c560", "value": 108.44242572784424, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp-a0f0de71d04fc9bc0dc20e63", "value": 103.56366634368896, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp-bbc0a1e0a3bc56aee2b2f37d", "value": 106.78684711456299, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp-b94c0117d326a8e6535c3dbc", "value": 102.60868072509766, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "exp-65a0a3441ef41db49351509d", "value": 99.92825984954834, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp-749cb2c2bca50822970b0ecb", "value": 100.45063495635986, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp-59d918389d8c6129ef651e71", "value": 97.77724742889404, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp-15032ced7e4e4f5965f26af2", "value": 95.65615653991699, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp-4e0a27862fe0b01577c7d86c", "value": 97.47922420501709, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp-af95f4038781d09bacba98dc", "value": 96.65417671203613, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "exp-86614bf62397f6f71b37c079", "value": 95.84295749664307, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "exp-28d72509fdd0969bed4dd47d", "value": 98.19471836090088, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "exp-cea688c4825fd85fc7716888", "value": 94.0544605255127, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "exp-e5837c4997891a7ca6a3e2fc", "value": 92.21482276916504, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp-9689abb32e252b53c4c90abc", "value": 94.88534927368164, "is_reference": false }, { "step": 64, "launch_seq": 65, "candidate_id": "exp-7599fa5cdc8964a9ada873cb", "value": 97.60546684265137, "is_reference": false }, { "step": 65, "launch_seq": 66, "candidate_id": "exp-d4a84d8a1cc3afe6f2139b3c", "value": 89.06209468841553, "is_reference": false }, { "step": 66, "launch_seq": 67, "candidate_id": "exp-54b617b7ae72f1298001ef8e", "value": 97.0308780670166, "is_reference": false }, { "step": 67, "launch_seq": 68, "candidate_id": "exp-980c2dcd964314c7e02d5feb", "value": 86.58277988433838, "is_reference": false }, { "step": 68, "launch_seq": 69, "candidate_id": "exp-4fc9bb25deac225b84fbf511", "value": 87.63635158538818, "is_reference": false }, { "step": 69, "launch_seq": 70, "candidate_id": "exp-6d18736b75dfa8319c05728c", "value": 85.27588844299316, "is_reference": false }, { "step": 70, "launch_seq": 71, "candidate_id": "exp-0cfb0bed450c7447230ef755", "value": 87.44335174560547, "is_reference": false }, { "step": 71, "launch_seq": 72, "candidate_id": "exp-480203fb3577bdd632c94484", "value": 84.82789993286133, "is_reference": false }, { "step": 72, "launch_seq": 73, "candidate_id": "exp-513eac7dfa2dd48367b944ef", "value": 87.0746374130249, "is_reference": false }, { "step": 73, "launch_seq": 74, "candidate_id": "exp-fb51940f7dc216b4f841998b", "value": 85.16669273376465, "is_reference": false }, { "step": 74, "launch_seq": 75, "candidate_id": "exp-fb7cc06228a59229aaf1bbd9", "value": 74.0346908569336, "is_reference": false }, { "step": 75, "launch_seq": 76, "candidate_id": "exp-efe2282b6e66b211eb237749", "value": 63.941240310668945, "is_reference": false }, { "step": 76, "launch_seq": 77, "candidate_id": "exp-cb4b12514a2b53892f9ecf38", "value": 53.21204662322998, "is_reference": false }, { "step": 77, "launch_seq": 78, "candidate_id": "exp-3bf68d98ca6c5f917b899186", "value": 42.07766056060791, "is_reference": false }, { "step": 79, "launch_seq": 80, "candidate_id": "exp-15625969c60b930f8f65b8e6", "value": 34.469008445739746, "is_reference": false }, { "step": 80, "launch_seq": 81, "candidate_id": "exp-3f424eb27fa29a5df16f7095", "value": 33.99157524108887, "is_reference": false }, { "step": 81, "launch_seq": 82, "candidate_id": "exp-178dfebccdf855015f645571", "value": 33.609628677368164, "is_reference": false }, { "step": 82, "launch_seq": 83, "candidate_id": "exp-645d4d68b9ce8695b604b662", "value": 32.735228538513184, "is_reference": false }, { "step": 83, "launch_seq": 84, "candidate_id": "exp-0d1b26b0521edafdc23419ad", "value": 32.96017646789551, "is_reference": false }, { "step": 86, "launch_seq": 87, "candidate_id": "exp-5f37b9c6795b5c3ae0938254", "value": 31.119346618652344, "is_reference": false }, { "step": 87, "launch_seq": 88, "candidate_id": "exp-bcf27bee966b0803f85b9262", "value": 29.457807540893555, "is_reference": false } ], "curve": [ { "step": 1, "value": 642.3138380050659 }, { "step": 2, "value": 631.2206983566284 }, { "step": 3, "value": 631.2206983566284 }, { "step": 4, "value": 612.2527122497559 }, { "step": 5, "value": 471.31896018981934 }, { "step": 6, "value": 471.31896018981934 }, { "step": 7, "value": 471.09735012054443 }, { "step": 8, "value": 440.7989978790283 }, { "step": 9, "value": 384.1249942779541 }, { "step": 10, "value": 384.1249942779541 }, { "step": 11, "value": 246.20914459228516 }, { "step": 12, "value": 246.20914459228516 }, { "step": 13, "value": 228.26385498046875 }, { "step": 14, "value": 228.26385498046875 }, { "step": 15, "value": 221.33123874664307 }, { "step": 16, "value": 221.33123874664307 }, { "step": 17, "value": 218.91868114471436 }, { "step": 18, "value": 218.91868114471436 }, { "step": 19, "value": 212.25523948669434 }, { "step": 20, "value": 212.25523948669434 }, { "step": 21, "value": 202.98945903778076 }, { "step": 22, "value": 202.98945903778076 }, { "step": 23, "value": 202.98945903778076 }, { "step": 24, "value": 192.4598217010498 }, { "step": 25, "value": 192.4598217010498 }, { "step": 26, "value": 190.7409429550171 }, { "step": 27, "value": 162.61839866638184 }, { "step": 28, "value": 147.99129962921143 }, { "step": 29, "value": 147.99129962921143 }, { "step": 30, "value": 147.99129962921143 }, { "step": 31, "value": 147.99129962921143 }, { "step": 32, "value": 143.09155941009521 }, { "step": 33, "value": 143.09155941009521 }, { "step": 34, "value": 140.7691240310669 }, { "step": 35, "value": 140.26248455047607 }, { "step": 36, "value": 137.54379749298096 }, { "step": 37, "value": 137.54379749298096 }, { "step": 38, "value": 135.0095272064209 }, { "step": 39, "value": 135.0095272064209 }, { "step": 40, "value": 126.87301635742188 }, { "step": 41, "value": 126.87301635742188 }, { "step": 42, "value": 120.64695358276367 }, { "step": 43, "value": 115.82529544830322 }, { "step": 44, "value": 115.82529544830322 }, { "step": 45, "value": 115.82529544830322 }, { "step": 46, "value": 111.29510402679443 }, { "step": 47, "value": 108.44242572784424 }, { "step": 48, "value": 108.44242572784424 }, { "step": 49, "value": 108.44242572784424 }, { "step": 50, "value": 103.56366634368896 }, { "step": 51, "value": 103.56366634368896 }, { "step": 52, "value": 102.60868072509766 }, { "step": 53, "value": 99.92825984954834 }, { "step": 54, "value": 99.92825984954834 }, { "step": 55, "value": 97.77724742889404 }, { "step": 56, "value": 95.65615653991699 }, { "step": 57, "value": 95.65615653991699 }, { "step": 58, "value": 95.65615653991699 }, { "step": 59, "value": 95.65615653991699 }, { "step": 60, "value": 95.65615653991699 }, { "step": 61, "value": 94.0544605255127 }, { "step": 62, "value": 92.21482276916504 }, { "step": 63, "value": 92.21482276916504 }, { "step": 64, "value": 92.21482276916504 }, { "step": 65, "value": 89.06209468841553 }, { "step": 66, "value": 89.06209468841553 }, { "step": 67, "value": 86.58277988433838 }, { "step": 68, "value": 86.58277988433838 }, { "step": 69, "value": 85.27588844299316 }, { "step": 70, "value": 85.27588844299316 }, { "step": 71, "value": 84.82789993286133 }, { "step": 72, "value": 84.82789993286133 }, { "step": 73, "value": 84.82789993286133 }, { "step": 74, "value": 74.0346908569336 }, { "step": 75, "value": 63.941240310668945 }, { "step": 76, "value": 53.21204662322998 }, { "step": 77, "value": 42.07766056060791 }, { "step": 78, "value": 42.07766056060791 }, { "step": 79, "value": 34.469008445739746 }, { "step": 80, "value": 33.99157524108887 }, { "step": 81, "value": 33.609628677368164 }, { "step": 82, "value": 32.735228538513184 }, { "step": 83, "value": 32.735228538513184 }, { "step": 84, "value": 32.735228538513184 }, { "step": 85, "value": 32.735228538513184 }, { "step": 86, "value": 31.119346618652344 }, { "step": 87, "value": 29.457807540893555 } ], "submission_sha256": "650f65b20c3471e77633b55f763f250b3937ecda86ff082b376c01c7aab3939b", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/autoscientists/request/submission.json", "comparison_value": 29.457807540893555, "comparison_rank_key": [ 29.457807540893555, 28.034090995788574, -0.0027847589251912552 ], "rank": 1 }, { "run_id": "arbor--request", "method_id": "arbor", "task_id": "request", "metric": "request_ms_median", "unit": "ms", "direction": "minimize", "best_value": 57.88278579711914, "reference_value": 641.8812274932861, "relative_improvement": 0.9098232144548509, "rank_key": [ 57.88278579711914, 48.476338386535645, -0.002639317582040057 ], "best_launch_seq": 89, "best_candidate_id": "exp-1e0fe72d72f81657495ae52e", "best_is_reference": false, "budget": { "spent": 89, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 18.77576333333333, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 641.8812274932861, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-7aabeac412927a629e057415", "value": 469.3831205368042, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp-21c1c5913169687f08cdb875", "value": 673.9263534545898, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-b4ac4df2f504a54943931c01", "value": 522.7757692337036, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-79cb9cf95b3c9429e3cdcd8c", "value": 353.96289825439453, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-946fe08bc4723b96970feb2e", "value": 214.52629566192627, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-52e4f0cd5a3d5e0bc7c2d372", "value": 303.1829595565796, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp-9148b9acdbb2e3535fe7823f", "value": 308.74693393707275, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp-68d0eab2748fe6d51d81f96a", "value": 489.3765449523926, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-ebf0cecbbce1d3c3ca89227c", "value": 141.7233943939209, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-34aec1452d80ad69e70e28a7", "value": 579.4081687927246, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-2814df402df20547ef8333ad", "value": 128.36837768554688, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-d14d65220cc6ecbba505884f", "value": 206.49433135986328, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-f28d2409291abaaa17c1abbd", "value": 111.42539978027344, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-f3bf9e49f5ac7cea7bef777b", "value": 111.053466796875, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp-47a114d2826da2fe02da06ea", "value": 105.22377490997314, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-ca23b0b6b0b64e6e59d3211c", "value": 111.19520664215088, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-45a0c2df9dd4edce1bacfb20", "value": 97.84436225891113, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-c94bb5a8dd5fd0cf63a70e11", "value": 111.40322685241699, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-6bebe8abc2eb997261e8915f", "value": 84.90550518035889, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp-950db70449cb9d2b1a06d0ca", "value": 83.7634801864624, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-6afa87c01b8631a89e4474f9", "value": 84.38539505004883, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp-3e785d363d6dd5e009408c2c", "value": 79.83803749084473, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp-c050647891f445297073b58c", "value": 79.3536901473999, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp-059720ab0bfcf77b5cc01763", "value": 79.55121994018555, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp-4ef90621609a57c4789d60e1", "value": 137.4732255935669, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp-1115c15e8f4c8df893d16539", "value": 79.63550090789795, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp-7b28ce17e88c6075dbf8e6d2", "value": 137.50410079956055, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-2eaf90a6409c2e911ccbf737", "value": 79.51498031616211, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-7a38ef1ecaa485b6ac1c7ae3", "value": 80.23619651794434, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-7653a6ff437020c06d521883", "value": 79.31983470916748, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-eaa269a19b10b4fb29e4d67f", "value": 79.32054996490479, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp-d971c0820fc40cab1f40c1a6", "value": 79.5280933380127, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp-8aca76cb383c8fce11807288", "value": 79.2456865310669, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp-e99dc30902056cd4fcec97ad", "value": 78.7132978439331, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "exp-b5bdcd28f3f3be29a63054be", "value": 68.66896152496338, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp-88e25eb618372a302295e869", "value": 72.03865051269531, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp-043e02652999ce580c24e810", "value": 70.02866268157959, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp-d1466fa2d8b4f7f4f5411b8d", "value": 96.48740291595459, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp-07eec5b5890fca86945638f5", "value": 66.80762767791748, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp-0740b13cb910118bb5dd420c", "value": 69.25654411315918, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "exp-04fe016de0f5614dad7b90e5", "value": 65.18673896789551, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp-76369287d0794df517073ac6", "value": 154.8004150390625, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp-2925a718209f5ab0a89f5bf8", "value": 63.814640045166016, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp-2cb683ff0a6a6b8b309a7eaf", "value": 63.2479190826416, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "exp-bad592240c1a1a1acb0a039e", "value": 64.06784057617188, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp-35dd862194039d84357e3dae", "value": 64.39352035522461, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp-202834e4bf6650eee979f50e", "value": 63.539981842041016, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp-978e78552cc24099348bf81f", "value": 64.60714340209961, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp-8504ebbb1862b98b2bfb5912", "value": 63.63058090209961, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp-4e3501b86d4f4098e06f104c", "value": 63.87650966644287, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "exp-fecdf0bb20f1ee5a699118b6", "value": 66.89512729644775, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "exp-c94371711dfce031235f6078", "value": 63.07828426361084, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp-1a81b8422210b4a422740839", "value": 63.47155570983887, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp-ff03dc1582952c697fdae33c", "value": 63.855528831481934, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "exp-a8ae4ca2b69b207fd7decef5", "value": 64.25964832305908, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp-805d7132370005f59636b265", "value": 63.44747543334961, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "exp-3807535ef886b97a7da51199", "value": 65.041184425354, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "exp-15a6c7d35eece0840e53e7a9", "value": 63.59589099884033, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "exp-701455dcb6df8d0178a1aebe", "value": 66.85280799865723, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "exp-4d5d3cebd5deb6b61b9c34d1", "value": 64.54885005950928, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp-a3c211b81042d547fa78bbba", "value": 63.555121421813965, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "exp-3a68225370d4d4b10662276f", "value": 63.00365924835205, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "exp-7840d037b577b0220a3f108c", "value": 64.25321102142334, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "exp-c9c69d68d73494d8d816e708", "value": 63.68875503540039, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "exp-e9c9435d2691422ddae5cf40", "value": 64.78679180145264, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "exp-e8283391364c295975cdbbc1", "value": 63.90976905822754, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "exp-23029a4c03d9ccd4cad28263", "value": 63.51661682128906, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "exp-95ea2298423ecb71a182d0f2", "value": 63.29345703125, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "exp-711fcc2a1e42977ee4c21876", "value": 64.77582454681396, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "exp-5d70d4f822242b6c53049ab4", "value": 63.973426818847656, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp-81b8aaeaa454f9d5cc9faeb0", "value": 63.616275787353516, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "exp-2b9301f1c9975d7cf8693539", "value": 59.77761745452881, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "exp-d806b610816fce731b66a717", "value": 58.219075202941895, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "exp-1e0fe72d72f81657495ae52e", "value": 57.88278579711914, "is_reference": false } ], "curve": [ { "step": 1, "value": 641.8812274932861 }, { "step": 2, "value": 469.3831205368042 }, { "step": 3, "value": 469.3831205368042 }, { "step": 4, "value": 469.3831205368042 }, { "step": 5, "value": 353.96289825439453 }, { "step": 6, "value": 214.52629566192627 }, { "step": 7, "value": 214.52629566192627 }, { "step": 8, "value": 214.52629566192627 }, { "step": 9, "value": 214.52629566192627 }, { "step": 10, "value": 141.7233943939209 }, { "step": 11, "value": 141.7233943939209 }, { "step": 12, "value": 128.36837768554688 }, { "step": 13, "value": 128.36837768554688 }, { "step": 14, "value": 111.42539978027344 }, { "step": 15, "value": 111.053466796875 }, { "step": 16, "value": 105.22377490997314 }, { "step": 17, "value": 105.22377490997314 }, { "step": 18, "value": 97.84436225891113 }, { "step": 19, "value": 97.84436225891113 }, { "step": 20, "value": 84.90550518035889 }, { "step": 21, "value": 84.90550518035889 }, { "step": 22, "value": 84.90550518035889 }, { "step": 23, "value": 84.90550518035889 }, { "step": 24, "value": 83.7634801864624 }, { "step": 25, "value": 83.7634801864624 }, { "step": 26, "value": 83.7634801864624 }, { "step": 27, "value": 79.83803749084473 }, { "step": 28, "value": 79.3536901473999 }, { "step": 29, "value": 79.3536901473999 }, { "step": 30, "value": 79.3536901473999 }, { "step": 31, "value": 79.3536901473999 }, { "step": 32, "value": 79.3536901473999 }, { "step": 33, "value": 79.3536901473999 }, { "step": 34, "value": 79.3536901473999 }, { "step": 35, "value": 79.31983470916748 }, { "step": 36, "value": 79.31983470916748 }, { "step": 37, "value": 79.31983470916748 }, { "step": 38, "value": 79.2456865310669 }, { "step": 39, "value": 78.7132978439331 }, { "step": 40, "value": 78.7132978439331 }, { "step": 41, "value": 78.7132978439331 }, { "step": 42, "value": 68.66896152496338 }, { "step": 43, "value": 68.66896152496338 }, { "step": 44, "value": 68.66896152496338 }, { "step": 45, "value": 68.66896152496338 }, { "step": 46, "value": 68.66896152496338 }, { "step": 47, "value": 66.80762767791748 }, { "step": 48, "value": 66.80762767791748 }, { "step": 49, "value": 65.18673896789551 }, { "step": 50, "value": 65.18673896789551 }, { "step": 51, "value": 63.814640045166016 }, { "step": 52, "value": 63.2479190826416 }, { "step": 53, "value": 63.2479190826416 }, { "step": 54, "value": 63.2479190826416 }, { "step": 55, "value": 63.2479190826416 }, { "step": 56, "value": 63.2479190826416 }, { "step": 57, "value": 63.2479190826416 }, { "step": 58, "value": 63.2479190826416 }, { "step": 59, "value": 63.2479190826416 }, { "step": 60, "value": 63.07828426361084 }, { "step": 61, "value": 63.07828426361084 }, { "step": 62, "value": 63.07828426361084 }, { "step": 63, "value": 63.07828426361084 }, { "step": 64, "value": 63.07828426361084 }, { "step": 65, "value": 63.07828426361084 }, { "step": 66, "value": 63.07828426361084 }, { "step": 67, "value": 63.07828426361084 }, { "step": 68, "value": 63.07828426361084 }, { "step": 69, "value": 63.07828426361084 }, { "step": 70, "value": 63.07828426361084 }, { "step": 71, "value": 63.07828426361084 }, { "step": 72, "value": 63.00365924835205 }, { "step": 73, "value": 63.00365924835205 }, { "step": 74, "value": 63.00365924835205 }, { "step": 75, "value": 63.00365924835205 }, { "step": 76, "value": 63.00365924835205 }, { "step": 77, "value": 63.00365924835205 }, { "step": 78, "value": 63.00365924835205 }, { "step": 79, "value": 63.00365924835205 }, { "step": 80, "value": 63.00365924835205 }, { "step": 81, "value": 63.00365924835205 }, { "step": 82, "value": 63.00365924835205 }, { "step": 83, "value": 63.00365924835205 }, { "step": 84, "value": 63.00365924835205 }, { "step": 85, "value": 63.00365924835205 }, { "step": 86, "value": 63.00365924835205 }, { "step": 87, "value": 59.77761745452881 }, { "step": 88, "value": 58.219075202941895 }, { "step": 89, "value": 57.88278579711914 } ], "submission_sha256": "3652fe3f37c50412aa5db27ad4f26f1ffb3e7de9469af9e0ffb107e8665a3300", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/arbor/request/submission.json", "comparison_value": 57.88278579711914, "comparison_rank_key": [ 57.88278579711914, 48.476338386535645, -0.002639317582040057 ], "rank": 2 }, { "run_id": "sequential-search--request", "method_id": "sequential-search", "task_id": "request", "metric": "request_ms_median", "unit": "ms", "direction": "minimize", "best_value": 61.29932403564453, "reference_value": 641.6399478912354, "relative_improvement": 0.9044646078581825, "rank_key": [ 61.29932403564453, 55.83930015563965, -0.0014693673157271103 ], "best_launch_seq": 76, "best_candidate_id": "c0074", "best_is_reference": false, "budget": { "spent": 80, "max_launches": 100 }, "compute": { "backend": "pluto", "evaluation_capacity_gpus": 1, "gpu_instance_type": null, "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "gpus_per_evaluation": 1, "workers": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 15.683438333333337, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 641.6399478912354, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "c0001", "value": 484.46667194366455, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "c0002", "value": 361.5250587463379, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "c0003", "value": 300.16040802001953, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "c0004", "value": 225.21567344665527, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "c0005", "value": 223.016619682312, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "c0006", "value": 219.71559524536133, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "c0007", "value": 227.6989221572876, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "c0008", "value": 223.1149673461914, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "c0009", "value": 222.86653518676758, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "c0011", "value": 222.90217876434326, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "c0012", "value": 213.57643604278564, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "c0013", "value": 199.43773746490479, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "c0014", "value": 208.86313915252686, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "c0015", "value": 197.3193883895874, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "c0016", "value": 208.28473567962646, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "c0017", "value": 197.26812839508057, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "c0018", "value": 197.0125436782837, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "c0019", "value": 194.0772533416748, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "c0020", "value": 191.72167778015137, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "c0021", "value": 190.51146507263184, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "c0022", "value": 190.71602821350098, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "c0023", "value": 190.38987159729004, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "c0024", "value": 188.5521411895752, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "c0025", "value": 187.90411949157715, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "c0026", "value": 167.92738437652588, "is_reference": false }, { "step": 28, "launch_seq": 29, "candidate_id": "c0027", "value": 145.7422971725464, "is_reference": false }, { "step": 29, "launch_seq": 30, "candidate_id": "c0028", "value": 124.23896789550781, "is_reference": false }, { "step": 30, "launch_seq": 31, "candidate_id": "c0029", "value": 102.62858867645264, "is_reference": false }, { "step": 31, "launch_seq": 32, "candidate_id": "c0030", "value": 117.41077899932861, "is_reference": false }, { "step": 32, "launch_seq": 33, "candidate_id": "c0031", "value": 93.90711784362793, "is_reference": false }, { "step": 33, "launch_seq": 34, "candidate_id": "c0032", "value": 102.62823104858398, "is_reference": false }, { "step": 36, "launch_seq": 37, "candidate_id": "c0035", "value": 88.66989612579346, "is_reference": false }, { "step": 37, "launch_seq": 38, "candidate_id": "c0036", "value": 90.63148498535156, "is_reference": false }, { "step": 38, "launch_seq": 39, "candidate_id": "c0037", "value": 89.7747278213501, "is_reference": false }, { "step": 39, "launch_seq": 40, "candidate_id": "c0038", "value": 86.89796924591064, "is_reference": false }, { "step": 41, "launch_seq": 42, "candidate_id": "c0040", "value": 84.46311950683594, "is_reference": false }, { "step": 42, "launch_seq": 43, "candidate_id": "c0041", "value": 82.15177059173584, "is_reference": false }, { "step": 43, "launch_seq": 44, "candidate_id": "c0042", "value": 81.34937286376953, "is_reference": false }, { "step": 44, "launch_seq": 45, "candidate_id": "c0043", "value": 82.03637599945068, "is_reference": false }, { "step": 45, "launch_seq": 46, "candidate_id": "c0044", "value": 81.50887489318848, "is_reference": false }, { "step": 46, "launch_seq": 47, "candidate_id": "c0045", "value": 81.52270317077637, "is_reference": false }, { "step": 47, "launch_seq": 48, "candidate_id": "c0046", "value": 81.1147689819336, "is_reference": false }, { "step": 48, "launch_seq": 49, "candidate_id": "c0047", "value": 81.91275596618652, "is_reference": false }, { "step": 49, "launch_seq": 50, "candidate_id": "c0048", "value": 80.98006248474121, "is_reference": false }, { "step": 51, "launch_seq": 52, "candidate_id": "c0050", "value": 79.98406887054443, "is_reference": false }, { "step": 52, "launch_seq": 53, "candidate_id": "c0051", "value": 87.98635005950928, "is_reference": false }, { "step": 53, "launch_seq": 54, "candidate_id": "c0052", "value": 79.94520664215088, "is_reference": false }, { "step": 54, "launch_seq": 55, "candidate_id": "c0053", "value": 74.85330104827881, "is_reference": false }, { "step": 55, "launch_seq": 56, "candidate_id": "c0054", "value": 73.76396656036377, "is_reference": false }, { "step": 57, "launch_seq": 58, "candidate_id": "c0056", "value": 73.84061813354492, "is_reference": false }, { "step": 58, "launch_seq": 59, "candidate_id": "c0057", "value": 81.9159746170044, "is_reference": false }, { "step": 59, "launch_seq": 60, "candidate_id": "c0058", "value": 73.3788013458252, "is_reference": false }, { "step": 60, "launch_seq": 61, "candidate_id": "c0059", "value": 73.10605049133301, "is_reference": false }, { "step": 61, "launch_seq": 62, "candidate_id": "c0060", "value": 72.9449987411499, "is_reference": false }, { "step": 62, "launch_seq": 63, "candidate_id": "c0061", "value": 73.36211204528809, "is_reference": false }, { "step": 63, "launch_seq": 64, "candidate_id": "c0062", "value": 73.11499118804932, "is_reference": false }, { "step": 65, "launch_seq": 66, "candidate_id": "c0064", "value": 72.91901111602783, "is_reference": false }, { "step": 66, "launch_seq": 67, "candidate_id": "c0065", "value": 67.43896007537842, "is_reference": false }, { "step": 67, "launch_seq": 68, "candidate_id": "c0066", "value": 66.79999828338623, "is_reference": false }, { "step": 68, "launch_seq": 69, "candidate_id": "c0067", "value": 68.63248348236084, "is_reference": false }, { "step": 69, "launch_seq": 70, "candidate_id": "c0068", "value": 67.58403778076172, "is_reference": false }, { "step": 70, "launch_seq": 71, "candidate_id": "c0069", "value": 64.16738033294678, "is_reference": false }, { "step": 71, "launch_seq": 72, "candidate_id": "c0070", "value": 63.89415264129639, "is_reference": false }, { "step": 72, "launch_seq": 73, "candidate_id": "c0071", "value": 61.606764793395996, "is_reference": false }, { "step": 73, "launch_seq": 74, "candidate_id": "c0072", "value": 65.7050609588623, "is_reference": false }, { "step": 74, "launch_seq": 75, "candidate_id": "c0073", "value": 62.57164478302002, "is_reference": false }, { "step": 75, "launch_seq": 76, "candidate_id": "c0074", "value": 61.29932403564453, "is_reference": false }, { "step": 76, "launch_seq": 77, "candidate_id": "c0075", "value": 65.4606819152832, "is_reference": false }, { "step": 77, "launch_seq": 78, "candidate_id": "c0076", "value": 62.28923797607422, "is_reference": false }, { "step": 79, "launch_seq": 80, "candidate_id": "c0078", "value": 62.27385997772217, "is_reference": false } ], "curve": [ { "step": 1, "value": 641.6399478912354 }, { "step": 2, "value": 484.46667194366455 }, { "step": 3, "value": 361.5250587463379 }, { "step": 4, "value": 300.16040802001953 }, { "step": 5, "value": 225.21567344665527 }, { "step": 6, "value": 223.016619682312 }, { "step": 7, "value": 219.71559524536133 }, { "step": 8, "value": 219.71559524536133 }, { "step": 9, "value": 219.71559524536133 }, { "step": 10, "value": 219.71559524536133 }, { "step": 11, "value": 219.71559524536133 }, { "step": 12, "value": 219.71559524536133 }, { "step": 13, "value": 213.57643604278564 }, { "step": 14, "value": 199.43773746490479 }, { "step": 15, "value": 199.43773746490479 }, { "step": 16, "value": 197.3193883895874 }, { "step": 17, "value": 197.3193883895874 }, { "step": 18, "value": 197.26812839508057 }, { "step": 19, "value": 197.0125436782837 }, { "step": 20, "value": 194.0772533416748 }, { "step": 21, "value": 191.72167778015137 }, { "step": 22, "value": 190.51146507263184 }, { "step": 23, "value": 190.51146507263184 }, { "step": 24, "value": 190.38987159729004 }, { "step": 25, "value": 188.5521411895752 }, { "step": 26, "value": 187.90411949157715 }, { "step": 27, "value": 167.92738437652588 }, { "step": 28, "value": 145.7422971725464 }, { "step": 29, "value": 124.23896789550781 }, { "step": 30, "value": 102.62858867645264 }, { "step": 31, "value": 102.62858867645264 }, { "step": 32, "value": 93.90711784362793 }, { "step": 33, "value": 93.90711784362793 }, { "step": 34, "value": 93.90711784362793 }, { "step": 35, "value": 93.90711784362793 }, { "step": 36, "value": 88.66989612579346 }, { "step": 37, "value": 88.66989612579346 }, { "step": 38, "value": 88.66989612579346 }, { "step": 39, "value": 86.89796924591064 }, { "step": 40, "value": 86.89796924591064 }, { "step": 41, "value": 84.46311950683594 }, { "step": 42, "value": 82.15177059173584 }, { "step": 43, "value": 81.34937286376953 }, { "step": 44, "value": 81.34937286376953 }, { "step": 45, "value": 81.34937286376953 }, { "step": 46, "value": 81.34937286376953 }, { "step": 47, "value": 81.1147689819336 }, { "step": 48, "value": 81.1147689819336 }, { "step": 49, "value": 80.98006248474121 }, { "step": 50, "value": 80.98006248474121 }, { "step": 51, "value": 79.98406887054443 }, { "step": 52, "value": 79.98406887054443 }, { "step": 53, "value": 79.94520664215088 }, { "step": 54, "value": 74.85330104827881 }, { "step": 55, "value": 73.76396656036377 }, { "step": 56, "value": 73.76396656036377 }, { "step": 57, "value": 73.76396656036377 }, { "step": 58, "value": 73.76396656036377 }, { "step": 59, "value": 73.3788013458252 }, { "step": 60, "value": 73.10605049133301 }, { "step": 61, "value": 72.9449987411499 }, { "step": 62, "value": 72.9449987411499 }, { "step": 63, "value": 72.9449987411499 }, { "step": 64, "value": 72.9449987411499 }, { "step": 65, "value": 72.91901111602783 }, { "step": 66, "value": 67.43896007537842 }, { "step": 67, "value": 66.79999828338623 }, { "step": 68, "value": 66.79999828338623 }, { "step": 69, "value": 66.79999828338623 }, { "step": 70, "value": 64.16738033294678 }, { "step": 71, "value": 63.89415264129639 }, { "step": 72, "value": 61.606764793395996 }, { "step": 73, "value": 61.606764793395996 }, { "step": 74, "value": 61.606764793395996 }, { "step": 75, "value": 61.29932403564453 }, { "step": 76, "value": 61.29932403564453 }, { "step": 77, "value": 61.29932403564453 }, { "step": 78, "value": 61.29932403564453 }, { "step": 79, "value": 61.29932403564453 }, { "step": 80, "value": 61.29932403564453 } ], "submission_sha256": "6055e89b95cd810a9558f9f7d8b7576ead7fb84a3710d1785ce18342e6f55c81", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/sequential-search/request/submission.json", "comparison_value": 61.29932403564453, "comparison_rank_key": [ 61.29932403564453, 55.83930015563965, -0.0014693673157271103 ], "rank": 3 }, { "run_id": "gear--request", "method_id": "gear", "task_id": "request", "metric": "request_ms_median", "unit": "ms", "direction": "minimize", "best_value": 97.84162044525146, "reference_value": 641.2346363067627, "relative_improvement": 0.8474168192024415, "rank_key": [ 97.84162044525146, 94.7798490524292, -0.002831657211574967 ], "best_launch_seq": 95, "best_candidate_id": "exp0107-score-once-on-best", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": null }, "evaluation_gpu_hours": 19.256951388888886, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 641.2346363067627, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp0002-compile-decode-step", "value": 519.2101001739502, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp0003-compile-both-branches", "value": 515.3186321258545, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp0004-triton-decode-attention", "value": 426.0491132736206, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp0005-head-major-cache", "value": 425.701379776001, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp0006-fused-triton-step", "value": 404.63876724243164, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp0007-one-launch-qkv", "value": 386.01577281951904, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp0008-compiled-step-split-key-attn", "value": 282.8594446182251, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp0009-attn-chain-tuned-block", "value": 422.9860305786133, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp0010-bothbranches-split-key-attn", "value": 277.4531841278076, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp0011-fixed-chunk-attn", "value": 274.91700649261475, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp0012-step-chunked-attn", "value": 276.82626247406006, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp0013-nomax-chunk-attn", "value": 273.6319303512573, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp0014-freeze-weight-casts", "value": 380.6650638580322, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp0015-gemm-autotune", "value": 258.48639011383057, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp0016-step-freeze-chunk", "value": 166.87190532684326, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp0017-eager-chunk-attn", "value": 421.9667911529541, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp0019-bf16-serving-weights", "value": 166.38195514678955, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp0020-handwritten-step-plus-attn", "value": 143.97823810577393, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp0021-step-retuned-matvecs", "value": 143.43833923339844, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp0022-window-extent-cache", "value": 422.9491949081421, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp0024-bf16-weights-no-flag", "value": 195.10197639465332, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp0025-no-pointwise-launch", "value": 140.76507091522217, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp0026-chunk16", "value": 417.37639904022217, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp0027-handwritten-step-bf16-weights", "value": 126.4580488204956, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp0028-fa3-num-splits-16", "value": 211.5941047668457, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp0029-step-chunk16", "value": 137.66789436340332, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp0030-compile-prefill-bf16", "value": 190.41979312896729, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp0031-champion-chunk16", "value": 124.3966817855835, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp0032-compiled-prefill-handwritten", "value": 134.87040996551514, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp0033-step-bf16-compiled-prefill", "value": 122.39599227905273, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp0034-frozen-split-bf16", "value": 212.54754066467285, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp0035-full-stack", "value": 119.41003799438477, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp0036-cyclic-cache-compiled", "value": 190.63711166381836, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp0037-champion-cyclic-cache", "value": 119.85659599304199, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp0038-combine-channel-split", "value": 130.16951084136963, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp0039-compiled-chunk16", "value": 189.22126293182373, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "exp0040-ring-combine-split", "value": 115.78834056854248, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp0041-combine-split-bf16", "value": 114.61937427520752, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "exp0042-compiled-combine-split", "value": 182.21497535705566, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp0043-bf16-all-params", "value": 114.05301094055176, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp0044-qkvpost-split", "value": 116.54496192932129, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp0045-widened-cast-replicate", "value": 181.7460060119629, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp0046-qkvpost-split-fp32", "value": 126.57487392425537, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp0047-widened-cast-replicate2", "value": 116.35947227478027, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp0048-compiled-stack-cyclic", "value": 181.72121047973633, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "exp0049-both-fixes-bf16", "value": 111.26601696014404, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp0050-matvec-retune", "value": 112.53345012664795, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp0051-step-on-cyclic-compiled", "value": 113.74795436859131, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp0052-fp32-matvec-retune", "value": 126.97184085845947, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "exp0053-all-ideas", "value": 106.34136199951172, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp0054-rotary-fix-on-cyclic", "value": 111.10973358154297, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp0055-cyclic-on-fp32", "value": 126.60443782806396, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp0056-combine-block16", "value": 107.06210136413574, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp0057-config-minus-head", "value": 106.39810562133789, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp0058-head-major-retest", "value": 126.41656398773193, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "exp0059-combine-block8", "value": 105.21435737609863, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "exp0060-block8-on-heldback", "value": 105.84330558776855, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "exp0061-bf16-on-headmajor", "value": 110.53001880645752, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "exp0062-per-layer-chunk", "value": 103.93357276916504, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp0063-perlayer-on-block8", "value": 104.00259494781494, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp0064-graphed-prefill", "value": 107.24055767059326, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "exp0066-head-kblock-on-perlayer", "value": 104.09784317016602, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp0067-graphed-prefill-on-champion", "value": 100.28553009033203, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "exp0068-graphed-prefill-fixed-chunk", "value": 102.67066955566406, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "exp0069-graphed-prefill-on-elite1", "value": 100.56030750274658, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "exp0071-perlayer-on-graphed-fixed", "value": 101.0218858718872, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "exp0072-embed-fold", "value": 101.02522373199463, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp0073-embed-fold-on-champion", "value": 99.79033470153809, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "exp0074-seq-fold", "value": 101.22442245483398, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "exp0076-drop-empty-chunk", "value": 98.98388385772705, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "exp0077-nopad-on-elite2", "value": 99.94816780090332, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "exp0078-nopad-on-elite1", "value": 99.47586059570312, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "exp0080-block8-on-elite2", "value": 99.2276668548584, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "exp0081-score-once", "value": 100.05605220794678, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "exp0082-embed-fold-on-elite0", "value": 99.33125972747803, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "exp0083-seq-fold-on-elite2", "value": 99.49338436126709, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "exp0084-both-folds", "value": 97.98908233642578, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "exp0085-both-folds-on-elite0", "value": 98.17337989807129, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "exp0086-degenerate-crossover-5", "value": 99.5323657989502, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "exp0087-folds-on-kblock256", "value": 98.15561771392822, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "exp0090-kblock512-on-elite0", "value": 98.40512275695801, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "exp0093-qkv-two-rows", "value": 98.5022783279419, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp0094-combine-block16", "value": 100.00896453857422, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "exp0095-qkv-two-rows-2", "value": 98.31833839416504, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "exp0098-drop-score-once", "value": 98.8006591796875, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "exp0099-proj-two-rows", "value": 99.14565086364746, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "exp0100-combine-block4", "value": 100.35133361816406, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "exp0101-qkv-four-rows", "value": 98.73509407043457, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "exp0102-qkv-rows-on-best", "value": 97.95701503753662, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "exp0103-qkv-rows-kblock256", "value": 98.13106060028076, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "exp0106-qkv-four-rows-2", "value": 98.69527816772461, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "exp0107-score-once-on-best", "value": 97.84162044525146, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "exp0110-proj-two-rows-2", "value": 98.2062816619873, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "exp0112-proj-rows-on-best", "value": 98.78194332122803, "is_reference": false }, { "step": 98, "launch_seq": 98, "candidate_id": "exp0113-combine-block16-3", "value": 98.76513481140137, "is_reference": false }, { "step": 99, "launch_seq": 99, "candidate_id": "exp0114-fc-four-rows", "value": 100.730299949646, "is_reference": false }, { "step": 100, "launch_seq": 100, "candidate_id": "exp0115-mlp-two-rows", "value": 100.16751289367676, "is_reference": false } ], "curve": [ { "step": 1, "value": 641.2346363067627 }, { "step": 2, "value": 519.2101001739502 }, { "step": 3, "value": 515.3186321258545 }, { "step": 4, "value": 426.0491132736206 }, { "step": 5, "value": 425.701379776001 }, { "step": 6, "value": 404.63876724243164 }, { "step": 7, "value": 386.01577281951904 }, { "step": 8, "value": 282.8594446182251 }, { "step": 9, "value": 282.8594446182251 }, { "step": 10, "value": 277.4531841278076 }, { "step": 11, "value": 274.91700649261475 }, { "step": 12, "value": 274.91700649261475 }, { "step": 13, "value": 273.6319303512573 }, { "step": 14, "value": 273.6319303512573 }, { "step": 15, "value": 258.48639011383057 }, { "step": 16, "value": 166.87190532684326 }, { "step": 17, "value": 166.87190532684326 }, { "step": 18, "value": 166.87190532684326 }, { "step": 19, "value": 166.38195514678955 }, { "step": 20, "value": 143.97823810577393 }, { "step": 21, "value": 143.43833923339844 }, { "step": 22, "value": 143.43833923339844 }, { "step": 23, "value": 143.43833923339844 }, { "step": 24, "value": 143.43833923339844 }, { "step": 25, "value": 140.76507091522217 }, { "step": 26, "value": 140.76507091522217 }, { "step": 27, "value": 126.4580488204956 }, { "step": 28, "value": 126.4580488204956 }, { "step": 29, "value": 126.4580488204956 }, { "step": 30, "value": 126.4580488204956 }, { "step": 31, "value": 124.3966817855835 }, { "step": 32, "value": 124.3966817855835 }, { "step": 33, "value": 122.39599227905273 }, { "step": 34, "value": 122.39599227905273 }, { "step": 35, "value": 119.41003799438477 }, { "step": 36, "value": 119.41003799438477 }, { "step": 37, "value": 119.41003799438477 }, { "step": 38, "value": 119.41003799438477 }, { "step": 39, "value": 119.41003799438477 }, { "step": 40, "value": 115.78834056854248 }, { "step": 41, "value": 114.61937427520752 }, { "step": 42, "value": 114.61937427520752 }, { "step": 43, "value": 114.05301094055176 }, { "step": 44, "value": 114.05301094055176 }, { "step": 45, "value": 114.05301094055176 }, { "step": 46, "value": 114.05301094055176 }, { "step": 47, "value": 114.05301094055176 }, { "step": 48, "value": 114.05301094055176 }, { "step": 49, "value": 111.26601696014404 }, { "step": 50, "value": 111.26601696014404 }, { "step": 51, "value": 111.26601696014404 }, { "step": 52, "value": 111.26601696014404 }, { "step": 53, "value": 106.34136199951172 }, { "step": 54, "value": 106.34136199951172 }, { "step": 55, "value": 106.34136199951172 }, { "step": 56, "value": 106.34136199951172 }, { "step": 57, "value": 106.34136199951172 }, { "step": 58, "value": 106.34136199951172 }, { "step": 59, "value": 105.21435737609863 }, { "step": 60, "value": 105.21435737609863 }, { "step": 61, "value": 105.21435737609863 }, { "step": 62, "value": 103.93357276916504 }, { "step": 63, "value": 103.93357276916504 }, { "step": 64, "value": 103.93357276916504 }, { "step": 65, "value": 103.93357276916504 }, { "step": 66, "value": 100.28553009033203 }, { "step": 67, "value": 100.28553009033203 }, { "step": 68, "value": 100.28553009033203 }, { "step": 69, "value": 100.28553009033203 }, { "step": 70, "value": 100.28553009033203 }, { "step": 71, "value": 99.79033470153809 }, { "step": 72, "value": 99.79033470153809 }, { "step": 73, "value": 98.98388385772705 }, { "step": 74, "value": 98.98388385772705 }, { "step": 75, "value": 98.98388385772705 }, { "step": 76, "value": 98.98388385772705 }, { "step": 77, "value": 98.98388385772705 }, { "step": 78, "value": 98.98388385772705 }, { "step": 79, "value": 98.98388385772705 }, { "step": 80, "value": 97.98908233642578 }, { "step": 81, "value": 97.98908233642578 }, { "step": 82, "value": 97.98908233642578 }, { "step": 83, "value": 97.98908233642578 }, { "step": 84, "value": 97.98908233642578 }, { "step": 85, "value": 97.98908233642578 }, { "step": 86, "value": 97.98908233642578 }, { "step": 87, "value": 97.98908233642578 }, { "step": 88, "value": 97.98908233642578 }, { "step": 89, "value": 97.98908233642578 }, { "step": 90, "value": 97.98908233642578 }, { "step": 91, "value": 97.98908233642578 }, { "step": 92, "value": 97.95701503753662 }, { "step": 93, "value": 97.95701503753662 }, { "step": 94, "value": 97.95701503753662 }, { "step": 95, "value": 97.84162044525146 }, { "step": 96, "value": 97.84162044525146 }, { "step": 97, "value": 97.84162044525146 }, { "step": 98, "value": 97.84162044525146 }, { "step": 99, "value": 97.84162044525146 }, { "step": 100, "value": 97.84162044525146 } ], "submission_sha256": "05767f0316968047f1466e4105ca66a4359e16c480431a24ce408005fbaff94e", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/gear/request/submission.json", "comparison_value": 97.84162044525146, "comparison_rank_key": [ 97.84162044525146, 94.7798490524292, -0.002831657211574967 ], "rank": 4 }, { "run_id": "heuresis-map-elites--request", "method_id": "heuresis-map-elites", "task_id": "request", "metric": "request_ms_median", "unit": "ms", "direction": "minimize", "best_value": 129.0278434753418, "reference_value": 640.6753063201904, "relative_improvement": 0.7986064981707637, "rank_key": [ 129.0278434753418, 124.49479103088379, -0.0008235909685170295 ], "best_launch_seq": 80, "best_candidate_id": "exp-f6d2a334c8aecd2327753401", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 14.875653611111114, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 640.6753063201904, "is_reference": true }, { "step": 3, "launch_seq": 3, "candidate_id": "exp-041c53907ef1060da6666649", "value": 398.52404594421387, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-4755820f49b649bccfa9f9bf", "value": 520.0506448745728, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-d4ec0526e41d0dd533b288fa", "value": 438.8371706008911, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-0579cf0e4a078f313e847f6c", "value": 361.2734079360962, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-06eb5ac2937bfc7b012a6097", "value": 297.27697372436523, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-64cc52eb12078dc0b02a8ebc", "value": 217.09132194519043, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-755592416976eda049b06954", "value": 539.9911403656006, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-d1c1e557c19f138668c1e969", "value": 331.9070339202881, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-c9a27f5e332bc4bf3a3d645e", "value": 242.57731437683105, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-d2764153f25729b897236ae0", "value": 203.48632335662842, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp-6c9771d8013458d7d569aef1", "value": 204.1466236114502, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp-b90cae40440d2b2699755151", "value": 193.22550296783447, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp-43edee18ca9e9dc4ce431ae7", "value": 186.40649318695068, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-d66947a172c7208eff26f5ae", "value": 184.54909324645996, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp-b47bff229831838362f508bf", "value": 419.95203495025635, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp-3fcd807ab710a2e2aa57289b", "value": 408.26714038848877, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-a26bb30c5008acfc411f7218", "value": 390.89787006378174, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-7d400508226aaa80c71bbf65", "value": 333.0552577972412, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp-73146ecacf4f34f075b86b70", "value": 488.65020275115967, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp-8556ed08daf5487b412e0658", "value": 566.6784048080444, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp-ea8ee383dbfb08dfd297e9d9", "value": 419.65460777282715, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp-fc326c53eac21f8d869e40e3", "value": 393.33808422088623, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp-4fe143b39ae6c48e254078d1", "value": 407.03511238098145, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp-76eb324afed288226470d7d4", "value": 394.00696754455566, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp-4c517034b38acdd15e85123f", "value": 487.8988265991211, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp-325aaff9bc4e2f89e9824dab", "value": 393.2241201400757, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp-28ed60cf1e792aa9c9f1a607", "value": 442.5619840621948, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp-673700ff27b3803f015e7830", "value": 345.46446800231934, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp-35ac50bcfc54ee2ac07dd8ea", "value": 302.40893363952637, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "exp-8d106560f398a6861c933c57", "value": 342.4121141433716, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "exp-810217614a7500034e418f5b", "value": 212.6561403274536, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp-dfd983ee53feb695c883c9dd", "value": 433.20608139038086, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp-4c77909690e63502a7896421", "value": 371.4408874511719, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp-839932d5c3bd7fdd6ed4720e", "value": 450.8093595504761, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "exp-a3592ca7514f3ee5a6d3d81b", "value": 526.6820192337036, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "exp-b16c93f2ba0af07328fe48f0", "value": 391.7044401168823, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp-6f13330d18845672fe3209ee", "value": 360.75782775878906, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "exp-cd171cd947f5f9cf8ed855c5", "value": 429.94189262390137, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "exp-4c1f481375c17f9a378ec18f", "value": 482.5037717819214, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "exp-57190a42baa74268e1d40b6c", "value": 381.6782236099243, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "exp-62a85e1285da5c5a82159d14", "value": 154.8614501953125, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "exp-857f1986f4091f4b2529dc98", "value": 129.14180755615234, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "exp-f6d2a334c8aecd2327753401", "value": 129.0278434753418, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "exp-dbe9fbc557a77440ae44f199", "value": 148.51689338684082, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "exp-e97a6b3f3dda9288872c338c", "value": 187.05201148986816, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "exp-b9f5797ee5da09c58ca461b5", "value": 178.10475826263428, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "exp-f56c6f8ac16736ab7bc3a9a6", "value": 155.76231479644775, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "exp-39b17384725131ae2780c47b", "value": 153.57840061187744, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "exp-3cc52747185f88f21d3cac82", "value": 172.88076877593994, "is_reference": false } ], "curve": [ { "step": 1, "value": 640.6753063201904 }, { "step": 2, "value": 640.6753063201904 }, { "step": 3, "value": 398.52404594421387 }, { "step": 4, "value": 398.52404594421387 }, { "step": 5, "value": 398.52404594421387 }, { "step": 6, "value": 361.2734079360962 }, { "step": 7, "value": 297.27697372436523 }, { "step": 8, "value": 297.27697372436523 }, { "step": 9, "value": 297.27697372436523 }, { "step": 10, "value": 217.09132194519043 }, { "step": 11, "value": 217.09132194519043 }, { "step": 12, "value": 217.09132194519043 }, { "step": 13, "value": 217.09132194519043 }, { "step": 14, "value": 217.09132194519043 }, { "step": 15, "value": 217.09132194519043 }, { "step": 16, "value": 217.09132194519043 }, { "step": 17, "value": 217.09132194519043 }, { "step": 18, "value": 217.09132194519043 }, { "step": 19, "value": 217.09132194519043 }, { "step": 20, "value": 217.09132194519043 }, { "step": 21, "value": 203.48632335662842 }, { "step": 22, "value": 203.48632335662842 }, { "step": 23, "value": 203.48632335662842 }, { "step": 24, "value": 193.22550296783447 }, { "step": 25, "value": 186.40649318695068 }, { "step": 26, "value": 184.54909324645996 }, { "step": 27, "value": 184.54909324645996 }, { "step": 28, "value": 184.54909324645996 }, { "step": 29, "value": 184.54909324645996 }, { "step": 30, "value": 184.54909324645996 }, { "step": 31, "value": 184.54909324645996 }, { "step": 32, "value": 184.54909324645996 }, { "step": 33, "value": 184.54909324645996 }, { "step": 34, "value": 184.54909324645996 }, { "step": 35, "value": 184.54909324645996 }, { "step": 36, "value": 184.54909324645996 }, { "step": 37, "value": 184.54909324645996 }, { "step": 38, "value": 184.54909324645996 }, { "step": 39, "value": 184.54909324645996 }, { "step": 40, "value": 184.54909324645996 }, { "step": 41, "value": 184.54909324645996 }, { "step": 42, "value": 184.54909324645996 }, { "step": 43, "value": 184.54909324645996 }, { "step": 44, "value": 184.54909324645996 }, { "step": 45, "value": 184.54909324645996 }, { "step": 46, "value": 184.54909324645996 }, { "step": 47, "value": 184.54909324645996 }, { "step": 48, "value": 184.54909324645996 }, { "step": 49, "value": 184.54909324645996 }, { "step": 50, "value": 184.54909324645996 }, { "step": 51, "value": 184.54909324645996 }, { "step": 52, "value": 184.54909324645996 }, { "step": 53, "value": 184.54909324645996 }, { "step": 54, "value": 184.54909324645996 }, { "step": 55, "value": 184.54909324645996 }, { "step": 56, "value": 184.54909324645996 }, { "step": 57, "value": 184.54909324645996 }, { "step": 58, "value": 184.54909324645996 }, { "step": 59, "value": 184.54909324645996 }, { "step": 60, "value": 184.54909324645996 }, { "step": 61, "value": 184.54909324645996 }, { "step": 62, "value": 184.54909324645996 }, { "step": 63, "value": 184.54909324645996 }, { "step": 64, "value": 184.54909324645996 }, { "step": 65, "value": 184.54909324645996 }, { "step": 66, "value": 184.54909324645996 }, { "step": 67, "value": 184.54909324645996 }, { "step": 68, "value": 184.54909324645996 }, { "step": 69, "value": 184.54909324645996 }, { "step": 70, "value": 184.54909324645996 }, { "step": 71, "value": 184.54909324645996 }, { "step": 72, "value": 184.54909324645996 }, { "step": 73, "value": 184.54909324645996 }, { "step": 74, "value": 184.54909324645996 }, { "step": 75, "value": 154.8614501953125 }, { "step": 76, "value": 154.8614501953125 }, { "step": 77, "value": 129.14180755615234 }, { "step": 78, "value": 129.14180755615234 }, { "step": 79, "value": 129.14180755615234 }, { "step": 80, "value": 129.0278434753418 }, { "step": 81, "value": 129.0278434753418 }, { "step": 82, "value": 129.0278434753418 }, { "step": 83, "value": 129.0278434753418 }, { "step": 84, "value": 129.0278434753418 }, { "step": 85, "value": 129.0278434753418 }, { "step": 86, "value": 129.0278434753418 }, { "step": 87, "value": 129.0278434753418 }, { "step": 88, "value": 129.0278434753418 }, { "step": 89, "value": 129.0278434753418 }, { "step": 90, "value": 129.0278434753418 }, { "step": 91, "value": 129.0278434753418 }, { "step": 92, "value": 129.0278434753418 }, { "step": 93, "value": 129.0278434753418 }, { "step": 94, "value": 129.0278434753418 }, { "step": 95, "value": 129.0278434753418 }, { "step": 96, "value": 129.0278434753418 }, { "step": 97, "value": 129.0278434753418 }, { "step": 98, "value": 129.0278434753418 }, { "step": 99, "value": 129.0278434753418 }, { "step": 100, "value": 129.0278434753418 } ], "submission_sha256": "7a964ccfc1f9e6473871664567f80fadbb24c41e5f464b196fe0dbabcb54d2a1", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/heuresis-map-elites/request/submission.json", "comparison_value": 129.0278434753418, "comparison_rank_key": [ 129.0278434753418, 124.49479103088379, -0.0008235909685170295 ], "rank": 5 }, { "run_id": "beam-search--request", "method_id": "beam-search", "task_id": "request", "metric": "request_ms_median", "unit": "ms", "direction": "minimize", "best_value": 180.55975437164307, "reference_value": 641.5977478027344, "relative_improvement": 0.7185779485822665, "rank_key": [ 180.55975437164307, 173.55501651763916, -0.0020740887594818957 ], "best_launch_seq": 74, "best_candidate_id": "g11c1-two-step-neighbourhood-on-the-flagless-host", "best_is_reference": false, "budget": { "spent": 97, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 4, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 4 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 21.878128611111116, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 641.5977478027344, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "g01c1-compiled-decode-step", "value": 520.0583934783936, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "g01c2-window-extent-ring-cache", "value": 632.3821544647217, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "g02c1-compiled-prefill-pass", "value": 515.8671140670776, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "g02c2-bf16-serving-weight-copies", "value": 412.87779808044434, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "g02c3-packed-input-projection", "value": 600.5210876464844, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "g02c4-plain-attention-on-full-ring", "value": 639.2555236816406, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "g02c5-kvcache-append-prefill", "value": 639.3774747848511, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "g02c6-minimal-width1-masks", "value": 644.389271736145, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "g03c1-packed-projection-bf16-serving", "value": 386.48974895477295, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "g03c2-float32-serving-weight-copies", "value": 517.676830291748, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "g03c3-prefill-final-row-last-block", "value": 515.0432586669922, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "g03c4-window-extent-ring-in-compiled-step", "value": 482.77366161346436, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "g03c5-int8-serving-weights", "value": 496.57750129699707, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "g03c6-lm-head-only-bf16-serving-copy", "value": 513.0351781845093, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "g03c7-unpinned-kvcache-splits", "value": 466.93146228790283, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "g03c8-compiled-decode-step-on-ring", "value": 435.2707862854004, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "g04c1-derived-key-split-count", "value": 210.12604236602783, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "g04c2-window-extent-ring-on-champion", "value": 361.6231679916382, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "g04c3-window-range-cache-leftpad", "value": 413.4413003921509, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "g04c4-kernel-fused-rotary", "value": 431.64026737213135, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "g04c5-bf16-packed-serving-weights-on-ring", "value": 360.64910888671875, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "g04c6-torch-native-ring-attention", "value": 342.0882225036621, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "g04c7-compiled-step-around-eager-attention", "value": 319.5302486419678, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "g04c8-joint-qk-rotary-and-norm", "value": 412.9316806793213, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "g05c1-key-extent-bounded-split-count", "value": 210.70170402526855, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "g05c2-transposed-serving-weight-operands", "value": 214.8057222366333, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "g05c3-bf16-serving-weights-on-cut-region", "value": 248.30782413482666, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "g05c4-packed-decode-embedding-gather", "value": 310.3598356246948, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "g05c5-tensor-attention-on-kernel-layers", "value": 276.3350009918213, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "g05c6-cache-layout-for-tensor-attention", "value": 338.77289295196533, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "g05c7-derived-split-count-on-ring", "value": 210.63411235809326, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "g05c8-tensor-attention-on-bf16-ring", "value": 266.53921604156494, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "g06c1-tensor-attention-on-uniform-cache", "value": 229.77614402770996, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "g06c2-joint-qk-rotary-and-norm-on-champion", "value": 225.58999061584473, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "g06c3-joint-qk-rotary-and-norm-on-extent-bounded", "value": 225.90816020965576, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "g06c4-packed-decode-embedding-gather-on-extent-bounded", "value": 215.12460708618164, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "g06c5-tensor-attention-on-transposed-operands", "value": 227.46503353118896, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "g06c6-packed-decode-embedding-gather-on-transposed-operands", "value": 216.91596508026123, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "g06c7-tensor-attention-on-ring-with-split-count", "value": 215.1179313659668, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "g06c8-joint-qk-rotary-and-norm-on-ring", "value": 226.48024559020996, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "g07c1-autotuned-gemm-lowering-on-champion", "value": 208.77647399902344, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "g07c2-contiguous-serving-weight-arena", "value": 216.98856353759766, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "g07c3-autotuned-gemm-lowering-on-extent-bounded", "value": 209.570050239563, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "g07c4-window-aligned-cache-leftpad-at-split-count", "value": 219.96748447418213, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "g07c5-serving-weight-arena-on-transposed-operands", "value": 215.74342250823975, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "g07c6-coordinate-descent-tuned-step-kernels", "value": 225.90374946594238, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "g07c7-ring-only-tensor-attention", "value": 215.1663303375244, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "g07c8-coordinate-descent-tuning-on-ring", "value": 186.25915050506592, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "g08c1-coordinate-descent-tuning-on-uniform-cache", "value": 189.68665599822998, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "g08c2-combo-kernel-horizontal-fusion", "value": 216.74227714538574, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "g08c3-unrolled-small-reductions-on-champion-lineage", "value": 317.1943426132202, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "g08c4-both-lowering-searches-on-one-region", "value": 186.2698793411255, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "g08c5-unrolled-small-reductions-on-extent-bounded", "value": 316.64586067199707, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "g08c6-constant-folded-residual-mix-scalars", "value": 211.39979362487793, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "g08c7-coordinate-descent-all-directions-on-champion", "value": 188.95328044891357, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "g09c2-champion-body-priced-in-its-own-launch-a2", "value": 186.92505359649658, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "g09c1-pointwise-autotune-start-config-on-champion-a2", "value": 185.70208549499512, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "g09c3-aten-only-gemm-backend-on-composition-a2", "value": 186.0809326171875, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "g09c4-per-reduction-form-chosen-by-measurement-a2", "value": 188.2380247116089, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "g09c5-cooperative-single-row-reductions-a2", "value": 216.0356044769287, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "g09c6-inactive-window-bound-dropped-at-step-call-a2", "value": 186.82527542114258, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "g09c7-wider-neighbourhood-where-the-target-reads-a2", "value": 183.83359909057617, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "g09c8-gemm-template-search-on-the-ring-host-a2", "value": 185.4228973388672, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "g10c1-search-radius-two-on-all-four-specialisations", "value": 183.5242509841919, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "g10c3-single-block-reductions-without-the-split", "value": 185.73284149169922, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "g10c4-spilling-configs-priced-not-rejected", "value": 186.97357177734375, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "g10c5-prefill-last-block-tail-on-the-final-row", "value": 185.31668186187744, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "g10c6-fusion-boundaries-decided-by-benchmark", "value": 188.9667510986328, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "g10c7-two-step-neighbourhood-on-the-uniform-host", "value": 183.38537216186523, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "g10c8-bfloat16-codegen-without-the-fp32-round-trip", "value": 189.40353393554688, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "g11c1-two-step-neighbourhood-on-the-flagless-host", "value": 180.55975437164307, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "g11c9-scored-search-without-a-stored-config", "value": 187.73365020751953, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "g11c3-faster-of-two-independent-searches", "value": 186.71298027038574, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "g11c4-max-autotune-as-one-named-set", "value": 188.69590759277344, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "g11c5-radius-two-only-where-the-ranked-key-reads", "value": 184.55779552459717, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "g11c6-radius-two-only-where-the-tiebreak-reads", "value": 185.19210815429688, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "g11c7-three-step-neighbourhood-at-one-specialisation", "value": 183.3508014678955, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "g11c8-generated-kernels-without-the-index-asserts", "value": 185.03355979919434, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "g12c1-the-two-step-neighbourhood-applied-to-every-kernel", "value": 182.39688873291016, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "g12c3-two-step-neighbourhood-beneath-the-three-step-one", "value": 186.76233291625977, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "g12c5-gemm-products-without-a-stored-configuration", "value": 187.27576732635498, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "g12c7-descent-from-the-heuristic-configuration", "value": 182.88230895996094, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "g12c8-the-ranked-lowering-without-a-stored-configuration", "value": 186.5941286087036, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "g13c1-the-stored-configuration-priced-in-one-launch", "value": 182.7157735824585, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "g13c2-the-search-paid-for-and-its-answers-discarded", "value": 181.91087245941162, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "g13c3-the-region-lowered-again-with-the-store-untouched", "value": 184.83948707580566, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "g13c4-the-stored-configuration-denied-and-nothing-else", "value": 183.9379072189331, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "g13c5-the-heuristic-start-read-from-its-own-stored-answers", "value": 183.6080551147461, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "g13c6-the-heuristic-start-without-a-stored-configuration", "value": 183.11333656311035, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "g13c7-the-third-step-removed-and-the-searched-lowering-given-an-error-bar", "value": 184.2215061187744, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "g13c8-the-narrowest-search-bought-instead-of-the-store", "value": 188.02642822265625, "is_reference": false } ], "curve": [ { "step": 1, "value": 641.5977478027344 }, { "step": 2, "value": 520.0583934783936 }, { "step": 3, "value": 520.0583934783936 }, { "step": 4, "value": 515.8671140670776 }, { "step": 5, "value": 412.87779808044434 }, { "step": 6, "value": 412.87779808044434 }, { "step": 7, "value": 412.87779808044434 }, { "step": 8, "value": 412.87779808044434 }, { "step": 9, "value": 412.87779808044434 }, { "step": 10, "value": 386.48974895477295 }, { "step": 11, "value": 386.48974895477295 }, { "step": 12, "value": 386.48974895477295 }, { "step": 13, "value": 386.48974895477295 }, { "step": 14, "value": 386.48974895477295 }, { "step": 15, "value": 386.48974895477295 }, { "step": 16, "value": 386.48974895477295 }, { "step": 17, "value": 386.48974895477295 }, { "step": 18, "value": 210.12604236602783 }, { "step": 19, "value": 210.12604236602783 }, { "step": 20, "value": 210.12604236602783 }, { "step": 21, "value": 210.12604236602783 }, { "step": 22, "value": 210.12604236602783 }, { "step": 23, "value": 210.12604236602783 }, { "step": 24, "value": 210.12604236602783 }, { "step": 25, "value": 210.12604236602783 }, { "step": 26, "value": 210.12604236602783 }, { "step": 27, "value": 210.12604236602783 }, { "step": 28, "value": 210.12604236602783 }, { "step": 29, "value": 210.12604236602783 }, { "step": 30, "value": 210.12604236602783 }, { "step": 31, "value": 210.12604236602783 }, { "step": 32, "value": 210.12604236602783 }, { "step": 33, "value": 210.12604236602783 }, { "step": 34, "value": 210.12604236602783 }, { "step": 35, "value": 210.12604236602783 }, { "step": 36, "value": 210.12604236602783 }, { "step": 37, "value": 210.12604236602783 }, { "step": 38, "value": 210.12604236602783 }, { "step": 39, "value": 210.12604236602783 }, { "step": 40, "value": 210.12604236602783 }, { "step": 41, "value": 210.12604236602783 }, { "step": 42, "value": 208.77647399902344 }, { "step": 43, "value": 208.77647399902344 }, { "step": 44, "value": 208.77647399902344 }, { "step": 45, "value": 208.77647399902344 }, { "step": 46, "value": 208.77647399902344 }, { "step": 47, "value": 208.77647399902344 }, { "step": 48, "value": 208.77647399902344 }, { "step": 49, "value": 186.25915050506592 }, { "step": 50, "value": 186.25915050506592 }, { "step": 51, "value": 186.25915050506592 }, { "step": 52, "value": 186.25915050506592 }, { "step": 53, "value": 186.25915050506592 }, { "step": 54, "value": 186.25915050506592 }, { "step": 55, "value": 186.25915050506592 }, { "step": 56, "value": 186.25915050506592 }, { "step": 57, "value": 186.25915050506592 }, { "step": 58, "value": 186.25915050506592 }, { "step": 59, "value": 185.70208549499512 }, { "step": 60, "value": 185.70208549499512 }, { "step": 61, "value": 185.70208549499512 }, { "step": 62, "value": 185.70208549499512 }, { "step": 63, "value": 185.70208549499512 }, { "step": 64, "value": 183.83359909057617 }, { "step": 65, "value": 183.83359909057617 }, { "step": 66, "value": 183.5242509841919 }, { "step": 67, "value": 183.5242509841919 }, { "step": 68, "value": 183.5242509841919 }, { "step": 69, "value": 183.5242509841919 }, { "step": 70, "value": 183.5242509841919 }, { "step": 71, "value": 183.5242509841919 }, { "step": 72, "value": 183.38537216186523 }, { "step": 73, "value": 183.38537216186523 }, { "step": 74, "value": 180.55975437164307 }, { "step": 75, "value": 180.55975437164307 }, { "step": 76, "value": 180.55975437164307 }, { "step": 77, "value": 180.55975437164307 }, { "step": 78, "value": 180.55975437164307 }, { "step": 79, "value": 180.55975437164307 }, { "step": 80, "value": 180.55975437164307 }, { "step": 81, "value": 180.55975437164307 }, { "step": 82, "value": 180.55975437164307 }, { "step": 83, "value": 180.55975437164307 }, { "step": 84, "value": 180.55975437164307 }, { "step": 85, "value": 180.55975437164307 }, { "step": 86, "value": 180.55975437164307 }, { "step": 87, "value": 180.55975437164307 }, { "step": 88, "value": 180.55975437164307 }, { "step": 89, "value": 180.55975437164307 }, { "step": 90, "value": 180.55975437164307 }, { "step": 91, "value": 180.55975437164307 }, { "step": 92, "value": 180.55975437164307 }, { "step": 93, "value": 180.55975437164307 }, { "step": 94, "value": 180.55975437164307 }, { "step": 95, "value": 180.55975437164307 }, { "step": 96, "value": 180.55975437164307 }, { "step": 97, "value": 180.55975437164307 } ], "submission_sha256": "4c05fe17e55d4e2f0f639bd37dd3dffe23b6190b064da66cf4a5e2337e75fe90", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/beam-search/request/submission.json", "comparison_value": 180.55975437164307, "comparison_rank_key": [ 180.55975437164307, 173.55501651763916, -0.0020740887594818957 ], "rank": 6 }, { "run_id": "tpe--request", "method_id": "tpe", "task_id": "request", "metric": "request_ms_median", "unit": "ms", "direction": "minimize", "best_value": 295.38750648498535, "reference_value": 654.543399810791, "relative_improvement": 0.5487121150860691, "rank_key": [ 295.38750648498535, 237.19584941864014, -0.001687077259002967 ], "best_launch_seq": 45, "best_candidate_id": "tpe-trial-0044", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 17.422812222222213, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 654.543399810791, "is_reference": true }, { "step": 14, "launch_seq": 14, "candidate_id": "tpe-trial-0013", "value": 696.8328952789307, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "tpe-trial-0014", "value": 769.4988250732422, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "tpe-trial-0015", "value": 686.7495775222778, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "tpe-trial-0021", "value": 697.0129013061523, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "tpe-trial-0022", "value": 612.0502948760986, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "tpe-trial-0023", "value": 543.3496236801147, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "tpe-trial-0024", "value": 463.18137645721436, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "tpe-trial-0029", "value": 643.781304359436, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "tpe-trial-0031", "value": 644.1733837127686, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "tpe-trial-0032", "value": 544.8399782180786, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "tpe-trial-0034", "value": 574.4146108627319, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "tpe-trial-0039", "value": 374.06039237976074, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "tpe-trial-0040", "value": 377.8839111328125, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "tpe-trial-0041", "value": 380.10549545288086, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "tpe-trial-0042", "value": 379.6805143356323, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "tpe-trial-0044", "value": 295.38750648498535, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "tpe-trial-0046", "value": 375.3318786621094, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "tpe-trial-0049", "value": 375.37121772766113, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "tpe-trial-0051", "value": 376.4299154281616, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "tpe-trial-0052", "value": 442.44396686553955, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "tpe-trial-0053", "value": 377.87485122680664, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "tpe-trial-0058", "value": 377.1395683288574, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "tpe-trial-0060", "value": 452.150821685791, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "tpe-trial-0063", "value": 375.6997585296631, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "tpe-trial-0066", "value": 449.770450592041, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "tpe-trial-0069", "value": 375.5568265914917, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "tpe-trial-0070", "value": 442.665696144104, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "tpe-trial-0071", "value": 375.7462501525879, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "tpe-trial-0073", "value": 376.89507007598877, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "tpe-trial-0076", "value": 459.1052532196045, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "tpe-trial-0077", "value": 431.1494827270508, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "tpe-trial-0081", "value": 375.82623958587646, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "tpe-trial-0082", "value": 356.78231716156006, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "tpe-trial-0083", "value": 506.5532922744751, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "tpe-trial-0085", "value": 375.7188320159912, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "tpe-trial-0091", "value": 377.3963451385498, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "tpe-trial-0092", "value": 375.7389783859253, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "tpe-trial-0096", "value": 376.1310577392578, "is_reference": false }, { "step": 98, "launch_seq": 98, "candidate_id": "tpe-trial-0097", "value": 463.0211591720581, "is_reference": false }, { "step": 99, "launch_seq": 99, "candidate_id": "tpe-trial-0098", "value": 370.4582452774048, "is_reference": false }, { "step": 100, "launch_seq": 100, "candidate_id": "tpe-trial-0099", "value": 515.7186985015869, "is_reference": false } ], "curve": [ { "step": 1, "value": 654.543399810791 }, { "step": 2, "value": 654.543399810791 }, { "step": 3, "value": 654.543399810791 }, { "step": 4, "value": 654.543399810791 }, { "step": 5, "value": 654.543399810791 }, { "step": 6, "value": 654.543399810791 }, { "step": 7, "value": 654.543399810791 }, { "step": 8, "value": 654.543399810791 }, { "step": 9, "value": 654.543399810791 }, { "step": 10, "value": 654.543399810791 }, { "step": 11, "value": 654.543399810791 }, { "step": 12, "value": 654.543399810791 }, { "step": 13, "value": 654.543399810791 }, { "step": 14, "value": 654.543399810791 }, { "step": 15, "value": 654.543399810791 }, { "step": 16, "value": 654.543399810791 }, { "step": 17, "value": 654.543399810791 }, { "step": 18, "value": 654.543399810791 }, { "step": 19, "value": 654.543399810791 }, { "step": 20, "value": 654.543399810791 }, { "step": 21, "value": 654.543399810791 }, { "step": 22, "value": 654.543399810791 }, { "step": 23, "value": 612.0502948760986 }, { "step": 24, "value": 543.3496236801147 }, { "step": 25, "value": 463.18137645721436 }, { "step": 26, "value": 463.18137645721436 }, { "step": 27, "value": 463.18137645721436 }, { "step": 28, "value": 463.18137645721436 }, { "step": 29, "value": 463.18137645721436 }, { "step": 30, "value": 463.18137645721436 }, { "step": 31, "value": 463.18137645721436 }, { "step": 32, "value": 463.18137645721436 }, { "step": 33, "value": 463.18137645721436 }, { "step": 34, "value": 463.18137645721436 }, { "step": 35, "value": 463.18137645721436 }, { "step": 36, "value": 463.18137645721436 }, { "step": 37, "value": 463.18137645721436 }, { "step": 38, "value": 463.18137645721436 }, { "step": 39, "value": 463.18137645721436 }, { "step": 40, "value": 374.06039237976074 }, { "step": 41, "value": 374.06039237976074 }, { "step": 42, "value": 374.06039237976074 }, { "step": 43, "value": 374.06039237976074 }, { "step": 44, "value": 374.06039237976074 }, { "step": 45, "value": 295.38750648498535 }, { "step": 46, "value": 295.38750648498535 }, { "step": 47, "value": 295.38750648498535 }, { "step": 48, "value": 295.38750648498535 }, { "step": 49, "value": 295.38750648498535 }, { "step": 50, "value": 295.38750648498535 }, { "step": 51, "value": 295.38750648498535 }, { "step": 52, "value": 295.38750648498535 }, { "step": 53, "value": 295.38750648498535 }, { "step": 54, "value": 295.38750648498535 }, { "step": 55, "value": 295.38750648498535 }, { "step": 56, "value": 295.38750648498535 }, { "step": 57, "value": 295.38750648498535 }, { "step": 58, "value": 295.38750648498535 }, { "step": 59, "value": 295.38750648498535 }, { "step": 60, "value": 295.38750648498535 }, { "step": 61, "value": 295.38750648498535 }, { "step": 62, "value": 295.38750648498535 }, { "step": 63, "value": 295.38750648498535 }, { "step": 64, "value": 295.38750648498535 }, { "step": 65, "value": 295.38750648498535 }, { "step": 66, "value": 295.38750648498535 }, { "step": 67, "value": 295.38750648498535 }, { "step": 68, "value": 295.38750648498535 }, { "step": 69, "value": 295.38750648498535 }, { "step": 70, "value": 295.38750648498535 }, { "step": 71, "value": 295.38750648498535 }, { "step": 72, "value": 295.38750648498535 }, { "step": 73, "value": 295.38750648498535 }, { "step": 74, "value": 295.38750648498535 }, { "step": 75, "value": 295.38750648498535 }, { "step": 76, "value": 295.38750648498535 }, { "step": 77, "value": 295.38750648498535 }, { "step": 78, "value": 295.38750648498535 }, { "step": 79, "value": 295.38750648498535 }, { "step": 80, "value": 295.38750648498535 }, { "step": 81, "value": 295.38750648498535 }, { "step": 82, "value": 295.38750648498535 }, { "step": 83, "value": 295.38750648498535 }, { "step": 84, "value": 295.38750648498535 }, { "step": 85, "value": 295.38750648498535 }, { "step": 86, "value": 295.38750648498535 }, { "step": 87, "value": 295.38750648498535 }, { "step": 88, "value": 295.38750648498535 }, { "step": 89, "value": 295.38750648498535 }, { "step": 90, "value": 295.38750648498535 }, { "step": 91, "value": 295.38750648498535 }, { "step": 92, "value": 295.38750648498535 }, { "step": 93, "value": 295.38750648498535 }, { "step": 94, "value": 295.38750648498535 }, { "step": 95, "value": 295.38750648498535 }, { "step": 96, "value": 295.38750648498535 }, { "step": 97, "value": 295.38750648498535 }, { "step": 98, "value": 295.38750648498535 }, { "step": 99, "value": 295.38750648498535 }, { "step": 100, "value": 295.38750648498535 } ], "submission_sha256": "d309e1e8edaba3e4a9cc2d36ed80d6f81352d71e83379d0fed9fb1e986398439", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/tpe/request/submission.json", "comparison_value": 295.38750648498535, "comparison_rank_key": [ 295.38750648498535, 237.19584941864014, -0.001687077259002967 ], "rank": 7 } ] }, { "id": "steps", "title": "Optimizer updates to quality at validation BPB < 1.05", "metric": "updates_to_target", "unit": "optimizer steps", "direction": "minimize", "budget": 100, "comparison_step": null, "quality_gate": { "metric": "val_bpb", "operator": "<", "value": 1.05 }, "tiebreaks": [ { "metric": "tokens_to_target", "direction": "minimize" } ], "runs": [ { "run_id": "autoscientists--steps", "method_id": "autoscientists", "task_id": "steps", "metric": "updates_to_target", "unit": "optimizer steps", "direction": "minimize", "best_value": 267, "reference_value": 748, "relative_improvement": 0.643048128342246, "rank_key": [ 267.0, 404488192.0, -0.00013586089970085702 ], "best_launch_seq": 89, "best_candidate_id": "exp-563046e7a290f0d19f0034dd", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 2, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 2 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": null }, "evaluation_gpu_hours": 30.315821111111116, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 748, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-3a9e55dbfdfbf43e79412c3f", "value": 563, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp-5d26471e2c697c3447a1d918", "value": 564, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-a3d6eefe753044a0c744ab78", "value": 675, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-b3d1098e98f2f44008ab63c1", "value": 564, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-4e58c10e82ea997f5bcf0c5d", "value": 564, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-22c042ea5f278ff61944a53c", "value": 405, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp-d4caf729007c28e7b8b7d144", "value": 564, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp-8e3afc132f8fa48b79ed380a", "value": 344, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-979c48bac27e7ac493093f44", "value": 307, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-173028020a8b0b39ee683e29", "value": 307, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-ab6bcb64978b78690ebe55e7", "value": 307, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-58d8b521783ac6e6c3d088ec", "value": 293, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-2b3c25170cd761661697d638", "value": 293, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-8451bfa7dff42da13e8ae108", "value": 293, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-9a3bbbc75de6bbcb5f49bda3", "value": 308, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-57808fbd33f5bb98960c1fa0", "value": 294, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp-8fa3843684548cde620697f4", "value": 308, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp-909baa1fd4ff17ad3c377586", "value": 284, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-da0689a1b55ee813a1fa36e3", "value": 281, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-e21d21659896a09dd198d0e6", "value": 284, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-abfed572a5062407f779304f", "value": 284, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-00c02c6a81cda6c8c0dd1c89", "value": 278, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp-5b59439e11047659f77645c1", "value": 276, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "exp-abad846334070289e0f5f31a", "value": 276, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp-1ea99be457b1fa24622eaade", "value": 282, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "exp-d6436681e4aa50b7e76d32a2", "value": 275, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp-bd99e9e207245343caf7a7a3", "value": 281, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp-a91b4303c509edf853876e5c", "value": 275, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp-26e22b6b7e65e71239d86c7c", "value": 281, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "exp-1072a8cb3d9869e1f973d26c", "value": 274, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "exp-63d65b2700a20b6ac3d56b7d", "value": 274, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp-1554205d1de841c7965be639", "value": 274, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "exp-8ff343de6a8dc4b4cc446ceb", "value": 273, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "exp-51b703dc0492b78338dd4c98", "value": 272, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp-bdb28f629eee4c3450687e38", "value": 272, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "exp-9dac42ebb25b51583b0fd901", "value": 273, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "exp-672c11f16ebac01d3d9e873e", "value": 270, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "exp-ea047c98df0cdbb79aa4f57d", "value": 271, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "exp-003bab1cbd01045b7cd3cee3", "value": 271, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "exp-a8b5317039a1bddba60cc8ab", "value": 272, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "exp-2d17d73486bd42106fe1e323", "value": 269, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "exp-7221d13cc37574c3f3ac1e99", "value": 270, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp-8df74410048c781f53fd6e47", "value": 269, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "exp-563046e7a290f0d19f0034dd", "value": 267, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "exp-6013ae9518f21aee1af5f4b3", "value": 269, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "exp-56e3f8460fd94c5340f4f362", "value": 268, "is_reference": false }, { "step": 98, "launch_seq": 98, "candidate_id": "exp-b9c82270b445849f2d2cb980", "value": 268, "is_reference": false } ], "curve": [ { "step": 1, "value": 748 }, { "step": 2, "value": 563 }, { "step": 3, "value": 563 }, { "step": 4, "value": 563 }, { "step": 5, "value": 563 }, { "step": 6, "value": 563 }, { "step": 7, "value": 405 }, { "step": 8, "value": 405 }, { "step": 9, "value": 344 }, { "step": 10, "value": 307 }, { "step": 11, "value": 307 }, { "step": 12, "value": 307 }, { "step": 13, "value": 307 }, { "step": 14, "value": 307 }, { "step": 15, "value": 293 }, { "step": 16, "value": 293 }, { "step": 17, "value": 293 }, { "step": 18, "value": 293 }, { "step": 19, "value": 293 }, { "step": 20, "value": 293 }, { "step": 21, "value": 293 }, { "step": 22, "value": 293 }, { "step": 23, "value": 293 }, { "step": 24, "value": 293 }, { "step": 25, "value": 293 }, { "step": 26, "value": 293 }, { "step": 27, "value": 293 }, { "step": 28, "value": 284 }, { "step": 29, "value": 284 }, { "step": 30, "value": 284 }, { "step": 31, "value": 284 }, { "step": 32, "value": 284 }, { "step": 33, "value": 281 }, { "step": 34, "value": 281 }, { "step": 35, "value": 281 }, { "step": 36, "value": 278 }, { "step": 37, "value": 278 }, { "step": 38, "value": 276 }, { "step": 39, "value": 276 }, { "step": 40, "value": 276 }, { "step": 41, "value": 276 }, { "step": 42, "value": 276 }, { "step": 43, "value": 276 }, { "step": 44, "value": 276 }, { "step": 45, "value": 276 }, { "step": 46, "value": 276 }, { "step": 47, "value": 276 }, { "step": 48, "value": 276 }, { "step": 49, "value": 276 }, { "step": 50, "value": 276 }, { "step": 51, "value": 276 }, { "step": 52, "value": 276 }, { "step": 53, "value": 275 }, { "step": 54, "value": 275 }, { "step": 55, "value": 275 }, { "step": 56, "value": 275 }, { "step": 57, "value": 275 }, { "step": 58, "value": 275 }, { "step": 59, "value": 275 }, { "step": 60, "value": 274 }, { "step": 61, "value": 274 }, { "step": 62, "value": 274 }, { "step": 63, "value": 274 }, { "step": 64, "value": 274 }, { "step": 65, "value": 273 }, { "step": 66, "value": 273 }, { "step": 67, "value": 272 }, { "step": 68, "value": 272 }, { "step": 69, "value": 272 }, { "step": 70, "value": 272 }, { "step": 71, "value": 272 }, { "step": 72, "value": 272 }, { "step": 73, "value": 272 }, { "step": 74, "value": 272 }, { "step": 75, "value": 272 }, { "step": 76, "value": 272 }, { "step": 77, "value": 272 }, { "step": 78, "value": 270 }, { "step": 79, "value": 270 }, { "step": 80, "value": 270 }, { "step": 81, "value": 270 }, { "step": 82, "value": 270 }, { "step": 83, "value": 269 }, { "step": 84, "value": 269 }, { "step": 85, "value": 269 }, { "step": 86, "value": 269 }, { "step": 87, "value": 269 }, { "step": 88, "value": 269 }, { "step": 89, "value": 267 }, { "step": 90, "value": 267 }, { "step": 91, "value": 267 }, { "step": 92, "value": 267 }, { "step": 93, "value": 267 }, { "step": 94, "value": 267 }, { "step": 95, "value": 267 }, { "step": 96, "value": 267 }, { "step": 97, "value": 267 }, { "step": 98, "value": 267 }, { "step": 99, "value": 267 }, { "step": 100, "value": 267 } ], "submission_sha256": "a6b2c162419782fa2ed0f131c7468d35fff099c0bee3235b4b18f4d3b20ff628", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/autoscientists/steps/submission.json", "comparison_value": 267, "comparison_rank_key": [ 267.0, 404488192.0, -0.00013586089970085702 ], "rank": 1 }, { "run_id": "sequential-search--steps", "method_id": "sequential-search", "task_id": "steps", "metric": "updates_to_target", "unit": "optimizer steps", "direction": "minimize", "best_value": 295, "reference_value": 756, "relative_improvement": 0.6097883597883598, "rank_key": [ 295.0, 347996160.0, -0.00022339556402695138 ], "best_launch_seq": 64, "best_candidate_id": "exp-037351cf429bc09bd59698bc", "best_is_reference": false, "budget": { "spent": 69, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 20.91512027777779, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 756, "is_reference": true }, { "step": 5, "launch_seq": 5, "candidate_id": "r004-anneal-horizon", "value": 752, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "r006-dose1p5-travel", "value": 506, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "r007-adamw-linear-lr", "value": 509, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "r010-dose1p75-mb64", "value": 391, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "r011-dose2x-mb64", "value": 362, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "r012-mb32", "value": 357, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "r014-mb16", "value": 344, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "r016-dose2p125-mb16", "value": 325, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "r017-warmdown0p8", "value": 325, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "r018-dose-step-at-anneal", "value": 328, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "r019-graded-windows", "value": 327, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "r020-loader-sync", "value": 321, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "r022-fp32-embed-masters", "value": 320, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "r023-ve-ends", "value": 319, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "r024-dose35-reverts", "value": 311, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "r026-momentum-tokens", "value": 310, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "r028-mlp2560-span6144", "value": 322, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "r030-head-dose4x", "value": 328, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "r032-attn-scale012", "value": 312, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "r035-graded-temp-dose36", "value": 314, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "r036-graded-temp-only", "value": 309, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "r037-inverse-graded-temp", "value": 308, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "r039-softcap10", "value": 307, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "r040-softcap14p4", "value": 307, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "r042-short-temp016", "value": 322, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "r043-ve-gate-mix", "value": 307, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "r044-rows14", "value": 315, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "r045-gate-fullwidth-pool", "value": 307, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp-007d7b4a00198c21069f77a0", "value": 315, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp-ec68b88c3b7f55f41ed932ae", "value": 302, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp-09bfb8b146c54aee0b5e15e7", "value": 303, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "exp-1cda73c6540af1e3aa8e9aac", "value": 303, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp-5789385f89241fc95384bfff", "value": 308, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp-7cecb2d52209b242c22f96e0", "value": 318, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp-88a15dbd1a85ea2628d6d556", "value": 300, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp-3db630ba7a47eeea34b80b95", "value": 314, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp-368c54a22e2cf66ee0cb8227", "value": 301, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp-c8878436b02e348792c6ff96", "value": 299, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp-1a5ea2c5fc6fe03d7fe1db02", "value": 315, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "exp-8608ef4b5ca4a96cc0b1e498", "value": 310, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "exp-07c1051f1defda7fd68ef96e", "value": 295, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp-26b8d19fb4757e85faa6a132", "value": 311, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp-037351cf429bc09bd59698bc", "value": 295, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "exp-c0c8172cd28494c18366a5ab", "value": 310, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp-1957ad38ae974b5602a9eea5", "value": 296, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "exp-ce846cb27fd29013b0f791b9", "value": 311, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "exp-5af47f53f2c5400ffa0f3049", "value": 296, "is_reference": false } ], "curve": [ { "step": 1, "value": 756 }, { "step": 2, "value": 756 }, { "step": 3, "value": 756 }, { "step": 4, "value": 756 }, { "step": 5, "value": 752 }, { "step": 6, "value": 752 }, { "step": 7, "value": 506 }, { "step": 8, "value": 506 }, { "step": 9, "value": 506 }, { "step": 10, "value": 506 }, { "step": 11, "value": 391 }, { "step": 12, "value": 362 }, { "step": 13, "value": 357 }, { "step": 14, "value": 357 }, { "step": 15, "value": 344 }, { "step": 16, "value": 344 }, { "step": 17, "value": 325 }, { "step": 18, "value": 325 }, { "step": 19, "value": 325 }, { "step": 20, "value": 325 }, { "step": 21, "value": 321 }, { "step": 22, "value": 321 }, { "step": 23, "value": 320 }, { "step": 24, "value": 319 }, { "step": 25, "value": 311 }, { "step": 26, "value": 311 }, { "step": 27, "value": 310 }, { "step": 28, "value": 310 }, { "step": 29, "value": 310 }, { "step": 30, "value": 310 }, { "step": 31, "value": 310 }, { "step": 32, "value": 310 }, { "step": 33, "value": 310 }, { "step": 34, "value": 310 }, { "step": 35, "value": 310 }, { "step": 36, "value": 309 }, { "step": 37, "value": 308 }, { "step": 38, "value": 308 }, { "step": 39, "value": 307 }, { "step": 40, "value": 307 }, { "step": 41, "value": 307 }, { "step": 42, "value": 307 }, { "step": 43, "value": 307 }, { "step": 44, "value": 307 }, { "step": 45, "value": 307 }, { "step": 46, "value": 307 }, { "step": 47, "value": 302 }, { "step": 48, "value": 302 }, { "step": 49, "value": 302 }, { "step": 50, "value": 302 }, { "step": 51, "value": 302 }, { "step": 52, "value": 302 }, { "step": 53, "value": 302 }, { "step": 54, "value": 300 }, { "step": 55, "value": 300 }, { "step": 56, "value": 300 }, { "step": 57, "value": 299 }, { "step": 58, "value": 299 }, { "step": 59, "value": 299 }, { "step": 60, "value": 299 }, { "step": 61, "value": 295 }, { "step": 62, "value": 295 }, { "step": 63, "value": 295 }, { "step": 64, "value": 295 }, { "step": 65, "value": 295 }, { "step": 66, "value": 295 }, { "step": 67, "value": 295 }, { "step": 68, "value": 295 }, { "step": 69, "value": 295 } ], "submission_sha256": "a5125344d742bed1901f5ec427be4de22638943fa7204acadbe95832356475ec", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/sequential-search/steps/submission.json", "comparison_value": 295, "comparison_rank_key": [ 295.0, 347996160.0, -0.00022339556402695138 ], "rank": 2 }, { "run_id": "gear--steps", "method_id": "gear", "task_id": "steps", "metric": "updates_to_target", "unit": "optimizer steps", "direction": "minimize", "best_value": 306, "reference_value": 725, "relative_improvement": 0.5779310344827586, "rank_key": [ 306.0, 360972288.0, -0.000940818580319025 ], "best_launch_seq": 97, "best_candidate_id": "exp0097", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": null }, "evaluation_gpu_hours": 30.16663472222222, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 725, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp0002", "value": 512, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp0003", "value": 545, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp0005", "value": 364, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp0006", "value": 457, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp0008", "value": 642, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp0009", "value": 359, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp0010", "value": 546, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp0011", "value": 456, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp0012", "value": 545, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp0014", "value": 544, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp0015", "value": 543, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp0016", "value": 358, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp0017", "value": 455, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp0018", "value": 543, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp0019", "value": 542, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp0020", "value": 454, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp0021", "value": 504, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp0022", "value": 484, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp0023", "value": 406, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp0025", "value": 470, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp0027", "value": 343, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp0028", "value": 496, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp0029", "value": 486, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp0030", "value": 430, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp0031", "value": 472, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp0032", "value": 322, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp0033", "value": 324, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp0034", "value": 431, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp0035", "value": 323, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp0036", "value": 325, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp0037", "value": 323, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp0038", "value": 323, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp0039", "value": 324, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp0041", "value": 323, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "exp0042", "value": 324, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp0045", "value": 309, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp0046", "value": 324, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp0048", "value": 324, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp0054", "value": 324, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp0055", "value": 324, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp0056", "value": 324, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp0058", "value": 324, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "exp0059", "value": 325, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "exp0060", "value": 325, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp0063", "value": 321, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp0064", "value": 325, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "exp0065", "value": 326, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp0066", "value": 337, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "exp0067", "value": 325, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "exp0069", "value": 309, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "exp0070", "value": 321, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp0071", "value": 310, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "exp0072", "value": 310, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "exp0075", "value": 310, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "exp0076", "value": 309, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "exp0077", "value": 310, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "exp0078", "value": 310, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "exp0079", "value": 325, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "exp0080", "value": 310, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "exp0081", "value": 310, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "exp0082", "value": 309, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "exp0083", "value": 325, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "exp0084", "value": 310, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "exp0085", "value": 309, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp0086", "value": 320, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "exp0088", "value": 321, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "exp0091", "value": 307, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "exp0092", "value": 324, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "exp0094", "value": 322, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "exp0095", "value": 307, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "exp0096", "value": 323, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "exp0097", "value": 306, "is_reference": false }, { "step": 98, "launch_seq": 98, "candidate_id": "exp0098", "value": 306, "is_reference": false }, { "step": 99, "launch_seq": 99, "candidate_id": "exp0099", "value": 322, "is_reference": false }, { "step": 100, "launch_seq": 100, "candidate_id": "exp0100", "value": 322, "is_reference": false } ], "curve": [ { "step": 1, "value": 725 }, { "step": 2, "value": 512 }, { "step": 3, "value": 512 }, { "step": 4, "value": 512 }, { "step": 5, "value": 364 }, { "step": 6, "value": 364 }, { "step": 7, "value": 364 }, { "step": 8, "value": 364 }, { "step": 9, "value": 359 }, { "step": 10, "value": 359 }, { "step": 11, "value": 359 }, { "step": 12, "value": 359 }, { "step": 13, "value": 359 }, { "step": 14, "value": 359 }, { "step": 15, "value": 359 }, { "step": 16, "value": 358 }, { "step": 17, "value": 358 }, { "step": 18, "value": 358 }, { "step": 19, "value": 358 }, { "step": 20, "value": 358 }, { "step": 21, "value": 358 }, { "step": 22, "value": 358 }, { "step": 23, "value": 358 }, { "step": 24, "value": 358 }, { "step": 25, "value": 358 }, { "step": 26, "value": 358 }, { "step": 27, "value": 343 }, { "step": 28, "value": 343 }, { "step": 29, "value": 343 }, { "step": 30, "value": 343 }, { "step": 31, "value": 343 }, { "step": 32, "value": 322 }, { "step": 33, "value": 322 }, { "step": 34, "value": 322 }, { "step": 35, "value": 322 }, { "step": 36, "value": 322 }, { "step": 37, "value": 322 }, { "step": 38, "value": 322 }, { "step": 39, "value": 322 }, { "step": 40, "value": 322 }, { "step": 41, "value": 322 }, { "step": 42, "value": 322 }, { "step": 43, "value": 322 }, { "step": 44, "value": 322 }, { "step": 45, "value": 309 }, { "step": 46, "value": 309 }, { "step": 47, "value": 309 }, { "step": 48, "value": 309 }, { "step": 49, "value": 309 }, { "step": 50, "value": 309 }, { "step": 51, "value": 309 }, { "step": 52, "value": 309 }, { "step": 53, "value": 309 }, { "step": 54, "value": 309 }, { "step": 55, "value": 309 }, { "step": 56, "value": 309 }, { "step": 57, "value": 309 }, { "step": 58, "value": 309 }, { "step": 59, "value": 309 }, { "step": 60, "value": 309 }, { "step": 61, "value": 309 }, { "step": 62, "value": 309 }, { "step": 63, "value": 309 }, { "step": 64, "value": 309 }, { "step": 65, "value": 309 }, { "step": 66, "value": 309 }, { "step": 67, "value": 309 }, { "step": 68, "value": 309 }, { "step": 69, "value": 309 }, { "step": 70, "value": 309 }, { "step": 71, "value": 309 }, { "step": 72, "value": 309 }, { "step": 73, "value": 309 }, { "step": 74, "value": 309 }, { "step": 75, "value": 309 }, { "step": 76, "value": 309 }, { "step": 77, "value": 309 }, { "step": 78, "value": 309 }, { "step": 79, "value": 309 }, { "step": 80, "value": 309 }, { "step": 81, "value": 309 }, { "step": 82, "value": 309 }, { "step": 83, "value": 309 }, { "step": 84, "value": 309 }, { "step": 85, "value": 309 }, { "step": 86, "value": 309 }, { "step": 87, "value": 309 }, { "step": 88, "value": 309 }, { "step": 89, "value": 309 }, { "step": 90, "value": 309 }, { "step": 91, "value": 307 }, { "step": 92, "value": 307 }, { "step": 93, "value": 307 }, { "step": 94, "value": 307 }, { "step": 95, "value": 307 }, { "step": 96, "value": 307 }, { "step": 97, "value": 306 }, { "step": 98, "value": 306 }, { "step": 99, "value": 306 }, { "step": 100, "value": 306 } ], "submission_sha256": "f91fed5fe285449e590dac2ff172b749099c031bc4d134164d7af373d8197ef7", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/gear/steps/submission.json", "comparison_value": 306, "comparison_rank_key": [ 306.0, 360972288.0, -0.000940818580319025 ], "rank": 3 }, { "run_id": "beam-search--steps", "method_id": "beam-search", "task_id": "steps", "metric": "updates_to_target", "unit": "optimizer steps", "direction": "minimize", "best_value": 335, "reference_value": 720, "relative_improvement": 0.5347222222222222, "rank_key": [ 335.0, 345784320.0, -0.00047949909967059945 ], "best_launch_seq": 94, "best_candidate_id": "g14-c8-fp32-adam-moments-with-rows-24-on-the-master-line", "best_is_reference": false, "budget": { "spent": 95, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 4, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 4 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 26.696180277777778, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 720, "is_reference": true }, { "step": 6, "launch_seq": 6, "candidate_id": "g03-c1-update-rung-1p5x", "value": 510, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "g04-c1-rung-3p5-microbatch112", "value": 417, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "g04-c2-orthogonalisation-depth-at-crossing-rung", "value": 480, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "g05-c1-ortho-depth-at-917504", "value": 416, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "g05-c3-momentum-ramp-update-matched", "value": 505, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "g05-c5-ortho-tail-fixed-point", "value": 500, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "g05-c2-rung-983040-microbatch120", "value": 410, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "g05-c4-weight-decay-off-the-progress-clock", "value": 505, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "g05-c6-normuon-output-axis", "value": 476, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "g06-c1-normuon-row-axis-at-983040", "value": 409, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "g06-c3-polar-express-fp32", "value": 409, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "g06-c5-normuon-per-entry-second-moment", "value": 408, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "g07-c1-rung-1032192-microbatch126", "value": 388, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "g07-c3-long-windows-top-two-layers", "value": 408, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "g07-c5-fp32-ortho-at-983040", "value": 404, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "g07-c7-per-entry-moment-at-983040", "value": 408, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "g07-c4-muon-symmetric-shape-factor", "value": 407, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "g07-c6-logit-softcap-width", "value": 435, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "g07-c8-schedule-horizon-at-the-deciding-boundary", "value": 412, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "g07-c2-value-residual-convex-gate", "value": 408, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "g08-c9-long-window-at-layer-5-on-the-top-rung", "value": 382, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "g08-c10-long-window-at-layer-5-on-the-fp32-ortho-member", "value": 406, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "g08-c11-no-per-shape-conversion-factor", "value": 402, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "g08-c12-three-full-context-layers-at-a-fixed-span", "value": 389, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "g08-c6-spectral-bound-prescale", "value": 407, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "g08-c8-depth-profiled-embedding-shortcut", "value": 402, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "g09-c5-four-full-context-layers-at-a-fixed-span", "value": 411, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "g09-c6-span-matched-attention-logit-scale", "value": 411, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "g09-c8-value-gate-out-of-the-orthogonalised-group", "value": 402, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "g10-c1-microbatch-60-accum-8-at-fixed-tokens-per-step", "value": 383, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "g10-c4-gate-out-of-the-orthogonalised-group-on-the-graded-span-carrier", "value": 411, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "g10-c6-uniform-attention-temperature-on-the-layer-5-carrier", "value": 389, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "g10-c2-gate-rate-inside-the-orthogonalised-group", "value": 405, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "g11-c1-accumulation-depth-16-at-fixed-tokens-per-update", "value": 358, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "g11-c3-tokens-per-update-1032192-on-the-graded-span-carrier", "value": 395, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "g11-c2-anneal-horizon-at-the-boundary-18-clock", "value": 365, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "g12-c9-accumulation-depth-40-at-fixed-tokens-per-update", "value": 350, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "g12-c11-fp32-master-copy-alone-for-the-bf16-embedding-family", "value": 363, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "g12-c12-accumulation-depth-12-on-the-top-rung-carrier", "value": 365, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "g12-c4-per-family-anneal-instead-of-one-multiplier", "value": 363, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "g12-c8-accumulation-depth-18-on-the-top-rung-carrier", "value": 339, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "g12-c2-fp32-accumulation-of-the-bf16-embedding-gradients", "value": 359, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "g13-c1-master-parameter-at-split-18-on-the-top-rung", "value": 338, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "g13-c3-master-parameter-at-split-40", "value": 338, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "g13-c5-accumulation-rows-20-at-fixed-tokens-per-update", "value": 350, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "g13-c9-fp32-adam-moment-buffers-with-widened-gradient-for-the-bf16-family", "value": 359, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "g13-c4-accumulation-rows-16-at-fixed-tokens-per-update", "value": 343, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "g13-c6-accumulation-rows-24-at-fixed-tokens-per-update", "value": 354, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "g13-c10-fp32-adam-moment-buffers-and-master-parameter-with-widened-gradient", "value": 356, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "g13-c2-accumulation-rows-21-on-the-top-rung", "value": 351, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "g14-c1-fp32-adam-moments-at-split-40", "value": 349, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "g14-c3-fp32-adam-moments-at-16-rows", "value": 347, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "g14-c5-accumulation-rows-24-on-the-top-rung-carrier", "value": 352, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "g14-c7-fp32-adam-moments-with-rows-21-on-the-top-rung-carrier", "value": 350, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "g14-c4-fp32-adam-moments-with-40-way-accumulation", "value": 350, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "g14-c6-fp32-adam-moments-with-rows-24-on-the-top-rung-carrier", "value": 337, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "g14-c8-fp32-adam-moments-with-rows-24-on-the-master-line", "value": 335, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "g14-c2-fp32-adam-moments-with-48-way-accumulation", "value": 350, "is_reference": false } ], "curve": [ { "step": 1, "value": 720 }, { "step": 2, "value": 720 }, { "step": 3, "value": 720 }, { "step": 4, "value": 720 }, { "step": 5, "value": 720 }, { "step": 6, "value": 510 }, { "step": 7, "value": 510 }, { "step": 8, "value": 417 }, { "step": 9, "value": 417 }, { "step": 10, "value": 417 }, { "step": 11, "value": 417 }, { "step": 12, "value": 416 }, { "step": 13, "value": 416 }, { "step": 14, "value": 416 }, { "step": 15, "value": 416 }, { "step": 16, "value": 410 }, { "step": 17, "value": 410 }, { "step": 18, "value": 410 }, { "step": 19, "value": 410 }, { "step": 20, "value": 409 }, { "step": 21, "value": 409 }, { "step": 22, "value": 408 }, { "step": 23, "value": 408 }, { "step": 24, "value": 408 }, { "step": 25, "value": 408 }, { "step": 26, "value": 408 }, { "step": 27, "value": 408 }, { "step": 28, "value": 388 }, { "step": 29, "value": 388 }, { "step": 30, "value": 388 }, { "step": 31, "value": 388 }, { "step": 32, "value": 388 }, { "step": 33, "value": 388 }, { "step": 34, "value": 388 }, { "step": 35, "value": 388 }, { "step": 36, "value": 382 }, { "step": 37, "value": 382 }, { "step": 38, "value": 382 }, { "step": 39, "value": 382 }, { "step": 40, "value": 382 }, { "step": 41, "value": 382 }, { "step": 42, "value": 382 }, { "step": 43, "value": 382 }, { "step": 44, "value": 382 }, { "step": 45, "value": 382 }, { "step": 46, "value": 382 }, { "step": 47, "value": 382 }, { "step": 48, "value": 382 }, { "step": 49, "value": 382 }, { "step": 50, "value": 382 }, { "step": 51, "value": 382 }, { "step": 52, "value": 382 }, { "step": 53, "value": 382 }, { "step": 54, "value": 382 }, { "step": 55, "value": 382 }, { "step": 56, "value": 382 }, { "step": 57, "value": 382 }, { "step": 58, "value": 382 }, { "step": 59, "value": 382 }, { "step": 60, "value": 358 }, { "step": 61, "value": 358 }, { "step": 62, "value": 358 }, { "step": 63, "value": 358 }, { "step": 64, "value": 358 }, { "step": 65, "value": 358 }, { "step": 66, "value": 358 }, { "step": 67, "value": 358 }, { "step": 68, "value": 358 }, { "step": 69, "value": 358 }, { "step": 70, "value": 358 }, { "step": 71, "value": 358 }, { "step": 72, "value": 350 }, { "step": 73, "value": 350 }, { "step": 74, "value": 350 }, { "step": 75, "value": 350 }, { "step": 76, "value": 350 }, { "step": 77, "value": 350 }, { "step": 78, "value": 339 }, { "step": 79, "value": 339 }, { "step": 80, "value": 338 }, { "step": 81, "value": 338 }, { "step": 82, "value": 338 }, { "step": 83, "value": 338 }, { "step": 84, "value": 338 }, { "step": 85, "value": 338 }, { "step": 86, "value": 338 }, { "step": 87, "value": 338 }, { "step": 88, "value": 338 }, { "step": 89, "value": 338 }, { "step": 90, "value": 338 }, { "step": 91, "value": 338 }, { "step": 92, "value": 338 }, { "step": 93, "value": 337 }, { "step": 94, "value": 335 }, { "step": 95, "value": 335 } ], "submission_sha256": "8f962ba2c8ddcb3ae39f9181ea43512ab5eedab2d5b18aa0f5ebf93131743352", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/beam-search/steps/submission.json", "comparison_value": 335, "comparison_rank_key": [ 335.0, 345784320.0, -0.00047949909967059945 ], "rank": 4 }, { "run_id": "arbor--steps", "method_id": "arbor", "task_id": "steps", "metric": "updates_to_target", "unit": "optimizer steps", "direction": "minimize", "best_value": 337, "reference_value": 750, "relative_improvement": 0.5506666666666666, "rank_key": [ 337.0, 389283840.0, -0.0007963633812522186 ], "best_launch_seq": 20, "best_candidate_id": "exp-d2a4c5c69c9d64c031f89942", "best_is_reference": false, "budget": { "spent": 32, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 10.614661944444446, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 750, "is_reference": true }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-58b54e83f4584c1a68d81aa2", "value": 712, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-4679296ebbfbed52ed602fe1", "value": 505, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp-a7303b6ce6ba861424733757", "value": 615, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-b86080c6eedb99a5558aad44", "value": 479, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-8fa4ec7ea8308b0440ec6a7a", "value": 442, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-671ce2ff1e430aac9056d3d1", "value": 363, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-684da598daefb82f82ba2ecf", "value": 343, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-d2a4c5c69c9d64c031f89942", "value": 337, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-945bd2db757327e8976ea420", "value": 341, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp-6e89cd1c3d1c3389092bf481", "value": 342, "is_reference": false } ], "curve": [ { "step": 1, "value": 750 }, { "step": 2, "value": 750 }, { "step": 3, "value": 750 }, { "step": 4, "value": 712 }, { "step": 5, "value": 505 }, { "step": 6, "value": 505 }, { "step": 7, "value": 505 }, { "step": 8, "value": 505 }, { "step": 9, "value": 505 }, { "step": 10, "value": 479 }, { "step": 11, "value": 479 }, { "step": 12, "value": 479 }, { "step": 13, "value": 442 }, { "step": 14, "value": 363 }, { "step": 15, "value": 363 }, { "step": 16, "value": 363 }, { "step": 17, "value": 343 }, { "step": 18, "value": 343 }, { "step": 19, "value": 343 }, { "step": 20, "value": 337 }, { "step": 21, "value": 337 }, { "step": 22, "value": 337 }, { "step": 23, "value": 337 }, { "step": 24, "value": 337 }, { "step": 25, "value": 337 }, { "step": 26, "value": 337 }, { "step": 27, "value": 337 }, { "step": 28, "value": 337 }, { "step": 29, "value": 337 }, { "step": 30, "value": 337 }, { "step": 31, "value": 337 }, { "step": 32, "value": 337 } ], "submission_sha256": "7e64f0ae72d85e0f94d4db8f048938f1673f2e3222eee48d75054620d327c562", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/arbor/steps/submission.json", "comparison_value": 337, "comparison_rank_key": [ 337.0, 389283840.0, -0.0007963633812522186 ], "rank": 5 }, { "run_id": "heuresis-map-elites--steps", "method_id": "heuresis-map-elites", "task_id": "steps", "metric": "updates_to_target", "unit": "optimizer steps", "direction": "minimize", "best_value": 377, "reference_value": 721, "relative_improvement": 0.47711511789181693, "rank_key": [ 377.0, 395313152.0, -0.0029298914327220427 ], "best_launch_seq": 40, "best_candidate_id": "exp-e0b90294f9faca724df10872", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 30.139203333333334, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 721, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-36fcbfe7d686ccc4950e6735", "value": 621, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp-f4fe74bfa96d7a43e53fe268", "value": 501, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-608526218d81685b6f493c88", "value": 440, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-28572936cf7bd41e0e0a97eb", "value": 379, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp-0cebb03c95f0efbb4d4512b7", "value": 614, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-89899d072dc95efdaca9db45", "value": 493, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-c1bbba9e2aa96ca836aa3ac7", "value": 438, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-d6ce9c2769b965bbe831d66e", "value": 396, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-83861f81aea7560792d41653", "value": 650, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp-d7e43235ca0ca0ff67f35c97", "value": 523, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-a93c5850d59b46fbdef3488a", "value": 438, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-687b68b342e31f7d9b534390", "value": 396, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-2f3bbbc206ecc0f15693ca2e", "value": 756, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-f6c7b5ae648e79f0fc30293b", "value": 540, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-c8918d54246150f16db22625", "value": 502, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp-5fc9ea872269f7734889e195", "value": 431, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp-d3600b7441a0f6c2535d9f79", "value": 748, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp-f0a67f87d7a32573dc9784e8", "value": 540, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp-a8fd8e672f9bb4d71977597b", "value": 476, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp-2be3f4b1a7289c521e2bb0c0", "value": 475, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-961a1ad510ec6510a8727ca7", "value": 450, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-8e0d735815c7c622f4ff72b6", "value": 450, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-faf13fc602839d8fbaf758e1", "value": 737, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp-56029a8a798c94ce991aa2ec", "value": 634, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp-4f2c437eab4ee6adc87e67b2", "value": 474, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "exp-e0b90294f9faca724df10872", "value": 377, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp-7449b5517f7846040682dbc0", "value": 656, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp-6c9ce1eb2d4f34b81012e2ad", "value": 466, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp-58abf99d7446d0057da34648", "value": 400, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp-9f395d2db40996c8b8ad0d01", "value": 401, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp-516ae0d696b3b985b9250ff8", "value": 381, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "exp-f8cc2369d330dc9053098c14", "value": 629, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp-0fd6c91c0a2c9ca92b7791a4", "value": 475, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp-89425b648acb75fc00d48b98", "value": 430, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp-9f37ebbde67dee6ab11ee25e", "value": 625, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "exp-c0195ee242dc77e3e3eb3b5e", "value": 455, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp-c956946c98bdc050fcc2749d", "value": 469, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp-106383bb5ffe227711e30747", "value": 470, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp-34f715687ec512774e412b94", "value": 630, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp-72cf890920341b7736e44755", "value": 475, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp-1fc4d0dc31c9ca7a4b479cbc", "value": 450, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "exp-5678254b18232dcfe34b9632", "value": 475, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "exp-2155bc81eecb58fa5dbfc9d8", "value": 410, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "exp-4422ca4786432db624398a99", "value": 431, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp-dd9844d9da2226499f4b1a29", "value": 419, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp-9c7ede5f8c7f8d7cb48f6588", "value": 637, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "exp-819bdceb69eebc09546df3d1", "value": 522, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp-04846bdf7fe06e590c842fcc", "value": 481, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "exp-401aa1dfdcb243a8323ee02e", "value": 481, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "exp-508fd22cceea8e1c6e7b32e0", "value": 481, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "exp-b38fe618a7a2b28b58bc642d", "value": 624, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "exp-a4e5c44af5147d87b9c85911", "value": 535, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp-3dbe1828e9316e804aa86bae", "value": 471, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "exp-a336165861e104927e339064", "value": 470, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "exp-3dbd2a6559a33a9cec186663", "value": 672, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "exp-6c6decf1025998562ea781a0", "value": 556, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "exp-07565c311c56746020e764ed", "value": 566, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "exp-f5714cb24ffbbaba041cc524", "value": 543, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "exp-7eac2cf72208792372fd9367", "value": 535, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "exp-2569e33d2c2892a46f9a122a", "value": 530, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "exp-aa1d40e892fb52eca77ce296", "value": 668, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "exp-35dbd2d415484595d1de1b48", "value": 594, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "exp-fd1d4d9cd0f59ca6f5d15cb5", "value": 513, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "exp-ce3368695600a216aa374d0a", "value": 635, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "exp-86328daaf18c7ecd11810846", "value": 478, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp-bf4ebbb4c8666e1f224f5caa", "value": 406, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "exp-f54539529e07e88bc58ba42b", "value": 389, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "exp-734604014351b0fd597d0303", "value": 640, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "exp-f36ee3c2f6607d761ef51365", "value": 569, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "exp-848f855fb6da4a0d3b9364e9", "value": 521, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "exp-72c415cffeacd899bb21435c", "value": 424, "is_reference": false } ], "curve": [ { "step": 1, "value": 721 }, { "step": 2, "value": 621 }, { "step": 3, "value": 501 }, { "step": 4, "value": 501 }, { "step": 5, "value": 440 }, { "step": 6, "value": 379 }, { "step": 7, "value": 379 }, { "step": 8, "value": 379 }, { "step": 9, "value": 379 }, { "step": 10, "value": 379 }, { "step": 11, "value": 379 }, { "step": 12, "value": 379 }, { "step": 13, "value": 379 }, { "step": 14, "value": 379 }, { "step": 15, "value": 379 }, { "step": 16, "value": 379 }, { "step": 17, "value": 379 }, { "step": 18, "value": 379 }, { "step": 19, "value": 379 }, { "step": 20, "value": 379 }, { "step": 21, "value": 379 }, { "step": 22, "value": 379 }, { "step": 23, "value": 379 }, { "step": 24, "value": 379 }, { "step": 25, "value": 379 }, { "step": 26, "value": 379 }, { "step": 27, "value": 379 }, { "step": 28, "value": 379 }, { "step": 29, "value": 379 }, { "step": 30, "value": 379 }, { "step": 31, "value": 379 }, { "step": 32, "value": 379 }, { "step": 33, "value": 379 }, { "step": 34, "value": 379 }, { "step": 35, "value": 379 }, { "step": 36, "value": 379 }, { "step": 37, "value": 379 }, { "step": 38, "value": 379 }, { "step": 39, "value": 379 }, { "step": 40, "value": 377 }, { "step": 41, "value": 377 }, { "step": 42, "value": 377 }, { "step": 43, "value": 377 }, { "step": 44, "value": 377 }, { "step": 45, "value": 377 }, { "step": 46, "value": 377 }, { "step": 47, "value": 377 }, { "step": 48, "value": 377 }, { "step": 49, "value": 377 }, { "step": 50, "value": 377 }, { "step": 51, "value": 377 }, { "step": 52, "value": 377 }, { "step": 53, "value": 377 }, { "step": 54, "value": 377 }, { "step": 55, "value": 377 }, { "step": 56, "value": 377 }, { "step": 57, "value": 377 }, { "step": 58, "value": 377 }, { "step": 59, "value": 377 }, { "step": 60, "value": 377 }, { "step": 61, "value": 377 }, { "step": 62, "value": 377 }, { "step": 63, "value": 377 }, { "step": 64, "value": 377 }, { "step": 65, "value": 377 }, { "step": 66, "value": 377 }, { "step": 67, "value": 377 }, { "step": 68, "value": 377 }, { "step": 69, "value": 377 }, { "step": 70, "value": 377 }, { "step": 71, "value": 377 }, { "step": 72, "value": 377 }, { "step": 73, "value": 377 }, { "step": 74, "value": 377 }, { "step": 75, "value": 377 }, { "step": 76, "value": 377 }, { "step": 77, "value": 377 }, { "step": 78, "value": 377 }, { "step": 79, "value": 377 }, { "step": 80, "value": 377 }, { "step": 81, "value": 377 }, { "step": 82, "value": 377 }, { "step": 83, "value": 377 }, { "step": 84, "value": 377 }, { "step": 85, "value": 377 }, { "step": 86, "value": 377 }, { "step": 87, "value": 377 }, { "step": 88, "value": 377 }, { "step": 89, "value": 377 }, { "step": 90, "value": 377 }, { "step": 91, "value": 377 }, { "step": 92, "value": 377 }, { "step": 93, "value": 377 }, { "step": 94, "value": 377 }, { "step": 95, "value": 377 }, { "step": 96, "value": 377 }, { "step": 97, "value": 377 }, { "step": 98, "value": 377 }, { "step": 99, "value": 377 }, { "step": 100, "value": 377 } ], "submission_sha256": "aafb190e84fa3689a26959537e419dca8743080695db71660754d7884d1ce6a9", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/heuresis-map-elites/steps/submission.json", "comparison_value": 377, "comparison_rank_key": [ 377.0, 395313152.0, -0.0029298914327220427 ], "rank": 6 }, { "run_id": "tpe--steps", "method_id": "tpe", "task_id": "steps", "metric": "updates_to_target", "unit": "optimizer steps", "direction": "minimize", "best_value": 753, "reference_value": 753, "relative_improvement": 0.0, "rank_key": [ 753.0, 394788864.0, -0.002720901303445933 ], "best_launch_seq": 1, "best_candidate_id": "reference", "best_is_reference": true, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 25.113068888888893, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 753, "is_reference": true }, { "step": 37, "launch_seq": 37, "candidate_id": "tpe-trial-0036", "value": 2477, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "tpe-trial-0037", "value": 2611, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "tpe-trial-0044", "value": 2472, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "tpe-trial-0045", "value": 2439, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "tpe-trial-0046", "value": 2594, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "tpe-trial-0048", "value": 2578, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "tpe-trial-0051", "value": 2583, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "tpe-trial-0054", "value": 2613, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "tpe-trial-0059", "value": 2521, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "tpe-trial-0061", "value": 2416, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "tpe-trial-0062", "value": 1920, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "tpe-trial-0070", "value": 1638, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "tpe-trial-0071", "value": 1636, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "tpe-trial-0073", "value": 1614, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "tpe-trial-0075", "value": 3020, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "tpe-trial-0078", "value": 2504, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "tpe-trial-0081", "value": 2529, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "tpe-trial-0082", "value": 2734, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "tpe-trial-0084", "value": 1495, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "tpe-trial-0085", "value": 1489, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "tpe-trial-0086", "value": 1494, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "tpe-trial-0091", "value": 1600, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "tpe-trial-0093", "value": 2712, "is_reference": false } ], "curve": [ { "step": 1, "value": 753 }, { "step": 2, "value": 753 }, { "step": 3, "value": 753 }, { "step": 4, "value": 753 }, { "step": 5, "value": 753 }, { "step": 6, "value": 753 }, { "step": 7, "value": 753 }, { "step": 8, "value": 753 }, { "step": 9, "value": 753 }, { "step": 10, "value": 753 }, { "step": 11, "value": 753 }, { "step": 12, "value": 753 }, { "step": 13, "value": 753 }, { "step": 14, "value": 753 }, { "step": 15, "value": 753 }, { "step": 16, "value": 753 }, { "step": 17, "value": 753 }, { "step": 18, "value": 753 }, { "step": 19, "value": 753 }, { "step": 20, "value": 753 }, { "step": 21, "value": 753 }, { "step": 22, "value": 753 }, { "step": 23, "value": 753 }, { "step": 24, "value": 753 }, { "step": 25, "value": 753 }, { "step": 26, "value": 753 }, { "step": 27, "value": 753 }, { "step": 28, "value": 753 }, { "step": 29, "value": 753 }, { "step": 30, "value": 753 }, { "step": 31, "value": 753 }, { "step": 32, "value": 753 }, { "step": 33, "value": 753 }, { "step": 34, "value": 753 }, { "step": 35, "value": 753 }, { "step": 36, "value": 753 }, { "step": 37, "value": 753 }, { "step": 38, "value": 753 }, { "step": 39, "value": 753 }, { "step": 40, "value": 753 }, { "step": 41, "value": 753 }, { "step": 42, "value": 753 }, { "step": 43, "value": 753 }, { "step": 44, "value": 753 }, { "step": 45, "value": 753 }, { "step": 46, "value": 753 }, { "step": 47, "value": 753 }, { "step": 48, "value": 753 }, { "step": 49, "value": 753 }, { "step": 50, "value": 753 }, { "step": 51, "value": 753 }, { "step": 52, "value": 753 }, { "step": 53, "value": 753 }, { "step": 54, "value": 753 }, { "step": 55, "value": 753 }, { "step": 56, "value": 753 }, { "step": 57, "value": 753 }, { "step": 58, "value": 753 }, { "step": 59, "value": 753 }, { "step": 60, "value": 753 }, { "step": 61, "value": 753 }, { "step": 62, "value": 753 }, { "step": 63, "value": 753 }, { "step": 64, "value": 753 }, { "step": 65, "value": 753 }, { "step": 66, "value": 753 }, { "step": 67, "value": 753 }, { "step": 68, "value": 753 }, { "step": 69, "value": 753 }, { "step": 70, "value": 753 }, { "step": 71, "value": 753 }, { "step": 72, "value": 753 }, { "step": 73, "value": 753 }, { "step": 74, "value": 753 }, { "step": 75, "value": 753 }, { "step": 76, "value": 753 }, { "step": 77, "value": 753 }, { "step": 78, "value": 753 }, { "step": 79, "value": 753 }, { "step": 80, "value": 753 }, { "step": 81, "value": 753 }, { "step": 82, "value": 753 }, { "step": 83, "value": 753 }, { "step": 84, "value": 753 }, { "step": 85, "value": 753 }, { "step": 86, "value": 753 }, { "step": 87, "value": 753 }, { "step": 88, "value": 753 }, { "step": 89, "value": 753 }, { "step": 90, "value": 753 }, { "step": 91, "value": 753 }, { "step": 92, "value": 753 }, { "step": 93, "value": 753 }, { "step": 94, "value": 753 }, { "step": 95, "value": 753 }, { "step": 96, "value": 753 }, { "step": 97, "value": 753 }, { "step": 98, "value": 753 }, { "step": 99, "value": 753 }, { "step": 100, "value": 753 } ], "submission_sha256": "cfb95745f46929d74e9a04e3665e503b3d7a553a245edcc194b3367dae7dc281", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/tpe/steps/submission.json", "comparison_value": 753, "comparison_rank_key": [ 753.0, 394788864.0, -0.002720901303445933 ], "rank": 7 } ] }, { "id": "traintime", "title": "Training time to quality at validation BPB < 1.05", "metric": "train_seconds_to_target", "unit": "s", "direction": "minimize", "budget": 100, "comparison_step": 30, "quality_gate": { "metric": "val_bpb", "operator": "<", "value": 1.05 }, "tiebreaks": [ { "metric": "total_tokens", "direction": "minimize" } ], "runs": [ { "run_id": "sequential-search--traintime", "method_id": "sequential-search", "task_id": "traintime", "metric": "train_seconds_to_target", "unit": "s", "direction": "minimize", "best_value": 210.00119256973267, "reference_value": 600.6982984542847, "relative_improvement": 0.6504048819347297, "rank_key": [ 210.00119256973267, 192872448.0, -0.001871805687920558 ], "best_launch_seq": 81, "best_candidate_id": "r086-replicate-frontier-draw", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 15.474251111111103, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 600, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-b9431c1d9a0882a60e8eef5b", "value": 420, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "r002-anneal-horizon", "value": 360, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "r004-microbatch-64", "value": 360, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "r005-anneal-330", "value": 330, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "r006-lr-trough", "value": 330, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "r007-plateau-cap", "value": 330, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "r008-muon-momentum", "value": 330, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "r009-rotary-band", "value": 300, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "r010-anneal-270", "value": 360, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "r011-head-split-8x64", "value": 330, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "r012-local-window-256", "value": 300, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "r013-span-ladder-half", "value": 300, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "r014-attn-temperature", "value": 300, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "r015-anneal270-span256", "value": 270, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "r016-unet-skip", "value": 270, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "r017-anneal-240", "value": 270, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "r018-warmdown-375", "value": 300, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "r020-mlp-width-1664", "value": 270, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "r021-ve-all-layers", "value": 300, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "r024-autotune-pointwise", "value": 270, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "r025-depth-7", "value": 270, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "r026-autotune-gemm-d7", "value": 270, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "r027-whole-horizon-decay", "value": 270, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "r028-drop-softcap", "value": 300, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "r029-softcap-10", "value": 270, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "r031-anneal-210-d7", "value": 270, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "r032-ve-ends", "value": 270, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "r033-lr-floor-04", "value": 300, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "r034-packed-qkv", "value": 270, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "r035-span-matched-rotary", "value": 270, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "r036-algebraic-softcap", "value": 270, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "r037-cap-threshold-matched", "value": 270, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "r039-flat-spectrum-init", "value": 270, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "r040-wd-hold-horizon", "value": 270, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "r042-lr-floor-002", "value": 270, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "r043-ve-gate-adamw", "value": 270, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "r044-unembedding-lr-008", "value": 270, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "r046-fused-lse-loss", "value": 270, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "r047-bf16-logit-store", "value": 270, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "r048-long-tier-1024", "value": 270, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "r049-short-tier-128", "value": 240, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "r050-coord-descent", "value": 240, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "r052-anneal-210-regrid", "value": 240, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "r053-depth-6-width-pinned", "value": 240, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "r054-microbatch-64x4096", "value": 240, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "r055-rows-1024", "value": 240, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "r056-identity-value-path", "value": 270, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "r057-attn-output-gate", "value": 240, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "r058-narrow-o-gate", "value": 240, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "r059-per-head-o-gate", "value": 240, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "r060-x0-skip-gate", "value": 270, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "r061-o-gate-reach", "value": 240, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "r062-cross-at-210", "value": 240, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "r063-mlp-width-1280-at-210", "value": 240, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "r064-half-update-tokens-at-210", "value": 240, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "r065-quarter-update-tokens", "value": 240, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "r066-granularity-keyed-step-size", "value": 240, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "r067-envelope-amplitude-scale", "value": 240, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "r068-per-head-o-gate-batch-scaled", "value": 240, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "r069-short-tier-64", "value": 210, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "r070-anneal-180", "value": 210, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "r071-horizon-keyed-amplitude", "value": 270, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "r072-coarse-update-2p17", "value": 210, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "r073-coarse-update-fixed-token-step", "value": 210, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "r074-amplitude-rung-down", "value": 240, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "r075-long-tier-full-row", "value": 240, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "r076-long-tier-at-layer-0", "value": 300, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "r077-depth-monotone-window-ladder", "value": 270, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "r079-six-value-tables", "value": 210, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "r080-ve-hole-off-interior-long", "value": 240, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "r081-untied-x0-table", "value": 240, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "r082-granularity-keyed-ema-horizons", "value": 300, "is_reference": false }, { "step": 79, "launch_seq": 80, "candidate_id": "r085-shared-tables-six-consumers", "value": 210, "is_reference": false }, { "step": 80, "launch_seq": 81, "candidate_id": "r086-replicate-frontier-draw", "value": 210, "is_reference": false }, { "step": 81, "launch_seq": 82, "candidate_id": "r087-gated-mlp-832", "value": 270, "is_reference": false }, { "step": 82, "launch_seq": 83, "candidate_id": "r088-shared-tables-second-draw", "value": 210, "is_reference": false }, { "step": 83, "launch_seq": 84, "candidate_id": "r089-head-split-8x64", "value": 240, "is_reference": false }, { "step": 84, "launch_seq": 85, "candidate_id": "r090-model-width-448", "value": 270, "is_reference": false }, { "step": 85, "launch_seq": 86, "candidate_id": "r091-eval-geometry-rows-2048", "value": 210, "is_reference": false }, { "step": 86, "launch_seq": 87, "candidate_id": "r092-norm-bf16-backward", "value": 210, "is_reference": false }, { "step": 87, "launch_seq": 88, "candidate_id": "r093-mlp-output-gate", "value": 240, "is_reference": false }, { "step": 88, "launch_seq": 89, "candidate_id": "r094-update-rung-2p15", "value": 330, "is_reference": false }, { "step": 89, "launch_seq": 90, "candidate_id": "r095-rotary-split-16", "value": 210, "is_reference": false }, { "step": 90, "launch_seq": 91, "candidate_id": "r096-neighbour-token-mix", "value": 210, "is_reference": false }, { "step": 91, "launch_seq": 92, "candidate_id": "r097-second-tap-reach", "value": 210, "is_reference": false }, { "step": 92, "launch_seq": 93, "candidate_id": "r098-prev-token-map", "value": 210, "is_reference": false }, { "step": 93, "launch_seq": 94, "candidate_id": "r099-per-head-logit-temperature", "value": 240, "is_reference": false }, { "step": 94, "launch_seq": 95, "candidate_id": "r100-front-loaded-warmdown", "value": 210, "is_reference": false }, { "step": 95, "launch_seq": 96, "candidate_id": "r101-polyak-averaged-iterate", "value": 210, "is_reference": false }, { "step": 96, "launch_seq": 97, "candidate_id": "r102-cheap-polyak-accumulate", "value": 240, "is_reference": false }, { "step": 97, "launch_seq": 98, "candidate_id": "r103-buffer-residency-separator", "value": 210, "is_reference": false }, { "step": 98, "launch_seq": 99, "candidate_id": "r104-resident-stacked-muon-params", "value": 210, "is_reference": false }, { "step": 99, "launch_seq": 100, "candidate_id": "r105-fused-step-polyak-accumulate", "value": 210, "is_reference": false }, { "step": 100, "launch_seq": 101, "candidate_id": "r106-averaging-horizon-15", "value": 210, "is_reference": false } ], "curve": [ { "step": 1, "value": 600 }, { "step": 2, "value": 420 }, { "step": 3, "value": 360 }, { "step": 4, "value": 360 }, { "step": 5, "value": 360 }, { "step": 6, "value": 330 }, { "step": 7, "value": 330 }, { "step": 8, "value": 330 }, { "step": 9, "value": 330 }, { "step": 10, "value": 300 }, { "step": 11, "value": 300 }, { "step": 12, "value": 300 }, { "step": 13, "value": 300 }, { "step": 14, "value": 300 }, { "step": 15, "value": 300 }, { "step": 16, "value": 270 }, { "step": 17, "value": 270 }, { "step": 18, "value": 270 }, { "step": 19, "value": 270 }, { "step": 20, "value": 270 }, { "step": 21, "value": 270 }, { "step": 22, "value": 270 }, { "step": 23, "value": 270 }, { "step": 24, "value": 270 }, { "step": 25, "value": 270 }, { "step": 26, "value": 270 }, { "step": 27, "value": 270 }, { "step": 28, "value": 270 }, { "step": 29, "value": 270 }, { "step": 30, "value": 270 }, { "step": 31, "value": 270 }, { "step": 32, "value": 270 }, { "step": 33, "value": 270 }, { "step": 34, "value": 270 }, { "step": 35, "value": 270 }, { "step": 36, "value": 270 }, { "step": 37, "value": 270 }, { "step": 38, "value": 270 }, { "step": 39, "value": 270 }, { "step": 40, "value": 270 }, { "step": 41, "value": 270 }, { "step": 42, "value": 270 }, { "step": 43, "value": 270 }, { "step": 44, "value": 270 }, { "step": 45, "value": 270 }, { "step": 46, "value": 240 }, { "step": 47, "value": 240 }, { "step": 48, "value": 240 }, { "step": 49, "value": 240 }, { "step": 50, "value": 240 }, { "step": 51, "value": 240 }, { "step": 52, "value": 240 }, { "step": 53, "value": 240 }, { "step": 54, "value": 240 }, { "step": 55, "value": 240 }, { "step": 56, "value": 240 }, { "step": 57, "value": 240 }, { "step": 58, "value": 240 }, { "step": 59, "value": 240 }, { "step": 60, "value": 240 }, { "step": 61, "value": 240 }, { "step": 62, "value": 240 }, { "step": 63, "value": 240 }, { "step": 64, "value": 240 }, { "step": 65, "value": 210 }, { "step": 66, "value": 210 }, { "step": 67, "value": 210 }, { "step": 68, "value": 210 }, { "step": 69, "value": 210 }, { "step": 70, "value": 210 }, { "step": 71, "value": 210 }, { "step": 72, "value": 210 }, { "step": 73, "value": 210 }, { "step": 74, "value": 210 }, { "step": 75, "value": 210 }, { "step": 76, "value": 210 }, { "step": 77, "value": 210 }, { "step": 78, "value": 210 }, { "step": 79, "value": 210 }, { "step": 80, "value": 210 }, { "step": 81, "value": 210 }, { "step": 82, "value": 210 }, { "step": 83, "value": 210 }, { "step": 84, "value": 210 }, { "step": 85, "value": 210 }, { "step": 86, "value": 210 }, { "step": 87, "value": 210 }, { "step": 88, "value": 210 }, { "step": 89, "value": 210 }, { "step": 90, "value": 210 }, { "step": 91, "value": 210 }, { "step": 92, "value": 210 }, { "step": 93, "value": 210 }, { "step": 94, "value": 210 }, { "step": 95, "value": 210 }, { "step": 96, "value": 210 }, { "step": 97, "value": 210 }, { "step": 98, "value": 210 }, { "step": 99, "value": 210 }, { "step": 100, "value": 210 } ], "submission_sha256": "04f9722e60d39c61772cf1f360d16499fd209fe76a429925044d40a0ee531b2f", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/sequential-search/traintime/submission.json", "comparison_value": 210, "comparison_rank_key": [ 210 ], "rank": 1 }, { "run_id": "autoscientists--traintime", "method_id": "autoscientists", "task_id": "traintime", "metric": "train_seconds_to_target", "unit": "s", "direction": "minimize", "best_value": 240.00037550926208, "reference_value": 600.5285093784332, "relative_improvement": 0.600351404202824, "rank_key": [ 240.00037550926208, 205875200.0, -0.00014159246283540483 ], "best_launch_seq": 59, "best_candidate_id": "exp-582a15b438214d13445dcf40", "best_is_reference": false, "budget": { "spent": 76, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 2, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 2 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": null }, "evaluation_gpu_hours": 18.967348888888885, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 600, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-5c4bfffad2ab02427665ea66", "value": 420, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-c02e8ab7288bc19e2093515b", "value": 450, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-c2fbf0f5340401410e2c7c20", "value": 330, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-be3556c699a8dc9c871f771f", "value": 300, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-3eff14e4b6124c930a6f74b1", "value": 360, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-e0212df05727d35f99eea747", "value": 330, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-c265dd2e0bb282dc0609dc6f", "value": 330, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-99a58654c52747521d2b63cb", "value": 360, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-bc4fc391460666c562de048f", "value": 330, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-5f07de0c543f40b4d39f6021", "value": 300, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp-a054eeea765dd8f5dba72032", "value": 330, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-f9cd986a429387b7398e770d", "value": 300, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-2d28871125edd023c5c35f7c", "value": 270, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-437c0d2212ede5687cec58be", "value": 300, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-522d853e71fa144620fe5baa", "value": 270, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-847c3533a2ae2f5b9c8d8345", "value": 270, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp-fdfe8fad02bbb7ee4ed518ec", "value": 270, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp-9a02fea6d836386318f617c9", "value": 270, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp-91c9e407d68aef9a368c77d1", "value": 270, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-c8e6320f9b201767b68f7d38", "value": 240, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp-fd842c2e251206caa2ca487e", "value": 240, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp-11fd64a6142ec6d3592b698f", "value": 240, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "exp-cae22b0b4fa74e687971f102", "value": 240, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp-64728e4b17c2e23646de3360", "value": 240, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp-fd7c7f454f54ee07f05ae6de", "value": 240, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp-5f782c4af02e4da21689157b", "value": 240, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp-8a023d32cbc2fa67e0d9cfa0", "value": 240, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "exp-582a15b438214d13445dcf40", "value": 240, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "exp-20bc509ccd856945285ace8c", "value": 240, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp-7308b909bea07c2ad1b56a24", "value": 240, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "exp-2895e8dd0b57fb680ed43c3f", "value": 240, "is_reference": false } ], "curve": [ { "step": 1, "value": 600 }, { "step": 2, "value": 420 }, { "step": 3, "value": 420 }, { "step": 4, "value": 420 }, { "step": 5, "value": 330 }, { "step": 6, "value": 300 }, { "step": 7, "value": 300 }, { "step": 8, "value": 300 }, { "step": 9, "value": 300 }, { "step": 10, "value": 300 }, { "step": 11, "value": 300 }, { "step": 12, "value": 300 }, { "step": 13, "value": 300 }, { "step": 14, "value": 300 }, { "step": 15, "value": 300 }, { "step": 16, "value": 300 }, { "step": 17, "value": 300 }, { "step": 18, "value": 270 }, { "step": 19, "value": 270 }, { "step": 20, "value": 270 }, { "step": 21, "value": 270 }, { "step": 22, "value": 270 }, { "step": 23, "value": 270 }, { "step": 24, "value": 270 }, { "step": 25, "value": 270 }, { "step": 26, "value": 270 }, { "step": 27, "value": 270 }, { "step": 28, "value": 270 }, { "step": 29, "value": 270 }, { "step": 30, "value": 270 }, { "step": 31, "value": 270 }, { "step": 32, "value": 270 }, { "step": 33, "value": 240 }, { "step": 34, "value": 240 }, { "step": 35, "value": 240 }, { "step": 36, "value": 240 }, { "step": 37, "value": 240 }, { "step": 38, "value": 240 }, { "step": 39, "value": 240 }, { "step": 40, "value": 240 }, { "step": 41, "value": 240 }, { "step": 42, "value": 240 }, { "step": 43, "value": 240 }, { "step": 44, "value": 240 }, { "step": 45, "value": 240 }, { "step": 46, "value": 240 }, { "step": 47, "value": 240 }, { "step": 48, "value": 240 }, { "step": 49, "value": 240 }, { "step": 50, "value": 240 }, { "step": 51, "value": 240 }, { "step": 52, "value": 240 }, { "step": 53, "value": 240 }, { "step": 54, "value": 240 }, { "step": 55, "value": 240 }, { "step": 56, "value": 240 }, { "step": 57, "value": 240 }, { "step": 58, "value": 240 }, { "step": 59, "value": 240 }, { "step": 60, "value": 240 }, { "step": 61, "value": 240 }, { "step": 62, "value": 240 }, { "step": 63, "value": 240 }, { "step": 64, "value": 240 }, { "step": 65, "value": 240 }, { "step": 66, "value": 240 }, { "step": 67, "value": 240 }, { "step": 68, "value": 240 }, { "step": 69, "value": 240 }, { "step": 70, "value": 240 }, { "step": 71, "value": 240 }, { "step": 72, "value": 240 }, { "step": 73, "value": 240 }, { "step": 74, "value": 240 }, { "step": 75, "value": 240 }, { "step": 76, "value": 240 } ], "submission_sha256": "d23fdf459a93426ab1ce4fd1c5e8911ac8e031412c24a8a6d15fc982a55d6297", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/autoscientists/traintime/submission.json", "comparison_value": 240, "comparison_rank_key": [ 240 ], "rank": 2 }, { "run_id": "arbor--traintime", "method_id": "arbor", "task_id": "traintime", "metric": "train_seconds_to_target", "unit": "s", "direction": "minimize", "best_value": 270.0539448261261, "reference_value": 570.4014205932617, "relative_improvement": 0.5265545717869198, "rank_key": [ 270.0539448261261, 220463104.0, -0.004733982398008818 ], "best_launch_seq": 43, "best_candidate_id": "exp-56e79f0b51216e67a4317582", "best_is_reference": false, "budget": { "spent": 47, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 9.384313888888887, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 570, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-9efaa604543ecd3b8d4a5766", "value": 450, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-c75af7a805ce1e6828a2ee2d", "value": 540, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp-3c0e5ecb11d464b757b5aef7", "value": 450, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-0dbd6a4ea8ca163f4300394d", "value": 540, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp-20ba6f4b88a6cee87597d0b8", "value": 510, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp-e8979372bcae4be0a66cb3ea", "value": 390, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp-d07030cd70943a25a50995c0", "value": 360, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-6cc5000d4553268e1aef3760", "value": 450, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-7a2a4eb5c57732c394fb333d", "value": 300, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-bf13521ecddcd749ae270d1c", "value": 330, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-ff2d06cc80d3805d13433f2b", "value": 300, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-af66d4fdc53a000a16c6f60a", "value": 330, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp-a1eaae131e00f448dd297b6d", "value": 300, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-0ae598026b8e006877f1bcf7", "value": 330, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-eccad79dacd53566cc47dbdd", "value": 300, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-d840e6200050f8f7916b9fee", "value": 300, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-9c9f2f14fb3e43b95274cd67", "value": 300, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-d28cdda187b5011e57faba3b", "value": 300, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp-a397457f2e5c80d1be0c85d1", "value": 510, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp-99d16fc981e3cd4e4462018e", "value": 300, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-a7f03a67c08c09318665050b", "value": 540, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp-d16c6aa25b15802a095dde83", "value": 300, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp-ffe2a3983c567518b71979ab", "value": 300, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp-c0017f621ea92c15fa351a4a", "value": 330, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp-46577fe5ba7b4d39c6c7f14e", "value": 300, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp-00ed98d074916df28d0c5de5", "value": 300, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-2b1480cb47917abeb6de4aae", "value": 300, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-34aac47920cd358c7ad6bb10", "value": 300, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-2949ad4c05e34ccea549139d", "value": 330, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-0195198bf9cb105e7bc5af65", "value": 300, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp-c52bb1493a032ae6b06f33b4", "value": 300, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp-0c57d4d96b771e7842533db6", "value": 330, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp-7199ed585a49ae81d2b4060f", "value": 270, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "exp-cc38a689a0823e9c8e01800e", "value": 300, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp-a0505a208e3b74fed9be5bb9", "value": 330, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "exp-f67b465234cb63c48465e651", "value": 300, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp-56e79f0b51216e67a4317582", "value": 270, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp-abe67416f15bd86f2530d7ea", "value": 300, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp-232b720e6423f2b718d9fd90", "value": 270, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp-6d72c3e8e3952c0ee9df40ab", "value": 300, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp-1049edeb26b4b511fe96666a", "value": 300, "is_reference": false } ], "curve": [ { "step": 1, "value": 570 }, { "step": 2, "value": 450 }, { "step": 3, "value": 450 }, { "step": 4, "value": 450 }, { "step": 5, "value": 450 }, { "step": 6, "value": 450 }, { "step": 7, "value": 450 }, { "step": 8, "value": 390 }, { "step": 9, "value": 360 }, { "step": 10, "value": 360 }, { "step": 11, "value": 300 }, { "step": 12, "value": 300 }, { "step": 13, "value": 300 }, { "step": 14, "value": 300 }, { "step": 15, "value": 300 }, { "step": 16, "value": 300 }, { "step": 17, "value": 300 }, { "step": 18, "value": 300 }, { "step": 19, "value": 300 }, { "step": 20, "value": 300 }, { "step": 21, "value": 300 }, { "step": 22, "value": 300 }, { "step": 23, "value": 300 }, { "step": 24, "value": 300 }, { "step": 25, "value": 300 }, { "step": 26, "value": 300 }, { "step": 27, "value": 300 }, { "step": 28, "value": 300 }, { "step": 29, "value": 300 }, { "step": 30, "value": 300 }, { "step": 31, "value": 300 }, { "step": 32, "value": 300 }, { "step": 33, "value": 300 }, { "step": 34, "value": 300 }, { "step": 35, "value": 300 }, { "step": 36, "value": 300 }, { "step": 37, "value": 300 }, { "step": 38, "value": 300 }, { "step": 39, "value": 270 }, { "step": 40, "value": 270 }, { "step": 41, "value": 270 }, { "step": 42, "value": 270 }, { "step": 43, "value": 270 }, { "step": 44, "value": 270 }, { "step": 45, "value": 270 }, { "step": 46, "value": 270 }, { "step": 47, "value": 270 } ], "submission_sha256": "0ac7807e2c3c14dec8adea8781425fc5375d768874ce49dac09c38da250caf75", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/arbor/traintime/submission.json", "comparison_value": 270, "comparison_rank_key": [ 270 ], "rank": 3 }, { "run_id": "beam-search--traintime", "method_id": "beam-search", "task_id": "traintime", "metric": "train_seconds_to_target", "unit": "s", "direction": "minimize", "best_value": 270.01983523368835, "reference_value": 570.1673939228058, "relative_improvement": 0.5264200687171424, "rank_key": [ 270.01983523368835, 194248704.0, -0.0037351664685638664 ], "best_launch_seq": 72, "best_candidate_id": "g11-c5", "best_is_reference": false, "budget": { "spent": 95, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB", "NVIDIA A100-SXM4-80GB" ], "workers": 4, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 4 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": "bedrock" }, "evaluation_gpu_hours": 18.01779527777778, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 570, "is_reference": true }, { "step": 3, "launch_seq": 3, "candidate_id": "g1-c2", "value": 450, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "g2-c1", "value": 360, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "g2-c2", "value": 480, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "g2-c3", "value": 480, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "g2-c4", "value": 420, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "g3-c1", "value": 330, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "g3-c2", "value": 360, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "g3-c3", "value": 360, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "g3-c4", "value": 420, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "g3-c5", "value": 420, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "g3-c7", "value": 450, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "g3-c8", "value": 480, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "g4-c1", "value": 330, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "g4-c2", "value": 360, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "g4-c3", "value": 330, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "g4-c4", "value": 330, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "g4-c5", "value": 330, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "g4-c7", "value": 360, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "g4-c8", "value": 330, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "g5-c1", "value": 330, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "g5-c2", "value": 330, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "g5-c4", "value": 330, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "g5-c5", "value": 330, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "g5-c6", "value": 330, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "g5-c7", "value": 330, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "g5-c8", "value": 330, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "g6-c1", "value": 330, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "g6-c2", "value": 360, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "g6-c3", "value": 330, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "g6-c4", "value": 330, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "g6-c5", "value": 330, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "g6-c6", "value": 330, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "g6-c7", "value": 300, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "g6-c8", "value": 360, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "g7b-c1", "value": 330, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "g7b-c2", "value": 330, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "g7b-c3", "value": 330, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "g7b-c4", "value": 330, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "g7b-c5", "value": 330, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "g7b-c6", "value": 330, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "g7b-c7", "value": 330, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "g7b-c8", "value": 300, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "g8b-c1", "value": 300, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "g8b-c2", "value": 300, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "g8b-c3", "value": 270, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "g8b-c4", "value": 330, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "g8b-c5", "value": 330, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "g8b-c6", "value": 330, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "g8b-c7", "value": 330, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "g8b-c8", "value": 330, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "g9-c1", "value": 300, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "g9-c2", "value": 300, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "g9-c3", "value": 300, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "g9-c5", "value": 300, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "g9-c6", "value": 300, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "g9-c7", "value": 300, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "g9-c8", "value": 330, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "g10-c1", "value": 270, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "g10-c2", "value": 270, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "g10-c3", "value": 300, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "g10-c4", "value": 330, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "g10-c5", "value": 300, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "g10-c6", "value": 300, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "g10-c7", "value": 270, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "g10-c8", "value": 300, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "g11-c5", "value": 270, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "g11-c6", "value": 300, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "g11-c7", "value": 300, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "g11-c9", "value": 270, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "g11-c10", "value": 270, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "g11-c11", "value": 270, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "g11-c12", "value": 270, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "g12-c1", "value": 300, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "g12-c2", "value": 300, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "g12-c3", "value": 270, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "g12-c4", "value": 270, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "g12-c5", "value": 300, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "g12-c6", "value": 300, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "g12-c7", "value": 300, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "g12-c8", "value": 300, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "g13-c1", "value": 360, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "g13-c2", "value": 270, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "g13-c3", "value": 300, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "g13-c4", "value": 270, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "g13-c5", "value": 270, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "g13-c6", "value": 270, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "g13-c7", "value": 270, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "g13-c8", "value": 270, "is_reference": false } ], "curve": [ { "step": 1, "value": 570 }, { "step": 2, "value": 570 }, { "step": 3, "value": 450 }, { "step": 4, "value": 360 }, { "step": 5, "value": 360 }, { "step": 6, "value": 360 }, { "step": 7, "value": 360 }, { "step": 8, "value": 330 }, { "step": 9, "value": 330 }, { "step": 10, "value": 330 }, { "step": 11, "value": 330 }, { "step": 12, "value": 330 }, { "step": 13, "value": 330 }, { "step": 14, "value": 330 }, { "step": 15, "value": 330 }, { "step": 16, "value": 330 }, { "step": 17, "value": 330 }, { "step": 18, "value": 330 }, { "step": 19, "value": 330 }, { "step": 20, "value": 330 }, { "step": 21, "value": 330 }, { "step": 22, "value": 330 }, { "step": 23, "value": 330 }, { "step": 24, "value": 330 }, { "step": 25, "value": 330 }, { "step": 26, "value": 330 }, { "step": 27, "value": 330 }, { "step": 28, "value": 330 }, { "step": 29, "value": 330 }, { "step": 30, "value": 330 }, { "step": 31, "value": 330 }, { "step": 32, "value": 330 }, { "step": 33, "value": 330 }, { "step": 34, "value": 330 }, { "step": 35, "value": 330 }, { "step": 36, "value": 330 }, { "step": 37, "value": 330 }, { "step": 38, "value": 300 }, { "step": 39, "value": 300 }, { "step": 40, "value": 300 }, { "step": 41, "value": 300 }, { "step": 42, "value": 300 }, { "step": 43, "value": 300 }, { "step": 44, "value": 300 }, { "step": 45, "value": 300 }, { "step": 46, "value": 300 }, { "step": 47, "value": 300 }, { "step": 48, "value": 300 }, { "step": 49, "value": 300 }, { "step": 50, "value": 270 }, { "step": 51, "value": 270 }, { "step": 52, "value": 270 }, { "step": 53, "value": 270 }, { "step": 54, "value": 270 }, { "step": 55, "value": 270 }, { "step": 56, "value": 270 }, { "step": 57, "value": 270 }, { "step": 58, "value": 270 }, { "step": 59, "value": 270 }, { "step": 60, "value": 270 }, { "step": 61, "value": 270 }, { "step": 62, "value": 270 }, { "step": 63, "value": 270 }, { "step": 64, "value": 270 }, { "step": 65, "value": 270 }, { "step": 66, "value": 270 }, { "step": 67, "value": 270 }, { "step": 68, "value": 270 }, { "step": 69, "value": 270 }, { "step": 70, "value": 270 }, { "step": 71, "value": 270 }, { "step": 72, "value": 270 }, { "step": 73, "value": 270 }, { "step": 74, "value": 270 }, { "step": 75, "value": 270 }, { "step": 76, "value": 270 }, { "step": 77, "value": 270 }, { "step": 78, "value": 270 }, { "step": 79, "value": 270 }, { "step": 80, "value": 270 }, { "step": 81, "value": 270 }, { "step": 82, "value": 270 }, { "step": 83, "value": 270 }, { "step": 84, "value": 270 }, { "step": 85, "value": 270 }, { "step": 86, "value": 270 }, { "step": 87, "value": 270 }, { "step": 88, "value": 270 }, { "step": 89, "value": 270 }, { "step": 90, "value": 270 }, { "step": 91, "value": 270 }, { "step": 92, "value": 270 }, { "step": 93, "value": 270 }, { "step": 94, "value": 270 }, { "step": 95, "value": 270 } ], "submission_sha256": "7d703373b6a6ed8968dcb93580636b982860727a9518bef9b7af7f1f75a6d66c", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/beam-search/traintime/submission.json", "comparison_value": 270, "comparison_rank_key": [ 270 ], "rank": 3 }, { "run_id": "gear--traintime", "method_id": "gear", "task_id": "traintime", "metric": "train_seconds_to_target", "unit": "s", "direction": "minimize", "best_value": 270.0019431114197, "reference_value": 570.4027774333954, "relative_improvement": 0.5266468646482929, "rank_key": [ 270.0019431114197, 232390656.0, -0.007882662718292632 ], "best_launch_seq": 64, "best_candidate_id": "exp0064-3eda371", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": "us.anthropic.claude-opus-5[1m]", "provider": null }, "evaluation_gpu_hours": 16.825998611111107, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 570, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp0002-bd5fa04", "value": 420, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp0003-2e827a7", "value": 450, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp0004-8c78a95", "value": 570, "is_reference": false }, { "step": 5, "launch_seq": 5, "candidate_id": "exp0005-045790d", "value": 540, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp0006-cc84256", "value": 330, "is_reference": false }, { "step": 7, "launch_seq": 7, "candidate_id": "exp0007-8e92f0c", "value": 450, "is_reference": false }, { "step": 8, "launch_seq": 8, "candidate_id": "exp0008-c36521e", "value": 540, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp0009-065128d", "value": 360, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp0010-8c7250c", "value": 360, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp0011-0056a20", "value": 300, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp0012-0157779", "value": 270, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp0013-db86fe4", "value": 330, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp0014-915cfd1", "value": 360, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp0015-5965da5", "value": 270, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp0016-81826ea", "value": 330, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp0017-fe025cf", "value": 300, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp0018-e77c936", "value": 300, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp0019-f08ea9a", "value": 360, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp0020-3af45d7", "value": 300, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp0021-b99f0bd", "value": 270, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp0022-648812a", "value": 300, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp0023-a69a0e4", "value": 270, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp0024-16846f2", "value": 330, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp0025-ff860dc", "value": 330, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp0026-38a14da", "value": 330, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp0027-8934ff0", "value": 300, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp0028-3f43ff2", "value": 270, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp0029-fddea92", "value": 270, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp0030-46080a9", "value": 270, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp0031-c8c5188", "value": 270, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp0032-5e2bb86", "value": 300, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp0033-dde70fd", "value": 270, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp0034-971fbaa", "value": 300, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp0035-879050e", "value": 270, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp0036-9f1d905", "value": 270, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp0037-1c3c642", "value": 300, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp0038-e9df306", "value": 270, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp0039-47bc76f", "value": 270, "is_reference": false }, { "step": 40, "launch_seq": 40, "candidate_id": "exp0040-1c80c4f", "value": 270, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp0041-ff2b52d", "value": 270, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "exp0042-15d1e7a", "value": 270, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp0043-8007e34", "value": 270, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp0044-fefa746", "value": 270, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp0045-822653a", "value": 270, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp0046-e24eb09", "value": 270, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp0047-e0285c4", "value": 270, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp0048-afe0ef6", "value": 300, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "exp0049-6dbd42b", "value": 270, "is_reference": false }, { "step": 50, "launch_seq": 50, "candidate_id": "exp0050-8d08a2b", "value": 300, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp0051-40de571", "value": 270, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp0052-595122c", "value": 270, "is_reference": false }, { "step": 53, "launch_seq": 53, "candidate_id": "exp0053-aebdff9", "value": 270, "is_reference": false }, { "step": 54, "launch_seq": 54, "candidate_id": "exp0054-427d120", "value": 270, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp0055-aa89f4b", "value": 270, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp0056-645abbe", "value": 270, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "exp0057-14cc04a", "value": 270, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp0058-2d5d4b5", "value": 270, "is_reference": false }, { "step": 59, "launch_seq": 59, "candidate_id": "exp0059-bf03247", "value": 270, "is_reference": false }, { "step": 60, "launch_seq": 60, "candidate_id": "exp0060-f144e6f", "value": 270, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "exp0061-6b379a6", "value": 270, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "exp0062-3bf71a1", "value": 270, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "exp0063-774bc4d", "value": 330, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp0064-3eda371", "value": 270, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "exp0065-64524f4", "value": 300, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "exp0066-b1e56c8", "value": 270, "is_reference": false }, { "step": 67, "launch_seq": 67, "candidate_id": "exp0067-685c7b6", "value": 270, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "exp0068-b90ea68", "value": 330, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "exp0069-44d979b", "value": 570, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "exp0070-379d72c", "value": 300, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp0071-a596c4a", "value": 330, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "exp0072-efa50b4", "value": 300, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "exp0073-a860f22", "value": 270, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "exp0074-676ff4b", "value": 300, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "exp0075-554256a", "value": 330, "is_reference": false }, { "step": 76, "launch_seq": 76, "candidate_id": "exp0076-262652d", "value": 360, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "exp0077-f124620", "value": 300, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "exp0078-353f919", "value": 270, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "exp0079-958fad4", "value": 270, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "exp0080-c8eedad", "value": 270, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "exp0081-6bae83b", "value": 270, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "exp0082-c6dec3e", "value": 270, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "exp0083-8dd92d4", "value": 360, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "exp0084-3bf8e3e", "value": 360, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "exp0085-0ebbc55", "value": 300, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp0086-53d2228", "value": 270, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "exp0087-462d34c", "value": 300, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "exp0088-f2c4f4c", "value": 270, "is_reference": false }, { "step": 89, "launch_seq": 89, "candidate_id": "exp0089-6edfed6", "value": 270, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "exp0090-d61d265", "value": 330, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "exp0091-54c372f", "value": 270, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "exp0092-8a139a0", "value": 270, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "exp0093-a83671d", "value": 300, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "exp0094-06536f0", "value": 300, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "exp0095-15f9b6c", "value": 270, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "exp0096-f41190d", "value": 270, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "exp0097-62c5f74", "value": 270, "is_reference": false }, { "step": 98, "launch_seq": 98, "candidate_id": "exp0098-f9cc2d5", "value": 270, "is_reference": false }, { "step": 99, "launch_seq": 99, "candidate_id": "exp0099-0d46b84", "value": 270, "is_reference": false }, { "step": 100, "launch_seq": 100, "candidate_id": "exp0100-8778e8a", "value": 270, "is_reference": false } ], "curve": [ { "step": 1, "value": 570 }, { "step": 2, "value": 420 }, { "step": 3, "value": 420 }, { "step": 4, "value": 420 }, { "step": 5, "value": 420 }, { "step": 6, "value": 330 }, { "step": 7, "value": 330 }, { "step": 8, "value": 330 }, { "step": 9, "value": 330 }, { "step": 10, "value": 330 }, { "step": 11, "value": 300 }, { "step": 12, "value": 270 }, { "step": 13, "value": 270 }, { "step": 14, "value": 270 }, { "step": 15, "value": 270 }, { "step": 16, "value": 270 }, { "step": 17, "value": 270 }, { "step": 18, "value": 270 }, { "step": 19, "value": 270 }, { "step": 20, "value": 270 }, { "step": 21, "value": 270 }, { "step": 22, "value": 270 }, { "step": 23, "value": 270 }, { "step": 24, "value": 270 }, { "step": 25, "value": 270 }, { "step": 26, "value": 270 }, { "step": 27, "value": 270 }, { "step": 28, "value": 270 }, { "step": 29, "value": 270 }, { "step": 30, "value": 270 }, { "step": 31, "value": 270 }, { "step": 32, "value": 270 }, { "step": 33, "value": 270 }, { "step": 34, "value": 270 }, { "step": 35, "value": 270 }, { "step": 36, "value": 270 }, { "step": 37, "value": 270 }, { "step": 38, "value": 270 }, { "step": 39, "value": 270 }, { "step": 40, "value": 270 }, { "step": 41, "value": 270 }, { "step": 42, "value": 270 }, { "step": 43, "value": 270 }, { "step": 44, "value": 270 }, { "step": 45, "value": 270 }, { "step": 46, "value": 270 }, { "step": 47, "value": 270 }, { "step": 48, "value": 270 }, { "step": 49, "value": 270 }, { "step": 50, "value": 270 }, { "step": 51, "value": 270 }, { "step": 52, "value": 270 }, { "step": 53, "value": 270 }, { "step": 54, "value": 270 }, { "step": 55, "value": 270 }, { "step": 56, "value": 270 }, { "step": 57, "value": 270 }, { "step": 58, "value": 270 }, { "step": 59, "value": 270 }, { "step": 60, "value": 270 }, { "step": 61, "value": 270 }, { "step": 62, "value": 270 }, { "step": 63, "value": 270 }, { "step": 64, "value": 270 }, { "step": 65, "value": 270 }, { "step": 66, "value": 270 }, { "step": 67, "value": 270 }, { "step": 68, "value": 270 }, { "step": 69, "value": 270 }, { "step": 70, "value": 270 }, { "step": 71, "value": 270 }, { "step": 72, "value": 270 }, { "step": 73, "value": 270 }, { "step": 74, "value": 270 }, { "step": 75, "value": 270 }, { "step": 76, "value": 270 }, { "step": 77, "value": 270 }, { "step": 78, "value": 270 }, { "step": 79, "value": 270 }, { "step": 80, "value": 270 }, { "step": 81, "value": 270 }, { "step": 82, "value": 270 }, { "step": 83, "value": 270 }, { "step": 84, "value": 270 }, { "step": 85, "value": 270 }, { "step": 86, "value": 270 }, { "step": 87, "value": 270 }, { "step": 88, "value": 270 }, { "step": 89, "value": 270 }, { "step": 90, "value": 270 }, { "step": 91, "value": 270 }, { "step": 92, "value": 270 }, { "step": 93, "value": 270 }, { "step": 94, "value": 270 }, { "step": 95, "value": 270 }, { "step": 96, "value": 270 }, { "step": 97, "value": 270 }, { "step": 98, "value": 270 }, { "step": 99, "value": 270 }, { "step": 100, "value": 270 } ], "submission_sha256": "afd13ac6b13ae1f4e968f6e8670c3a3258e46453d309dbe4ccdbc3de41ab1883", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/gear/traintime/submission.json", "comparison_value": 270, "comparison_rank_key": [ 270 ], "rank": 3 }, { "run_id": "heuresis-map-elites--traintime", "method_id": "heuresis-map-elites", "task_id": "traintime", "metric": "train_seconds_to_target", "unit": "s", "direction": "minimize", "best_value": 270.09792137145996, "reference_value": 570.7785224914551, "relative_improvement": 0.526790321064501, "rank_key": [ 270.09792137145996, 168820736.0, -0.000845781340318208 ], "best_launch_seq": 100, "best_candidate_id": "exp-e0c3fe127eccc7634c8ea2a4", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 28.06571694444443, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 570, "is_reference": true }, { "step": 2, "launch_seq": 2, "candidate_id": "exp-3500b4b89ceffd19a7bdbed5", "value": 540, "is_reference": false }, { "step": 3, "launch_seq": 3, "candidate_id": "exp-42a3d012e149d9f09ef7218d", "value": 510, "is_reference": false }, { "step": 4, "launch_seq": 4, "candidate_id": "exp-982403a0f9702a7eaaba2102", "value": 540, "is_reference": false }, { "step": 6, "launch_seq": 6, "candidate_id": "exp-1e1686e60a4722c60c6deae6", "value": 510, "is_reference": false }, { "step": 9, "launch_seq": 9, "candidate_id": "exp-3720568896dc0db26a804b4e", "value": 420, "is_reference": false }, { "step": 10, "launch_seq": 10, "candidate_id": "exp-06e10ec4b59087122dac0347", "value": 360, "is_reference": false }, { "step": 11, "launch_seq": 11, "candidate_id": "exp-05d2ba9a1a5e496ad72ed651", "value": 360, "is_reference": false }, { "step": 12, "launch_seq": 12, "candidate_id": "exp-c0b93018fb0ebf5568bc76b6", "value": 360, "is_reference": false }, { "step": 13, "launch_seq": 13, "candidate_id": "exp-38a910704e683b6a6b3bdf3f", "value": 570, "is_reference": false }, { "step": 14, "launch_seq": 14, "candidate_id": "exp-2d4ecc798be2560e6a5d4680", "value": 570, "is_reference": false }, { "step": 15, "launch_seq": 15, "candidate_id": "exp-b739f4a1ce1c53e6fbdfe963", "value": 570, "is_reference": false }, { "step": 16, "launch_seq": 16, "candidate_id": "exp-0f1f7f2be48a0812fe1206b3", "value": 570, "is_reference": false }, { "step": 17, "launch_seq": 17, "candidate_id": "exp-c799dc12c061e400a9f3bf70", "value": 570, "is_reference": false }, { "step": 18, "launch_seq": 18, "candidate_id": "exp-a4bc1f5055751ae6d4904a8a", "value": 540, "is_reference": false }, { "step": 19, "launch_seq": 19, "candidate_id": "exp-e2e604b31182702ba39985e1", "value": 570, "is_reference": false }, { "step": 20, "launch_seq": 20, "candidate_id": "exp-9ed7da024f7d0995c8287bf5", "value": 540, "is_reference": false }, { "step": 21, "launch_seq": 21, "candidate_id": "exp-47c50b7d5dbdf096f026b2d6", "value": 540, "is_reference": false }, { "step": 22, "launch_seq": 22, "candidate_id": "exp-851832eb2bc19ad4244e6d1a", "value": 510, "is_reference": false }, { "step": 23, "launch_seq": 23, "candidate_id": "exp-82d4e3b956882b651d6c01b7", "value": 570, "is_reference": false }, { "step": 24, "launch_seq": 24, "candidate_id": "exp-beb30068c7556fd6c9c627c5", "value": 540, "is_reference": false }, { "step": 25, "launch_seq": 25, "candidate_id": "exp-73a4479ceeaf0f7f424bf5a9", "value": 540, "is_reference": false }, { "step": 26, "launch_seq": 26, "candidate_id": "exp-44d349ad64c8579a2808fed4", "value": 540, "is_reference": false }, { "step": 27, "launch_seq": 27, "candidate_id": "exp-61d7931572904c10a46cd996", "value": 540, "is_reference": false }, { "step": 28, "launch_seq": 28, "candidate_id": "exp-90ef026357f5d3982bfe1759", "value": 510, "is_reference": false }, { "step": 29, "launch_seq": 29, "candidate_id": "exp-97cddfa8c9cce228debe4646", "value": 510, "is_reference": false }, { "step": 30, "launch_seq": 30, "candidate_id": "exp-0794cb0bcaffd51c8ed1e3d1", "value": 570, "is_reference": false }, { "step": 31, "launch_seq": 31, "candidate_id": "exp-bc1de0a576a43531c6714250", "value": 510, "is_reference": false }, { "step": 32, "launch_seq": 32, "candidate_id": "exp-7c0dd2c5d3618085c7e3c1a0", "value": 510, "is_reference": false }, { "step": 33, "launch_seq": 33, "candidate_id": "exp-92d301297f68ae9f5baaa34e", "value": 510, "is_reference": false }, { "step": 34, "launch_seq": 34, "candidate_id": "exp-171d071be2c977246057e6b4", "value": 540, "is_reference": false }, { "step": 35, "launch_seq": 35, "candidate_id": "exp-39ca2895617d5a48ea906397", "value": 510, "is_reference": false }, { "step": 36, "launch_seq": 36, "candidate_id": "exp-d2a03db353ebbdeecd471089", "value": 480, "is_reference": false }, { "step": 37, "launch_seq": 37, "candidate_id": "exp-4c6bc5b9741af59c134d1f61", "value": 480, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "exp-a898f4b5bfd3fcd4f31d21a5", "value": 540, "is_reference": false }, { "step": 39, "launch_seq": 39, "candidate_id": "exp-087fd2e36c1511157c5d0189", "value": 480, "is_reference": false }, { "step": 41, "launch_seq": 41, "candidate_id": "exp-347ffc0e57258470f637e616", "value": 540, "is_reference": false }, { "step": 42, "launch_seq": 42, "candidate_id": "exp-15bce93f946e1fe7897366b4", "value": 510, "is_reference": false }, { "step": 43, "launch_seq": 43, "candidate_id": "exp-2d0d4efe79bf19815fa2cb91", "value": 570, "is_reference": false }, { "step": 44, "launch_seq": 44, "candidate_id": "exp-bf84011456c6f6956307bd3c", "value": 570, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "exp-017125bdf3dc59242c44bc59", "value": 510, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "exp-af0131f973844fd12d9ca427", "value": 510, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "exp-ab0fc322bf9e0b63fbab9109", "value": 510, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "exp-cfa67bca20fadd109b217e7d", "value": 510, "is_reference": false }, { "step": 49, "launch_seq": 49, "candidate_id": "exp-cd105808409012f87c5a8fee", "value": 540, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "exp-bd598e447c31350035bc7840", "value": 600, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "exp-5d07963d0bec9c77ad9bfc7d", "value": 570, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "exp-1ab7349179c67044fd5eb045", "value": 570, "is_reference": false }, { "step": 56, "launch_seq": 56, "candidate_id": "exp-9ee37311a160f83f5b19bc9b", "value": 600, "is_reference": false }, { "step": 58, "launch_seq": 58, "candidate_id": "exp-46dc3eaf59b4a3baf43ac3ff", "value": 600, "is_reference": false }, { "step": 64, "launch_seq": 64, "candidate_id": "exp-2ece2fbe575323b2932f8fe3", "value": 570, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "exp-3c6b9f582e475292f3ee5248", "value": 360, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "exp-3dd3e5c7ae29fab3b319e784", "value": 330, "is_reference": false }, { "step": 71, "launch_seq": 71, "candidate_id": "exp-22aa400a206c177e528e81e6", "value": 570, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "exp-635952a2311484fe022d3c7c", "value": 510, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "exp-fecd0da830c81d470e64a015", "value": 510, "is_reference": false }, { "step": 75, "launch_seq": 75, "candidate_id": "exp-4b6a6342b0d701165e1de1cc", "value": 540, "is_reference": false }, { "step": 77, "launch_seq": 77, "candidate_id": "exp-06aa459a3ddf8571bc728e50", "value": 450, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "exp-e704e5d68a8fd5093b03227b", "value": 450, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "exp-679b6cda50dd9e2126d24266", "value": 540, "is_reference": false }, { "step": 83, "launch_seq": 83, "candidate_id": "exp-0f9a484a25714799c43fbc8f", "value": 570, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "exp-4c5258b72c4bc7ed095505f7", "value": 540, "is_reference": false }, { "step": 86, "launch_seq": 86, "candidate_id": "exp-755a4189953d67e75d452f48", "value": 510, "is_reference": false }, { "step": 87, "launch_seq": 87, "candidate_id": "exp-6f27987dd20aad5e1a8cc058", "value": 540, "is_reference": false }, { "step": 88, "launch_seq": 88, "candidate_id": "exp-173d06fb393da05d9139adb6", "value": 540, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "exp-f22062c3e7efa5adc628da2a", "value": 540, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "exp-202b3e06818af7204c9ba8e5", "value": 540, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "exp-29b97ddacac28d5045fdcab5", "value": 600, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "exp-9f666094b765896de11f0318", "value": 570, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "exp-892bd8a710300bf350eeb55a", "value": 600, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "exp-5bc06e548295a04109ea565b", "value": 540, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "exp-dfb84f6a6dd2fd33ee0b384e", "value": 570, "is_reference": false }, { "step": 98, "launch_seq": 98, "candidate_id": "exp-8fa56c240f661d4f0474fe91", "value": 510, "is_reference": false }, { "step": 99, "launch_seq": 99, "candidate_id": "exp-7498c3a4a0f9e1ba946c80a3", "value": 270, "is_reference": false }, { "step": 100, "launch_seq": 100, "candidate_id": "exp-e0c3fe127eccc7634c8ea2a4", "value": 270, "is_reference": false } ], "curve": [ { "step": 1, "value": 570 }, { "step": 2, "value": 540 }, { "step": 3, "value": 510 }, { "step": 4, "value": 510 }, { "step": 5, "value": 510 }, { "step": 6, "value": 510 }, { "step": 7, "value": 510 }, { "step": 8, "value": 510 }, { "step": 9, "value": 420 }, { "step": 10, "value": 360 }, { "step": 11, "value": 360 }, { "step": 12, "value": 360 }, { "step": 13, "value": 360 }, { "step": 14, "value": 360 }, { "step": 15, "value": 360 }, { "step": 16, "value": 360 }, { "step": 17, "value": 360 }, { "step": 18, "value": 360 }, { "step": 19, "value": 360 }, { "step": 20, "value": 360 }, { "step": 21, "value": 360 }, { "step": 22, "value": 360 }, { "step": 23, "value": 360 }, { "step": 24, "value": 360 }, { "step": 25, "value": 360 }, { "step": 26, "value": 360 }, { "step": 27, "value": 360 }, { "step": 28, "value": 360 }, { "step": 29, "value": 360 }, { "step": 30, "value": 360 }, { "step": 31, "value": 360 }, { "step": 32, "value": 360 }, { "step": 33, "value": 360 }, { "step": 34, "value": 360 }, { "step": 35, "value": 360 }, { "step": 36, "value": 360 }, { "step": 37, "value": 360 }, { "step": 38, "value": 360 }, { "step": 39, "value": 360 }, { "step": 40, "value": 360 }, { "step": 41, "value": 360 }, { "step": 42, "value": 360 }, { "step": 43, "value": 360 }, { "step": 44, "value": 360 }, { "step": 45, "value": 360 }, { "step": 46, "value": 360 }, { "step": 47, "value": 360 }, { "step": 48, "value": 360 }, { "step": 49, "value": 360 }, { "step": 50, "value": 360 }, { "step": 51, "value": 360 }, { "step": 52, "value": 360 }, { "step": 53, "value": 360 }, { "step": 54, "value": 360 }, { "step": 55, "value": 360 }, { "step": 56, "value": 360 }, { "step": 57, "value": 360 }, { "step": 58, "value": 360 }, { "step": 59, "value": 360 }, { "step": 60, "value": 360 }, { "step": 61, "value": 360 }, { "step": 62, "value": 360 }, { "step": 63, "value": 360 }, { "step": 64, "value": 360 }, { "step": 65, "value": 360 }, { "step": 66, "value": 360 }, { "step": 67, "value": 360 }, { "step": 68, "value": 360 }, { "step": 69, "value": 330 }, { "step": 70, "value": 330 }, { "step": 71, "value": 330 }, { "step": 72, "value": 330 }, { "step": 73, "value": 330 }, { "step": 74, "value": 330 }, { "step": 75, "value": 330 }, { "step": 76, "value": 330 }, { "step": 77, "value": 330 }, { "step": 78, "value": 330 }, { "step": 79, "value": 330 }, { "step": 80, "value": 330 }, { "step": 81, "value": 330 }, { "step": 82, "value": 330 }, { "step": 83, "value": 330 }, { "step": 84, "value": 330 }, { "step": 85, "value": 330 }, { "step": 86, "value": 330 }, { "step": 87, "value": 330 }, { "step": 88, "value": 330 }, { "step": 89, "value": 330 }, { "step": 90, "value": 330 }, { "step": 91, "value": 330 }, { "step": 92, "value": 330 }, { "step": 93, "value": 330 }, { "step": 94, "value": 330 }, { "step": 95, "value": 330 }, { "step": 96, "value": 330 }, { "step": 97, "value": 330 }, { "step": 98, "value": 330 }, { "step": 99, "value": 270 }, { "step": 100, "value": 270 } ], "submission_sha256": "06b7b4e2681af1eca098d91ce1fa461c0e511c09e8501c376cd0018175bbf70b", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/heuresis-map-elites/traintime/submission.json", "comparison_value": 270, "comparison_rank_key": [ 270 ], "rank": 3 }, { "run_id": "tpe--traintime", "method_id": "tpe", "task_id": "traintime", "metric": "train_seconds_to_target", "unit": "s", "direction": "minimize", "best_value": 510.0716371536255, "reference_value": 600.6067867279053, "relative_improvement": 0.15073947143939817, "rank_key": [ 510.0716371536255, 349503488.0, -0.00463630408097826 ], "best_launch_seq": 97, "best_candidate_id": "tpe-trial-0095", "best_is_reference": false, "budget": { "spent": 100, "max_launches": 100 }, "compute": { "backend": "pluto", "gpu_instance_type": "p4de.24xlarge", "gpu_model": [ "NVIDIA A100-SXM4-80GB" ], "workers": 1, "gpus_per_evaluation": 1, "evaluation_capacity_gpus": 1 }, "research_llm": { "model": null, "provider": null }, "evaluation_gpu_hours": 23.277646388888886, "points": [ { "step": 1, "launch_seq": 1, "candidate_id": "reference", "value": 600, "is_reference": true }, { "step": 37, "launch_seq": 37, "candidate_id": "tpe-trial-0036", "value": 570, "is_reference": false }, { "step": 38, "launch_seq": 38, "candidate_id": "tpe-trial-0037", "value": 600, "is_reference": false }, { "step": 45, "launch_seq": 45, "candidate_id": "tpe-trial-0044", "value": 570, "is_reference": false }, { "step": 46, "launch_seq": 46, "candidate_id": "tpe-trial-0045", "value": 570, "is_reference": false }, { "step": 47, "launch_seq": 47, "candidate_id": "tpe-trial-0046", "value": 600, "is_reference": false }, { "step": 48, "launch_seq": 48, "candidate_id": "tpe-trial-0047", "value": 600, "is_reference": false }, { "step": 51, "launch_seq": 51, "candidate_id": "tpe-trial-0050", "value": 570, "is_reference": false }, { "step": 52, "launch_seq": 52, "candidate_id": "tpe-trial-0051", "value": 600, "is_reference": false }, { "step": 55, "launch_seq": 55, "candidate_id": "tpe-trial-0054", "value": 570, "is_reference": false }, { "step": 57, "launch_seq": 57, "candidate_id": "tpe-trial-0056", "value": 540, "is_reference": false }, { "step": 61, "launch_seq": 61, "candidate_id": "tpe-trial-0060", "value": 570, "is_reference": false }, { "step": 62, "launch_seq": 62, "candidate_id": "tpe-trial-0061", "value": 570, "is_reference": false }, { "step": 63, "launch_seq": 63, "candidate_id": "tpe-trial-0062", "value": 570, "is_reference": false }, { "step": 65, "launch_seq": 65, "candidate_id": "tpe-trial-0064", "value": 540, "is_reference": false }, { "step": 66, "launch_seq": 66, "candidate_id": "tpe-trial-0065", "value": 600, "is_reference": false }, { "step": 68, "launch_seq": 68, "candidate_id": "tpe-trial-0067", "value": 570, "is_reference": false }, { "step": 69, "launch_seq": 69, "candidate_id": "tpe-trial-0068", "value": 570, "is_reference": false }, { "step": 70, "launch_seq": 70, "candidate_id": "tpe-trial-0069", "value": 540, "is_reference": false }, { "step": 72, "launch_seq": 72, "candidate_id": "tpe-trial-0071", "value": 540, "is_reference": false }, { "step": 73, "launch_seq": 73, "candidate_id": "tpe-trial-0072", "value": 540, "is_reference": false }, { "step": 74, "launch_seq": 74, "candidate_id": "tpe-trial-0073", "value": 540, "is_reference": false }, { "step": 78, "launch_seq": 78, "candidate_id": "tpe-trial-0076", "value": 540, "is_reference": false }, { "step": 79, "launch_seq": 79, "candidate_id": "tpe-trial-0077", "value": 540, "is_reference": false }, { "step": 80, "launch_seq": 80, "candidate_id": "tpe-trial-0078", "value": 570, "is_reference": false }, { "step": 81, "launch_seq": 81, "candidate_id": "tpe-trial-0079", "value": 570, "is_reference": false }, { "step": 82, "launch_seq": 82, "candidate_id": "tpe-trial-0080", "value": 540, "is_reference": false }, { "step": 84, "launch_seq": 84, "candidate_id": "tpe-trial-0082", "value": 570, "is_reference": false }, { "step": 85, "launch_seq": 85, "candidate_id": "tpe-trial-0083", "value": 600, "is_reference": false }, { "step": 90, "launch_seq": 90, "candidate_id": "tpe-trial-0088", "value": 540, "is_reference": false }, { "step": 91, "launch_seq": 91, "candidate_id": "tpe-trial-0089", "value": 540, "is_reference": false }, { "step": 92, "launch_seq": 92, "candidate_id": "tpe-trial-0090", "value": 540, "is_reference": false }, { "step": 93, "launch_seq": 93, "candidate_id": "tpe-trial-0091", "value": 570, "is_reference": false }, { "step": 94, "launch_seq": 94, "candidate_id": "tpe-trial-0092", "value": 510, "is_reference": false }, { "step": 95, "launch_seq": 95, "candidate_id": "tpe-trial-0093", "value": 540, "is_reference": false }, { "step": 96, "launch_seq": 96, "candidate_id": "tpe-trial-0094", "value": 600, "is_reference": false }, { "step": 97, "launch_seq": 97, "candidate_id": "tpe-trial-0095", "value": 510, "is_reference": false }, { "step": 99, "launch_seq": 99, "candidate_id": "tpe-trial-0097", "value": 570, "is_reference": false } ], "curve": [ { "step": 1, "value": 600 }, { "step": 2, "value": 600 }, { "step": 3, "value": 600 }, { "step": 4, "value": 600 }, { "step": 5, "value": 600 }, { "step": 6, "value": 600 }, { "step": 7, "value": 600 }, { "step": 8, "value": 600 }, { "step": 9, "value": 600 }, { "step": 10, "value": 600 }, { "step": 11, "value": 600 }, { "step": 12, "value": 600 }, { "step": 13, "value": 600 }, { "step": 14, "value": 600 }, { "step": 15, "value": 600 }, { "step": 16, "value": 600 }, { "step": 17, "value": 600 }, { "step": 18, "value": 600 }, { "step": 19, "value": 600 }, { "step": 20, "value": 600 }, { "step": 21, "value": 600 }, { "step": 22, "value": 600 }, { "step": 23, "value": 600 }, { "step": 24, "value": 600 }, { "step": 25, "value": 600 }, { "step": 26, "value": 600 }, { "step": 27, "value": 600 }, { "step": 28, "value": 600 }, { "step": 29, "value": 600 }, { "step": 30, "value": 600 }, { "step": 31, "value": 600 }, { "step": 32, "value": 600 }, { "step": 33, "value": 600 }, { "step": 34, "value": 600 }, { "step": 35, "value": 600 }, { "step": 36, "value": 600 }, { "step": 37, "value": 570 }, { "step": 38, "value": 570 }, { "step": 39, "value": 570 }, { "step": 40, "value": 570 }, { "step": 41, "value": 570 }, { "step": 42, "value": 570 }, { "step": 43, "value": 570 }, { "step": 44, "value": 570 }, { "step": 45, "value": 570 }, { "step": 46, "value": 570 }, { "step": 47, "value": 570 }, { "step": 48, "value": 570 }, { "step": 49, "value": 570 }, { "step": 50, "value": 570 }, { "step": 51, "value": 570 }, { "step": 52, "value": 570 }, { "step": 53, "value": 570 }, { "step": 54, "value": 570 }, { "step": 55, "value": 570 }, { "step": 56, "value": 570 }, { "step": 57, "value": 540 }, { "step": 58, "value": 540 }, { "step": 59, "value": 540 }, { "step": 60, "value": 540 }, { "step": 61, "value": 540 }, { "step": 62, "value": 540 }, { "step": 63, "value": 540 }, { "step": 64, "value": 540 }, { "step": 65, "value": 540 }, { "step": 66, "value": 540 }, { "step": 67, "value": 540 }, { "step": 68, "value": 540 }, { "step": 69, "value": 540 }, { "step": 70, "value": 540 }, { "step": 71, "value": 540 }, { "step": 72, "value": 540 }, { "step": 73, "value": 540 }, { "step": 74, "value": 540 }, { "step": 75, "value": 540 }, { "step": 76, "value": 540 }, { "step": 77, "value": 540 }, { "step": 78, "value": 540 }, { "step": 79, "value": 540 }, { "step": 80, "value": 540 }, { "step": 81, "value": 540 }, { "step": 82, "value": 540 }, { "step": 83, "value": 540 }, { "step": 84, "value": 540 }, { "step": 85, "value": 540 }, { "step": 86, "value": 540 }, { "step": 87, "value": 540 }, { "step": 88, "value": 540 }, { "step": 89, "value": 540 }, { "step": 90, "value": 540 }, { "step": 91, "value": 540 }, { "step": 92, "value": 540 }, { "step": 93, "value": 540 }, { "step": 94, "value": 510 }, { "step": 95, "value": 510 }, { "step": 96, "value": 510 }, { "step": 97, "value": 510 }, { "step": 98, "value": 510 }, { "step": 99, "value": 510 }, { "step": 100, "value": 510 } ], "submission_sha256": "a5feab6f8aef2825a423bdeb5683e2efdbae35739e7dfcd0bbc880efedd3cb93", "code_url": null, "logs_url": null, "report_url": null, "submission_url": "submissions/tpe/traintime/submission.json", "comparison_value": 510, "comparison_rank_key": [ 510 ], "rank": 7 } ] } ], "overall": { "methods": [ { "method_id": "autoscientists", "cells": { "params": { "rank": 2, "value": 13910060, "run_id": "autoscientists--params", "report_url": null }, "flops": { "rank": 1, "value": 52148760, "run_id": "autoscientists--flops", "report_url": null }, "memory": { "rank": 3, "value": 559877120, "run_id": "autoscientists--memory", "report_url": null }, "traintime": { "rank": 2, "value": 240, "run_id": "autoscientists--traintime", "report_url": null }, "steps": { "rank": 1, "value": 267, "run_id": "autoscientists--steps", "report_url": null }, "data": { "rank": 5, "value": 184549376, "run_id": "autoscientists--data", "report_url": null }, "kvcache": { "rank": 2, "value": 59392, "run_id": "autoscientists--kvcache", "report_url": null }, "decode": { "rank": 1, "value": 49.695491790771484, "run_id": "autoscientists--decode", "report_url": null }, "request": { "rank": 1, "value": 29.457807540893555, "run_id": "autoscientists--request", "report_url": null } }, "coverage": 9, "matches": 54, "elo": 1729.3701786268632, "match_order_sd": 16.007790631060917, "rank": 1 }, { "method_id": "sequential-search", "cells": { "params": { "rank": 5, "value": 15151312, "run_id": "sequential-search--params", "report_url": null }, "flops": { "rank": 2, "value": 56663808, "run_id": "sequential-search--flops", "report_url": null }, "memory": { "rank": 4, "value": 608924672, "run_id": "sequential-search--memory", "report_url": null }, "traintime": { "rank": 1, "value": 210, "run_id": "sequential-search--traintime", "report_url": null }, "steps": { "rank": 2, "value": 295, "run_id": "sequential-search--steps", "report_url": null }, "data": { "rank": 1, "value": 152455168, "run_id": "sequential-search--data", "report_url": null }, "kvcache": { "rank": 1, "value": 1024, "run_id": "sequential-search--kvcache", "report_url": null }, "decode": { "rank": 6, "value": 98.5555648803711, "run_id": "sequential-search--decode", "report_url": null }, "request": { "rank": 3, "value": 61.29932403564453, "run_id": "sequential-search--request", "report_url": null } }, "coverage": 9, "matches": 54, "elo": 1687.6113691095902, "match_order_sd": 17.939556954840572, "rank": 2 }, { "method_id": "arbor", "cells": { "params": { "rank": 4, "value": 14549136, "run_id": "arbor--params", "report_url": null }, "flops": { "rank": 3, "value": 58296576, "run_id": "arbor--flops", "report_url": null }, "memory": { "rank": 1, "value": 400084992, "run_id": "arbor--memory", "report_url": null }, "traintime": { "rank": 3, "value": 270, "run_id": "arbor--traintime", "report_url": null }, "steps": { "rank": 5, "value": 337, "run_id": "arbor--steps", "report_url": null }, "data": { "rank": 3, "value": 159776768, "run_id": "arbor--data", "report_url": null }, "kvcache": { "rank": 4, "value": 96768, "run_id": "arbor--kvcache", "report_url": null }, "decode": { "rank": 3, "value": 57.08944797515869, "run_id": "arbor--decode", "report_url": null }, "request": { "rank": 2, "value": 57.88278579711914, "run_id": "arbor--request", "report_url": null } }, "coverage": 9, "matches": 54, "elo": 1632.374102089888, "match_order_sd": 12.447135436229603, "rank": 3 }, { "method_id": "gear", "cells": { "params": { "rank": 1, "value": 13197712, "run_id": "gear--params", "report_url": null }, "flops": { "rank": 4, "value": 62233344, "run_id": "gear--flops", "report_url": null }, "memory": { "rank": 2, "value": 466297856, "run_id": "gear--memory", "report_url": null }, "traintime": { "rank": 3, "value": 270, "run_id": "gear--traintime", "report_url": null }, "steps": { "rank": 3, "value": 306, "run_id": "gear--steps", "report_url": null }, "data": { "rank": 4, "value": 164429824, "run_id": "gear--data", "report_url": null }, "kvcache": { "rank": 5, "value": 124416, "run_id": "gear--kvcache", "report_url": null }, "decode": { "rank": 2, "value": 56.014180183410645, "run_id": "gear--decode", "report_url": null }, "request": { "rank": 4, "value": 97.84162044525146, "run_id": "gear--request", "report_url": null } }, "coverage": 9, "matches": 54, "elo": 1622.4111446669015, "match_order_sd": 13.475478386820553, "rank": 4 }, { "method_id": "beam-search", "cells": { "params": { "rank": 3, "value": 14221456, "run_id": "beam-search--params", "report_url": null }, "flops": { "rank": 5, "value": 69847872, "run_id": "beam-search--flops", "report_url": null }, "memory": { "rank": 5, "value": 1020860928, "run_id": "beam-search--memory", "report_url": null }, "traintime": { "rank": 3, "value": 270, "run_id": "beam-search--traintime", "report_url": null }, "steps": { "rank": 4, "value": 335, "run_id": "beam-search--steps", "report_url": null }, "data": { "rank": 6, "value": 197656576, "run_id": "beam-search--data", "report_url": null }, "kvcache": { "rank": 3, "value": 69632, "run_id": "beam-search--kvcache", "report_url": null }, "decode": { "rank": 4, "value": 65.14537334442139, "run_id": "beam-search--decode", "report_url": null }, "request": { "rank": 6, "value": 180.55975437164307, "run_id": "beam-search--request", "report_url": null } }, "coverage": 9, "matches": 54, "elo": 1438.6132957487964, "match_order_sd": 13.550908028890996, "rank": 5 }, { "method_id": "heuresis-map-elites", "cells": { "params": { "rank": 6, "value": 17826840, "run_id": "heuresis-map-elites--params", "report_url": null }, "flops": { "rank": 6, "value": 84154368, "run_id": "heuresis-map-elites--flops", "report_url": null }, "memory": { "rank": 6, "value": 1698522112, "run_id": "heuresis-map-elites--memory", "report_url": null }, "traintime": { "rank": 3, "value": 270, "run_id": "heuresis-map-elites--traintime", "report_url": null }, "steps": { "rank": 6, "value": 377, "run_id": "heuresis-map-elites--steps", "report_url": null }, "data": { "rank": 2, "value": 156172288, "run_id": "heuresis-map-elites--data", "report_url": null }, "kvcache": { "rank": 6, "value": 500224, "run_id": "heuresis-map-elites--kvcache", "report_url": null }, "decode": { "rank": 5, "value": 97.81789779663086, "run_id": "heuresis-map-elites--decode", "report_url": null }, "request": { "rank": 5, "value": 129.0278434753418, "run_id": "heuresis-map-elites--request", "report_url": null } }, "coverage": 9, "matches": 54, "elo": 1350.4494901637602, "match_order_sd": 14.18883770983771, "rank": 6 }, { "method_id": "tpe", "cells": { "params": { "rank": 7, "value": 26345772, "run_id": "tpe--params", "report_url": null }, "flops": { "rank": 7, "value": 100271808, "run_id": "tpe--flops", "report_url": null }, "memory": { "rank": 7, "value": 6387505664, "run_id": "tpe--memory", "report_url": null }, "traintime": { "rank": 7, "value": 510, "run_id": "tpe--traintime", "report_url": null }, "steps": { "rank": 7, "value": 753, "run_id": "tpe--steps", "report_url": null }, "data": { "rank": 7, "value": 387710976, "run_id": "tpe--data", "report_url": null }, "kvcache": { "rank": 7, "value": 18219520, "run_id": "tpe--kvcache", "report_url": null }, "decode": { "rank": 7, "value": 230.74018955230713, "run_id": "tpe--decode", "report_url": null }, "request": { "rank": 7, "value": 295.38750648498535, "run_id": "tpe--request", "report_url": null } }, "coverage": 9, "matches": 54, "elo": 1039.1704195942107, "match_order_sd": 6.855684393795971, "rank": 7 } ], "status": "ready", "task_count": 9, "comparison_count": 189, "rating": { "method": "Order-averaged margin-aware Elo", "base": 1500, "scale": 400, "cohort_mean": 1500, "leader_anchor": null, "shuffles": 4000, "seed": 20260902, "k_factor": 32.0, "method_order": [ "sequential-search", "beam-search", "autoscientists", "arbor", "gear", "heuresis-map-elites", "tpe" ], "target_order": [ "params", "flops", "memory", "traintime", "steps", "data", "kvcache", "decode", "request" ], "weighting": "equal_per_included_target", "target_weights": { "params": 0.1111111111111111, "flops": 0.1111111111111111, "memory": 0.1111111111111111, "traintime": 0.1111111111111111, "steps": 0.1111111111111111, "data": 0.1111111111111111, "kvcache": 0.1111111111111111, "decode": 0.1111111111111111, "request": 0.1111111111111111 }, "rounding_steps": { "traintime": 30 }, "rounding_formula": "step * floor(raw_value / step + 0.5)", "margin": "min(3, log1p(abs(log(right / left)) / target_scale) / log(2))", "target_scale": "median nonzero absolute pairwise log gap, floored at 1e-9; 1 if all tied", "comparison_order": "configured target and method order; new methods appended by ID", "run_selection": "best accepted run per method and task", "missing_targets": "excluded from comparisons", "ties": "equal target values: outcome 0.5, zero margin and zero update", "match_order_sd": "population standard deviation across shuffled passes; order sensitivity" } }, "method_metadata": { "arbor": { "name": "Arbor", "source_url": "https://github.com/RUC-NLPIR/Arbor", "license": "Apache-2.0", "license_url": "https://github.com/RUC-NLPIR/Arbor/blob/main/LICENSE" }, "autoscientists": { "name": "AutoScientists", "source_url": "https://github.com/mims-harvard/AutoScientists" }, "heuresis-map-elites": { "name": "Heuresis", "source_url": "https://github.com/a-antoniades/Heuresis", "license": "Apache-2.0", "license_url": "https://github.com/a-antoniades/Heuresis/blob/main/LICENSE" }, "tpe": { "name": "TPE", "source_url": "https://github.com/ferreirafabio/autoresearch-automl", "license": "MIT", "license_url": "https://github.com/ferreirafabio/autoresearch-automl/blob/main/LICENSE" }, "beam-search": { "name": "Beam Search" }, "sequential-search": { "name": "Sequential Search" }, "gear": { "name": "GEAR", "source_url": "https://github.com/nm-le/genetic-autoresearch" } } }