{ "mode": "shuffled", "args": { "mode": "shuffled", "seed": 53, "modulus": 11, "observe_mod": 4, "observe_prob": 0.3, "op_family": "full", "query_family": "all", "train_min_len": 1, "train_max_len": 6, "eval_lengths": "3,6,9,12", "eval_k": "0,1,2,3,6,9,12", "eval_query_types": "all", "state_dim": 256, "instr_dim": 128, "heads": 4, "compiler_layers": 2, "dropout": 0.0, "train_steps": 3000, "batch_size": 512, "eval_batch_size": 512, "eval_examples": 512, "lr": 0.001, "weight_decay": 0.0001, "probe_steps": 1500, "probe_batch_size": 512, "probe_lr": 0.001, "probe_weight_decay": 0.0001, "probe_dropout": 0.0, "max_grad_norm": 1.0, "log_every": 200, "probe_log_every": 150, "output_dir": "experiments/dense_latent_query_executor/runs/control_shuffled_mod11", "checkpoint_dir": "large_artifacts/dense_latent_query_executor/checkpoints/control_shuffled_mod11" }, "env": { "python": "3.12.3", "torch": "2.8.0+cu128", "device": "cuda", "cuda_name": "NVIDIA RTX 6000 Ada Generation" }, "train": [ { "step": 1, "loss": 2.4101452827453613, "grad_norm": 0.5073246359825134, "query_target_mass": 0.4494544565677643, "query_target_nll": 2.314271926879883, "query_top1_on_support": 0.626953125, "mean_query_support_size": 4.75390625 }, { "step": 200, "loss": 2.2057628631591797, "grad_norm": 0.3715651333332062, "query_target_mass": 0.5222771763801575, "query_target_nll": 2.1963918209075928, "query_top1_on_support": 0.595703125, "mean_query_support_size": 4.9140625 }, { "step": 400, "loss": 2.1615304946899414, "grad_norm": 0.3894904553890228, "query_target_mass": 0.5595998764038086, "query_target_nll": 2.140998363494873, "query_top1_on_support": 0.615234375, "mean_query_support_size": 5.076171875 }, { "step": 600, "loss": 2.062584400177002, "grad_norm": 0.4491767883300781, "query_target_mass": 0.5549965500831604, "query_target_nll": 2.0611565113067627, "query_top1_on_support": 0.634765625, "mean_query_support_size": 4.62890625 }, { "step": 800, "loss": 1.975420355796814, "grad_norm": 0.5282651782035828, "query_target_mass": 0.6208239793777466, "query_target_nll": 1.9412305355072021, "query_top1_on_support": 0.70703125, "mean_query_support_size": 4.859375 }, { "step": 1000, "loss": 1.9855742454528809, "grad_norm": 0.5730775594711304, "query_target_mass": 0.6662647724151611, "query_target_nll": 1.9229474067687988, "query_top1_on_support": 0.755859375, "mean_query_support_size": 5.09375 }, { "step": 1200, "loss": 1.8914744853973389, "grad_norm": 0.5228585600852966, "query_target_mass": 0.645916223526001, "query_target_nll": 1.895882248878479, "query_top1_on_support": 0.740234375, "mean_query_support_size": 4.83203125 }, { "step": 1400, "loss": 1.8006564378738403, "grad_norm": 0.48106133937835693, "query_target_mass": 0.6616517901420593, "query_target_nll": 1.7839674949645996, "query_top1_on_support": 0.736328125, "mean_query_support_size": 4.568359375 }, { "step": 1600, "loss": 1.866619348526001, "grad_norm": 0.5122888684272766, "query_target_mass": 0.6894643902778625, "query_target_nll": 1.847915530204773, "query_top1_on_support": 0.767578125, "mean_query_support_size": 5.001953125 }, { "step": 1800, "loss": 1.8387041091918945, "grad_norm": 0.508590579032898, "query_target_mass": 0.7150973677635193, "query_target_nll": 1.8278909921646118, "query_top1_on_support": 0.76953125, "mean_query_support_size": 5.134765625 }, { "step": 2000, "loss": 1.8676083087921143, "grad_norm": 0.49065127968788147, "query_target_mass": 0.6920478343963623, "query_target_nll": 1.8648737668991089, "query_top1_on_support": 0.763671875, "mean_query_support_size": 5.052734375 }, { "step": 2200, "loss": 1.8557870388031006, "grad_norm": 0.4731358587741852, "query_target_mass": 0.6856834888458252, "query_target_nll": 1.8346469402313232, "query_top1_on_support": 0.7734375, "mean_query_support_size": 4.8671875 }, { "step": 2400, "loss": 1.7805043458938599, "grad_norm": 0.5287941694259644, "query_target_mass": 0.6960857510566711, "query_target_nll": 1.7645814418792725, "query_top1_on_support": 0.779296875, "mean_query_support_size": 4.7109375 }, { "step": 2600, "loss": 1.7921371459960938, "grad_norm": 0.49723106622695923, "query_target_mass": 0.6912848353385925, "query_target_nll": 1.765580654144287, "query_top1_on_support": 0.7734375, "mean_query_support_size": 4.685546875 }, { "step": 2800, "loss": 1.8218697309494019, "grad_norm": 0.48986369371414185, "query_target_mass": 0.6913171410560608, "query_target_nll": 1.8157546520233154, "query_top1_on_support": 0.771484375, "mean_query_support_size": 4.97265625 }, { "step": 3000, "loss": 1.844421148300171, "grad_norm": 0.488616019487381, "query_target_mass": 0.7008047103881836, "query_target_nll": 1.8218097686767578, "query_top1_on_support": 0.765625, "mean_query_support_size": 4.916015625 } ], "probe_train": [ { "step": 1, "loss": 4.80712366104126, "grad_norm": 0.3541412651538849, "probe_belief_target_mass": 0.060917481780052185, "probe_belief_target_nll": 4.80712366104126, "probe_belief_top1_on_support": 0.072823666036129, "mean_belief_support_size": 7.322545051574707 }, { "step": 150, "loss": 3.249880313873291, "grad_norm": 0.2168380469083786, "probe_belief_target_mass": 0.46531373262405396, "probe_belief_target_nll": 3.249880313873291, "probe_belief_top1_on_support": 0.6280692219734192, "mean_belief_support_size": 7.147321701049805 }, { "step": 300, "loss": 3.2649080753326416, "grad_norm": 0.20299457013607025, "probe_belief_target_mass": 0.466341108083725, "probe_belief_target_nll": 3.2649080753326416, "probe_belief_top1_on_support": 0.6241629719734192, "mean_belief_support_size": 7.188058376312256 }, { "step": 450, "loss": 3.1710751056671143, "grad_norm": 0.2033037692308426, "probe_belief_target_mass": 0.48856478929519653, "probe_belief_target_nll": 3.1710751056671143, "probe_belief_top1_on_support": 0.6395089626312256, "mean_belief_support_size": 7.131417751312256 }, { "step": 600, "loss": 3.233023166656494, "grad_norm": 0.2018553763628006, "probe_belief_target_mass": 0.4825521409511566, "probe_belief_target_nll": 3.233023166656494, "probe_belief_top1_on_support": 0.642578125, "mean_belief_support_size": 7.301339626312256 }, { "step": 750, "loss": 3.282351016998291, "grad_norm": 0.1959386169910431, "probe_belief_target_mass": 0.4678323268890381, "probe_belief_target_nll": 3.282351016998291, "probe_belief_top1_on_support": 0.6127232313156128, "mean_belief_support_size": 7.330357551574707 }, { "step": 900, "loss": 3.188288688659668, "grad_norm": 0.19982124865055084, "probe_belief_target_mass": 0.4924211800098419, "probe_belief_target_nll": 3.188288688659668, "probe_belief_top1_on_support": 0.6378348469734192, "mean_belief_support_size": 7.323940277099609 }, { "step": 1050, "loss": 3.198838949203491, "grad_norm": 0.19535432755947113, "probe_belief_target_mass": 0.49614110589027405, "probe_belief_target_nll": 3.198838949203491, "probe_belief_top1_on_support": 0.6478794813156128, "mean_belief_support_size": 7.611328601837158 }, { "step": 1200, "loss": 3.261735677719116, "grad_norm": 0.2075437307357788, "probe_belief_target_mass": 0.47067299485206604, "probe_belief_target_nll": 3.261735677719116, "probe_belief_top1_on_support": 0.609933078289032, "mean_belief_support_size": 7.290737152099609 }, { "step": 1350, "loss": 3.253826379776001, "grad_norm": 0.20045940577983856, "probe_belief_target_mass": 0.468263179063797, "probe_belief_target_nll": 3.253826379776001, "probe_belief_top1_on_support": 0.6130022406578064, "mean_belief_support_size": 7.146205902099609 }, { "step": 1500, "loss": 3.179474115371704, "grad_norm": 0.19239500164985657, "probe_belief_target_mass": 0.4877963066101074, "probe_belief_target_nll": 3.179474115371704, "probe_belief_top1_on_support": 0.62109375, "mean_belief_support_size": 7.107143402099609 } ], "eval": [ { "step": 3000, "rows": [ { "model": "shuffled", "length": 3, "query_type": "A", "k": 0, "n": 512, "query_target_mass": 0.4455578625202179, "query_target_nll": 2.6512224674224854, "query_top1_on_support": 0.443359375, "mean_query_support_size": 4.888671875, "probe_belief_target_mass": 0.05778220295906067, "probe_belief_target_nll": 13.583930015563965, "probe_belief_top1_on_support": 0.048828125, "mean_belief_support_size": 5.791015625 }, { "model": "shuffled", "length": 3, "query_type": "A", "k": 1, "n": 512, "query_target_mass": 0.48225829005241394, "query_target_nll": 2.3794384002685547, "query_top1_on_support": 0.48828125, "mean_query_support_size": 4.888671875, "probe_belief_target_mass": 0.10074115544557571, "probe_belief_target_nll": 5.099181175231934, "probe_belief_top1_on_support": 0.125, "mean_belief_support_size": 5.791015625 }, { "model": "shuffled", "length": 3, "query_type": "A", "k": 2, "n": 512, "query_target_mass": 0.6052368879318237, "query_target_nll": 2.027223587036133, "query_top1_on_support": 0.6484375, "mean_query_support_size": 4.888671875, "probe_belief_target_mass": 0.24359533190727234, "probe_belief_target_nll": 3.5786094665527344, "probe_belief_top1_on_support": 0.3046875, "mean_belief_support_size": 5.791015625 }, { "model": "shuffled", "length": 3, "query_type": "A", "k": 3, "n": 512, "query_target_mass": 0.7449176907539368, "query_target_nll": 1.7177138328552246, "query_top1_on_support": 0.8203125, "mean_query_support_size": 4.888671875, "probe_belief_target_mass": 0.46867355704307556, "probe_belief_target_nll": 2.7017455101013184, "probe_belief_top1_on_support": 0.70703125, "mean_belief_support_size": 5.791015625 }, { "model": "shuffled", "length": 3, "query_type": "A", "k": 6, "n": 512, "query_target_mass": 0.7449176907539368, "query_target_nll": 1.7177138328552246, "query_top1_on_support": 0.8203125, "mean_query_support_size": 4.888671875, "probe_belief_target_mass": 0.46867355704307556, "probe_belief_target_nll": 2.7017455101013184, "probe_belief_top1_on_support": 0.70703125, "mean_belief_support_size": 5.791015625 }, { "model": "shuffled", "length": 3, "query_type": "A", "k": 9, "n": 512, "query_target_mass": 0.7449176907539368, "query_target_nll": 1.7177138328552246, "query_top1_on_support": 0.8203125, "mean_query_support_size": 4.888671875, "probe_belief_target_mass": 0.46867355704307556, "probe_belief_target_nll": 2.7017455101013184, "probe_belief_top1_on_support": 0.70703125, "mean_belief_support_size": 5.791015625 }, { "model": "shuffled", "length": 3, "query_type": "A", "k": 12, "n": 512, "query_target_mass": 0.7449176907539368, "query_target_nll": 1.7177138328552246, "query_top1_on_support": 0.8203125, "mean_query_support_size": 4.888671875, "probe_belief_target_mass": 0.46867355704307556, "probe_belief_target_nll": 2.7017455101013184, "probe_belief_top1_on_support": 0.70703125, "mean_belief_support_size": 5.791015625 }, { "model": "shuffled", "length": 3, "query_type": "B", "k": 0, "n": 512, "query_target_mass": 0.44005638360977173, "query_target_nll": 2.6357903480529785, "query_top1_on_support": 0.443359375, "mean_query_support_size": 4.830078125, "probe_belief_target_mass": 0.06099677085876465, "probe_belief_target_nll": 13.09136962890625, "probe_belief_top1_on_support": 0.05859375, "mean_belief_support_size": 5.55859375 }, { "model": "shuffled", "length": 3, "query_type": "B", "k": 1, "n": 512, "query_target_mass": 0.4819532632827759, "query_target_nll": 2.343909502029419, "query_top1_on_support": 0.4765625, "mean_query_support_size": 4.830078125, "probe_belief_target_mass": 0.10447762906551361, "probe_belief_target_nll": 5.0095038414001465, "probe_belief_top1_on_support": 0.1171875, "mean_belief_support_size": 5.55859375 }, { "model": "shuffled", "length": 3, "query_type": "B", "k": 2, "n": 512, "query_target_mass": 0.5534405708312988, "query_target_nll": 2.137195110321045, "query_top1_on_support": 0.5859375, "mean_query_support_size": 4.830078125, "probe_belief_target_mass": 0.2305627167224884, "probe_belief_target_nll": 3.479694366455078, "probe_belief_top1_on_support": 0.314453125, "mean_belief_support_size": 5.55859375 }, { "model": "shuffled", "length": 3, "query_type": "B", "k": 3, "n": 512, "query_target_mass": 0.717720627784729, "query_target_nll": 1.7879459857940674, "query_top1_on_support": 0.798828125, "mean_query_support_size": 4.830078125, "probe_belief_target_mass": 0.4892575740814209, "probe_belief_target_nll": 2.5495786666870117, "probe_belief_top1_on_support": 0.71484375, "mean_belief_support_size": 5.55859375 }, { "model": "shuffled", "length": 3, "query_type": "B", "k": 6, "n": 512, "query_target_mass": 0.717720627784729, "query_target_nll": 1.7879459857940674, "query_top1_on_support": 0.798828125, "mean_query_support_size": 4.830078125, "probe_belief_target_mass": 0.4892575740814209, "probe_belief_target_nll": 2.5495786666870117, "probe_belief_top1_on_support": 0.71484375, "mean_belief_support_size": 5.55859375 }, { "model": "shuffled", "length": 3, "query_type": "B", "k": 9, "n": 512, "query_target_mass": 0.717720627784729, "query_target_nll": 1.7879459857940674, "query_top1_on_support": 0.798828125, "mean_query_support_size": 4.830078125, "probe_belief_target_mass": 0.4892575740814209, "probe_belief_target_nll": 2.5495786666870117, "probe_belief_top1_on_support": 0.71484375, "mean_belief_support_size": 5.55859375 }, { "model": "shuffled", "length": 3, "query_type": "B", "k": 12, "n": 512, "query_target_mass": 0.717720627784729, "query_target_nll": 1.7879459857940674, "query_top1_on_support": 0.798828125, "mean_query_support_size": 4.830078125, "probe_belief_target_mass": 0.4892575740814209, "probe_belief_target_nll": 2.5495786666870117, "probe_belief_top1_on_support": 0.71484375, "mean_belief_support_size": 5.55859375 }, { "model": "shuffled", "length": 3, "query_type": "A_PLUS_B", "k": 0, "n": 512, "query_target_mass": 0.5116761922836304, "query_target_nll": 2.6110222339630127, "query_top1_on_support": 0.51171875, "mean_query_support_size": 5.654296875, "probe_belief_target_mass": 0.05617488920688629, "probe_belief_target_nll": 13.187122344970703, "probe_belief_top1_on_support": 0.06640625, "mean_belief_support_size": 5.87109375 }, { "model": "shuffled", "length": 3, "query_type": "A_PLUS_B", "k": 1, "n": 512, "query_target_mass": 0.5333318114280701, "query_target_nll": 2.381298065185547, "query_top1_on_support": 0.53125, "mean_query_support_size": 5.654296875, "probe_belief_target_mass": 0.12744465470314026, "probe_belief_target_nll": 4.8475823402404785, "probe_belief_top1_on_support": 0.203125, "mean_belief_support_size": 5.87109375 }, { "model": "shuffled", "length": 3, "query_type": "A_PLUS_B", "k": 2, "n": 512, "query_target_mass": 0.6082482933998108, "query_target_nll": 2.202113628387451, "query_top1_on_support": 0.62890625, "mean_query_support_size": 5.654296875, "probe_belief_target_mass": 0.2612214684486389, "probe_belief_target_nll": 3.4608306884765625, "probe_belief_top1_on_support": 0.326171875, "mean_belief_support_size": 5.87109375 }, { "model": "shuffled", "length": 3, "query_type": "A_PLUS_B", "k": 3, "n": 512, "query_target_mass": 0.7275001406669617, "query_target_nll": 1.9530500173568726, "query_top1_on_support": 0.830078125, "mean_query_support_size": 5.654296875, "probe_belief_target_mass": 0.5033478736877441, "probe_belief_target_nll": 2.544081211090088, "probe_belief_top1_on_support": 0.740234375, "mean_belief_support_size": 5.87109375 }, { "model": "shuffled", "length": 3, "query_type": "A_PLUS_B", "k": 6, "n": 512, "query_target_mass": 0.7275001406669617, "query_target_nll": 1.9530500173568726, "query_top1_on_support": 0.830078125, "mean_query_support_size": 5.654296875, "probe_belief_target_mass": 0.5033478736877441, "probe_belief_target_nll": 2.544081211090088, "probe_belief_top1_on_support": 0.740234375, "mean_belief_support_size": 5.87109375 }, { "model": "shuffled", "length": 3, "query_type": "A_PLUS_B", "k": 9, "n": 512, "query_target_mass": 0.7275001406669617, "query_target_nll": 1.9530500173568726, "query_top1_on_support": 0.830078125, "mean_query_support_size": 5.654296875, "probe_belief_target_mass": 0.5033478736877441, "probe_belief_target_nll": 2.544081211090088, "probe_belief_top1_on_support": 0.740234375, "mean_belief_support_size": 5.87109375 }, { "model": "shuffled", "length": 3, "query_type": "A_PLUS_B", "k": 12, "n": 512, "query_target_mass": 0.7275001406669617, "query_target_nll": 1.9530500173568726, "query_top1_on_support": 0.830078125, "mean_query_support_size": 5.654296875, "probe_belief_target_mass": 0.5033478736877441, "probe_belief_target_nll": 2.544081211090088, "probe_belief_top1_on_support": 0.740234375, "mean_belief_support_size": 5.87109375 }, { "model": "shuffled", "length": 3, "query_type": "A_MINUS_B", "k": 0, "n": 512, "query_target_mass": 0.48435312509536743, "query_target_nll": 11.451425552368164, "query_top1_on_support": 0.484375, "mean_query_support_size": 4.8515625, "probe_belief_target_mass": 0.059441715478897095, "probe_belief_target_nll": 13.334722518920898, "probe_belief_top1_on_support": 0.068359375, "mean_belief_support_size": 6.1484375 }, { "model": "shuffled", "length": 3, "query_type": "A_MINUS_B", "k": 1, "n": 512, "query_target_mass": 0.5569222569465637, "query_target_nll": 5.665999412536621, "query_top1_on_support": 0.55859375, "mean_query_support_size": 4.8515625, "probe_belief_target_mass": 0.11393251270055771, "probe_belief_target_nll": 4.971986770629883, "probe_belief_top1_on_support": 0.150390625, "mean_belief_support_size": 6.1484375 }, { "model": "shuffled", "length": 3, "query_type": "A_MINUS_B", "k": 2, "n": 512, "query_target_mass": 0.6991961002349854, "query_target_nll": 2.8912642002105713, "query_top1_on_support": 0.71875, "mean_query_support_size": 4.8515625, "probe_belief_target_mass": 0.24340519309043884, "probe_belief_target_nll": 3.5947422981262207, "probe_belief_top1_on_support": 0.3125, "mean_belief_support_size": 6.1484375 }, { "model": "shuffled", "length": 3, "query_type": "A_MINUS_B", "k": 3, "n": 512, "query_target_mass": 0.8423808813095093, "query_target_nll": 1.4027557373046875, "query_top1_on_support": 0.9140625, "mean_query_support_size": 4.8515625, "probe_belief_target_mass": 0.48594972491264343, "probe_belief_target_nll": 2.648925304412842, "probe_belief_top1_on_support": 0.71875, "mean_belief_support_size": 6.1484375 }, { "model": "shuffled", "length": 3, "query_type": "A_MINUS_B", "k": 6, "n": 512, "query_target_mass": 0.8423808813095093, "query_target_nll": 1.4027557373046875, "query_top1_on_support": 0.9140625, "mean_query_support_size": 4.8515625, "probe_belief_target_mass": 0.48594972491264343, "probe_belief_target_nll": 2.648925304412842, "probe_belief_top1_on_support": 0.71875, "mean_belief_support_size": 6.1484375 }, { "model": "shuffled", "length": 3, "query_type": "A_MINUS_B", "k": 9, "n": 512, "query_target_mass": 0.8423808813095093, "query_target_nll": 1.4027557373046875, "query_top1_on_support": 0.9140625, "mean_query_support_size": 4.8515625, "probe_belief_target_mass": 0.48594972491264343, "probe_belief_target_nll": 2.648925304412842, "probe_belief_top1_on_support": 0.71875, "mean_belief_support_size": 6.1484375 }, { "model": "shuffled", "length": 3, "query_type": "A_MINUS_B", "k": 12, "n": 512, "query_target_mass": 0.8423808813095093, "query_target_nll": 1.4027557373046875, "query_top1_on_support": 0.9140625, "mean_query_support_size": 4.8515625, "probe_belief_target_mass": 0.48594972491264343, "probe_belief_target_nll": 2.648925304412842, "probe_belief_top1_on_support": 0.71875, "mean_belief_support_size": 6.1484375 }, { "model": "shuffled", "length": 6, "query_type": "A", "k": 0, "n": 512, "query_target_mass": 0.2879578471183777, "query_target_nll": 2.7188305854797363, "query_top1_on_support": 0.271484375, "mean_query_support_size": 3.263671875, "probe_belief_target_mass": 0.03275463357567787, "probe_belief_target_nll": 13.957342147827148, "probe_belief_top1_on_support": 0.03515625, "mean_belief_support_size": 3.919921875 }, { "model": "shuffled", "length": 6, "query_type": "A", "k": 1, "n": 512, "query_target_mass": 0.2981910705566406, "query_target_nll": 2.444216251373291, "query_top1_on_support": 0.294921875, "mean_query_support_size": 3.263671875, "probe_belief_target_mass": 0.037958405911922455, "probe_belief_target_nll": 5.549799919128418, "probe_belief_top1_on_support": 0.04296875, "mean_belief_support_size": 3.919921875 }, { "model": "shuffled", "length": 6, "query_type": "A", "k": 2, "n": 512, "query_target_mass": 0.3116975426673889, "query_target_nll": 2.443662643432617, "query_top1_on_support": 0.29296875, "mean_query_support_size": 3.263671875, "probe_belief_target_mass": 0.053832150995731354, "probe_belief_target_nll": 4.904109477996826, "probe_belief_top1_on_support": 0.1015625, "mean_belief_support_size": 3.919921875 }, { "model": "shuffled", "length": 6, "query_type": "A", "k": 3, "n": 512, "query_target_mass": 0.3285207152366638, "query_target_nll": 2.3671457767486572, "query_top1_on_support": 0.361328125, "mean_query_support_size": 3.263671875, "probe_belief_target_mass": 0.06832811236381531, "probe_belief_target_nll": 4.567434310913086, "probe_belief_top1_on_support": 0.078125, "mean_belief_support_size": 3.919921875 }, { "model": "shuffled", "length": 6, "query_type": "A", "k": 6, "n": 512, "query_target_mass": 0.44856417179107666, "query_target_nll": 1.9864228963851929, "query_top1_on_support": 0.568359375, "mean_query_support_size": 3.263671875, "probe_belief_target_mass": 0.1721211075782776, "probe_belief_target_nll": 3.7245962619781494, "probe_belief_top1_on_support": 0.328125, "mean_belief_support_size": 3.919921875 }, { "model": "shuffled", "length": 6, "query_type": "A", "k": 9, "n": 512, "query_target_mass": 0.44856417179107666, "query_target_nll": 1.9864228963851929, "query_top1_on_support": 0.568359375, "mean_query_support_size": 3.263671875, "probe_belief_target_mass": 0.1721211075782776, "probe_belief_target_nll": 3.7245962619781494, "probe_belief_top1_on_support": 0.328125, "mean_belief_support_size": 3.919921875 }, { "model": "shuffled", "length": 6, "query_type": "A", "k": 12, "n": 512, "query_target_mass": 0.44856417179107666, "query_target_nll": 1.9864228963851929, "query_top1_on_support": 0.568359375, "mean_query_support_size": 3.263671875, "probe_belief_target_mass": 0.1721211075782776, "probe_belief_target_nll": 3.7245962619781494, "probe_belief_top1_on_support": 0.328125, "mean_belief_support_size": 3.919921875 }, { "model": "shuffled", "length": 6, "query_type": "B", "k": 0, "n": 512, "query_target_mass": 0.28858500719070435, "query_target_nll": 2.678116798400879, "query_top1_on_support": 0.314453125, "mean_query_support_size": 3.142578125, "probe_belief_target_mass": 0.0337277427315712, "probe_belief_target_nll": 13.975494384765625, "probe_belief_top1_on_support": 0.021484375, "mean_belief_support_size": 3.673828125 }, { "model": "shuffled", "length": 6, "query_type": "B", "k": 1, "n": 512, "query_target_mass": 0.2844121754169464, "query_target_nll": 2.440549373626709, "query_top1_on_support": 0.30078125, "mean_query_support_size": 3.142578125, "probe_belief_target_mass": 0.038055866956710815, "probe_belief_target_nll": 5.5236592292785645, "probe_belief_top1_on_support": 0.05078125, "mean_belief_support_size": 3.673828125 }, { "model": "shuffled", "length": 6, "query_type": "B", "k": 2, "n": 512, "query_target_mass": 0.2847364544868469, "query_target_nll": 2.442559242248535, "query_top1_on_support": 0.2421875, "mean_query_support_size": 3.142578125, "probe_belief_target_mass": 0.04236369580030441, "probe_belief_target_nll": 4.917013645172119, "probe_belief_top1_on_support": 0.083984375, "mean_belief_support_size": 3.673828125 }, { "model": "shuffled", "length": 6, "query_type": "B", "k": 3, "n": 512, "query_target_mass": 0.30518800020217896, "query_target_nll": 2.3695473670959473, "query_top1_on_support": 0.333984375, "mean_query_support_size": 3.142578125, "probe_belief_target_mass": 0.063015878200531, "probe_belief_target_nll": 4.529212951660156, "probe_belief_top1_on_support": 0.1171875, "mean_belief_support_size": 3.673828125 }, { "model": "shuffled", "length": 6, "query_type": "B", "k": 6, "n": 512, "query_target_mass": 0.42179057002067566, "query_target_nll": 2.048110008239746, "query_top1_on_support": 0.51953125, "mean_query_support_size": 3.142578125, "probe_belief_target_mass": 0.16706712543964386, "probe_belief_target_nll": 3.652801990509033, "probe_belief_top1_on_support": 0.3203125, "mean_belief_support_size": 3.673828125 }, { "model": "shuffled", "length": 6, "query_type": "B", "k": 9, "n": 512, "query_target_mass": 0.42179057002067566, "query_target_nll": 2.048110008239746, "query_top1_on_support": 0.51953125, "mean_query_support_size": 3.142578125, "probe_belief_target_mass": 0.16706712543964386, "probe_belief_target_nll": 3.652801990509033, "probe_belief_top1_on_support": 0.3203125, "mean_belief_support_size": 3.673828125 }, { "model": "shuffled", "length": 6, "query_type": "B", "k": 12, "n": 512, "query_target_mass": 0.42179057002067566, "query_target_nll": 2.048110008239746, "query_top1_on_support": 0.51953125, "mean_query_support_size": 3.142578125, "probe_belief_target_mass": 0.16706712543964386, "probe_belief_target_nll": 3.652801990509033, "probe_belief_top1_on_support": 0.3203125, "mean_belief_support_size": 3.673828125 }, { "model": "shuffled", "length": 6, "query_type": "A_PLUS_B", "k": 0, "n": 512, "query_target_mass": 0.29890596866607666, "query_target_nll": 2.6533098220825195, "query_top1_on_support": 0.29296875, "mean_query_support_size": 3.419921875, "probe_belief_target_mass": 0.03558526188135147, "probe_belief_target_nll": 13.695941925048828, "probe_belief_top1_on_support": 0.03515625, "mean_belief_support_size": 3.583984375 }, { "model": "shuffled", "length": 6, "query_type": "A_PLUS_B", "k": 1, "n": 512, "query_target_mass": 0.3110019266605377, "query_target_nll": 2.4252967834472656, "query_top1_on_support": 0.31640625, "mean_query_support_size": 3.419921875, "probe_belief_target_mass": 0.0402023121714592, "probe_belief_target_nll": 5.447030067443848, "probe_belief_top1_on_support": 0.072265625, "mean_belief_support_size": 3.583984375 }, { "model": "shuffled", "length": 6, "query_type": "A_PLUS_B", "k": 2, "n": 512, "query_target_mass": 0.31128400564193726, "query_target_nll": 2.4266464710235596, "query_top1_on_support": 0.32421875, "mean_query_support_size": 3.419921875, "probe_belief_target_mass": 0.04732372611761093, "probe_belief_target_nll": 4.8353986740112305, "probe_belief_top1_on_support": 0.0546875, "mean_belief_support_size": 3.583984375 }, { "model": "shuffled", "length": 6, "query_type": "A_PLUS_B", "k": 3, "n": 512, "query_target_mass": 0.32887494564056396, "query_target_nll": 2.3694357872009277, "query_top1_on_support": 0.3359375, "mean_query_support_size": 3.419921875, "probe_belief_target_mass": 0.07277940213680267, "probe_belief_target_nll": 4.4490065574646, "probe_belief_top1_on_support": 0.123046875, "mean_belief_support_size": 3.583984375 }, { "model": "shuffled", "length": 6, "query_type": "A_PLUS_B", "k": 6, "n": 512, "query_target_mass": 0.39520102739334106, "query_target_nll": 2.1909236907958984, "query_top1_on_support": 0.451171875, "mean_query_support_size": 3.419921875, "probe_belief_target_mass": 0.17562651634216309, "probe_belief_target_nll": 3.6487679481506348, "probe_belief_top1_on_support": 0.31640625, "mean_belief_support_size": 3.583984375 }, { "model": "shuffled", "length": 6, "query_type": "A_PLUS_B", "k": 9, "n": 512, "query_target_mass": 0.39520102739334106, "query_target_nll": 2.1909236907958984, "query_top1_on_support": 0.451171875, "mean_query_support_size": 3.419921875, "probe_belief_target_mass": 0.17562651634216309, "probe_belief_target_nll": 3.6487679481506348, "probe_belief_top1_on_support": 0.31640625, "mean_belief_support_size": 3.583984375 }, { "model": "shuffled", "length": 6, "query_type": "A_PLUS_B", "k": 12, "n": 512, "query_target_mass": 0.39520102739334106, "query_target_nll": 2.1909236907958984, "query_top1_on_support": 0.451171875, "mean_query_support_size": 3.419921875, "probe_belief_target_mass": 0.17562651634216309, "probe_belief_target_nll": 3.6487679481506348, "probe_belief_top1_on_support": 0.31640625, "mean_belief_support_size": 3.583984375 }, { "model": "shuffled", "length": 6, "query_type": "A_MINUS_B", "k": 0, "n": 512, "query_target_mass": 0.32225972414016724, "query_target_nll": 11.85603141784668, "query_top1_on_support": 0.322265625, "mean_query_support_size": 3.419921875, "probe_belief_target_mass": 0.03453746438026428, "probe_belief_target_nll": 13.992301940917969, "probe_belief_top1_on_support": 0.033203125, "mean_belief_support_size": 3.73046875 }, { "model": "shuffled", "length": 6, "query_type": "A_MINUS_B", "k": 1, "n": 512, "query_target_mass": 0.350530743598938, "query_target_nll": 7.351190090179443, "query_top1_on_support": 0.353515625, "mean_query_support_size": 3.419921875, "probe_belief_target_mass": 0.03586255759000778, "probe_belief_target_nll": 5.628395080566406, "probe_belief_top1_on_support": 0.033203125, "mean_belief_support_size": 3.73046875 }, { "model": "shuffled", "length": 6, "query_type": "A_MINUS_B", "k": 2, "n": 512, "query_target_mass": 0.33851563930511475, "query_target_nll": 5.833085536956787, "query_top1_on_support": 0.3515625, "mean_query_support_size": 3.419921875, "probe_belief_target_mass": 0.041831355541944504, "probe_belief_target_nll": 4.989966869354248, "probe_belief_top1_on_support": 0.0625, "mean_belief_support_size": 3.73046875 }, { "model": "shuffled", "length": 6, "query_type": "A_MINUS_B", "k": 3, "n": 512, "query_target_mass": 0.3593279719352722, "query_target_nll": 4.070653915405273, "query_top1_on_support": 0.369140625, "mean_query_support_size": 3.419921875, "probe_belief_target_mass": 0.06411527097225189, "probe_belief_target_nll": 4.5402326583862305, "probe_belief_top1_on_support": 0.109375, "mean_belief_support_size": 3.73046875 }, { "model": "shuffled", "length": 6, "query_type": "A_MINUS_B", "k": 6, "n": 512, "query_target_mass": 0.45666033029556274, "query_target_nll": 2.0252017974853516, "query_top1_on_support": 0.548828125, "mean_query_support_size": 3.419921875, "probe_belief_target_mass": 0.17538997530937195, "probe_belief_target_nll": 3.6319375038146973, "probe_belief_top1_on_support": 0.31640625, "mean_belief_support_size": 3.73046875 }, { "model": "shuffled", "length": 6, "query_type": "A_MINUS_B", "k": 9, "n": 512, "query_target_mass": 0.45666033029556274, "query_target_nll": 2.0252017974853516, "query_top1_on_support": 0.548828125, "mean_query_support_size": 3.419921875, "probe_belief_target_mass": 0.17538997530937195, "probe_belief_target_nll": 3.6319375038146973, "probe_belief_top1_on_support": 0.31640625, "mean_belief_support_size": 3.73046875 }, { "model": "shuffled", "length": 6, "query_type": "A_MINUS_B", "k": 12, "n": 512, "query_target_mass": 0.45666033029556274, "query_target_nll": 2.0252017974853516, "query_top1_on_support": 0.548828125, "mean_query_support_size": 3.419921875, "probe_belief_target_mass": 0.17538997530937195, "probe_belief_target_nll": 3.6319375038146973, "probe_belief_top1_on_support": 0.31640625, "mean_belief_support_size": 3.73046875 }, { "model": "shuffled", "length": 9, "query_type": "A", "k": 0, "n": 512, "query_target_mass": 0.2025718241930008, "query_target_nll": 2.6320276260375977, "query_top1_on_support": 0.20703125, "mean_query_support_size": 2.224609375, "probe_belief_target_mass": 0.02080417051911354, "probe_belief_target_nll": 14.040159225463867, "probe_belief_top1_on_support": 0.0078125, "mean_belief_support_size": 2.44140625 }, { "model": "shuffled", "length": 9, "query_type": "A", "k": 1, "n": 512, "query_target_mass": 0.20099569857120514, "query_target_nll": 2.4366397857666016, "query_top1_on_support": 0.19140625, "mean_query_support_size": 2.224609375, "probe_belief_target_mass": 0.01997956447303295, "probe_belief_target_nll": 5.632221221923828, "probe_belief_top1_on_support": 0.0234375, "mean_belief_support_size": 2.44140625 }, { "model": "shuffled", "length": 9, "query_type": "A", "k": 2, "n": 512, "query_target_mass": 0.20212829113006592, "query_target_nll": 2.42724347114563, "query_top1_on_support": 0.2109375, "mean_query_support_size": 2.224609375, "probe_belief_target_mass": 0.023978684097528458, "probe_belief_target_nll": 5.091657638549805, "probe_belief_top1_on_support": 0.0390625, "mean_belief_support_size": 2.44140625 }, { "model": "shuffled", "length": 9, "query_type": "A", "k": 3, "n": 512, "query_target_mass": 0.20230676233768463, "query_target_nll": 2.4351999759674072, "query_top1_on_support": 0.205078125, "mean_query_support_size": 2.224609375, "probe_belief_target_mass": 0.02339593879878521, "probe_belief_target_nll": 4.928763389587402, "probe_belief_top1_on_support": 0.033203125, "mean_belief_support_size": 2.44140625 }, { "model": "shuffled", "length": 9, "query_type": "A", "k": 6, "n": 512, "query_target_mass": 0.23503610491752625, "query_target_nll": 2.319472074508667, "query_top1_on_support": 0.265625, "mean_query_support_size": 2.224609375, "probe_belief_target_mass": 0.03631928563117981, "probe_belief_target_nll": 4.547234535217285, "probe_belief_top1_on_support": 0.05859375, "mean_belief_support_size": 2.44140625 }, { "model": "shuffled", "length": 9, "query_type": "A", "k": 9, "n": 512, "query_target_mass": 0.2680590748786926, "query_target_nll": 2.238205909729004, "query_top1_on_support": 0.349609375, "mean_query_support_size": 2.224609375, "probe_belief_target_mass": 0.0543341338634491, "probe_belief_target_nll": 4.315460681915283, "probe_belief_top1_on_support": 0.125, "mean_belief_support_size": 2.44140625 }, { "model": "shuffled", "length": 9, "query_type": "A", "k": 12, "n": 512, "query_target_mass": 0.2680590748786926, "query_target_nll": 2.238205909729004, "query_top1_on_support": 0.349609375, "mean_query_support_size": 2.224609375, "probe_belief_target_mass": 0.0543341338634491, "probe_belief_target_nll": 4.315460681915283, "probe_belief_top1_on_support": 0.125, "mean_belief_support_size": 2.44140625 }, { "model": "shuffled", "length": 9, "query_type": "B", "k": 0, "n": 512, "query_target_mass": 0.19646945595741272, "query_target_nll": 2.7197322845458984, "query_top1_on_support": 0.1953125, "mean_query_support_size": 2.19140625, "probe_belief_target_mass": 0.01673978753387928, "probe_belief_target_nll": 14.636980056762695, "probe_belief_top1_on_support": 0.015625, "mean_belief_support_size": 2.490234375 }, { "model": "shuffled", "length": 9, "query_type": "B", "k": 1, "n": 512, "query_target_mass": 0.1989315152168274, "query_target_nll": 2.427213668823242, "query_top1_on_support": 0.232421875, "mean_query_support_size": 2.19140625, "probe_belief_target_mass": 0.02190161868929863, "probe_belief_target_nll": 5.662467002868652, "probe_belief_top1_on_support": 0.029296875, "mean_belief_support_size": 2.490234375 }, { "model": "shuffled", "length": 9, "query_type": "B", "k": 2, "n": 512, "query_target_mass": 0.19991019368171692, "query_target_nll": 2.4136528968811035, "query_top1_on_support": 0.185546875, "mean_query_support_size": 2.19140625, "probe_belief_target_mass": 0.02221425622701645, "probe_belief_target_nll": 5.099221706390381, "probe_belief_top1_on_support": 0.025390625, "mean_belief_support_size": 2.490234375 }, { "model": "shuffled", "length": 9, "query_type": "B", "k": 3, "n": 512, "query_target_mass": 0.19869333505630493, "query_target_nll": 2.425276041030884, "query_top1_on_support": 0.1796875, "mean_query_support_size": 2.19140625, "probe_belief_target_mass": 0.02455630898475647, "probe_belief_target_nll": 4.935386657714844, "probe_belief_top1_on_support": 0.03125, "mean_belief_support_size": 2.490234375 }, { "model": "shuffled", "length": 9, "query_type": "B", "k": 6, "n": 512, "query_target_mass": 0.2195580005645752, "query_target_nll": 2.3516993522644043, "query_top1_on_support": 0.25390625, "mean_query_support_size": 2.19140625, "probe_belief_target_mass": 0.04686664789915085, "probe_belief_target_nll": 4.449546813964844, "probe_belief_top1_on_support": 0.087890625, "mean_belief_support_size": 2.490234375 }, { "model": "shuffled", "length": 9, "query_type": "B", "k": 9, "n": 512, "query_target_mass": 0.2626667022705078, "query_target_nll": 2.2183456420898438, "query_top1_on_support": 0.326171875, "mean_query_support_size": 2.19140625, "probe_belief_target_mass": 0.060981638729572296, "probe_belief_target_nll": 4.267167091369629, "probe_belief_top1_on_support": 0.12890625, "mean_belief_support_size": 2.490234375 }, { "model": "shuffled", "length": 9, "query_type": "B", "k": 12, "n": 512, "query_target_mass": 0.2626667022705078, "query_target_nll": 2.2183456420898438, "query_top1_on_support": 0.326171875, "mean_query_support_size": 2.19140625, "probe_belief_target_mass": 0.060981638729572296, "probe_belief_target_nll": 4.267167091369629, "probe_belief_top1_on_support": 0.12890625, "mean_belief_support_size": 2.490234375 }, { "model": "shuffled", "length": 9, "query_type": "A_PLUS_B", "k": 0, "n": 512, "query_target_mass": 0.2274394929409027, "query_target_nll": 2.644409656524658, "query_top1_on_support": 0.244140625, "mean_query_support_size": 2.4921875, "probe_belief_target_mass": 0.02478604204952717, "probe_belief_target_nll": 14.000293731689453, "probe_belief_top1_on_support": 0.02734375, "mean_belief_support_size": 2.705078125 }, { "model": "shuffled", "length": 9, "query_type": "A_PLUS_B", "k": 1, "n": 512, "query_target_mass": 0.2284444272518158, "query_target_nll": 2.4133896827697754, "query_top1_on_support": 0.244140625, "mean_query_support_size": 2.4921875, "probe_belief_target_mass": 0.022574305534362793, "probe_belief_target_nll": 5.636862754821777, "probe_belief_top1_on_support": 0.029296875, "mean_belief_support_size": 2.705078125 }, { "model": "shuffled", "length": 9, "query_type": "A_PLUS_B", "k": 2, "n": 512, "query_target_mass": 0.22592586278915405, "query_target_nll": 2.4304609298706055, "query_top1_on_support": 0.208984375, "mean_query_support_size": 2.4921875, "probe_belief_target_mass": 0.022186677902936935, "probe_belief_target_nll": 5.147761344909668, "probe_belief_top1_on_support": 0.02734375, "mean_belief_support_size": 2.705078125 }, { "model": "shuffled", "length": 9, "query_type": "A_PLUS_B", "k": 3, "n": 512, "query_target_mass": 0.22608427703380585, "query_target_nll": 2.4367763996124268, "query_top1_on_support": 0.23046875, "mean_query_support_size": 2.4921875, "probe_belief_target_mass": 0.024930283427238464, "probe_belief_target_nll": 5.020646572113037, "probe_belief_top1_on_support": 0.029296875, "mean_belief_support_size": 2.705078125 }, { "model": "shuffled", "length": 9, "query_type": "A_PLUS_B", "k": 6, "n": 512, "query_target_mass": 0.2381986677646637, "query_target_nll": 2.3629746437072754, "query_top1_on_support": 0.26953125, "mean_query_support_size": 2.4921875, "probe_belief_target_mass": 0.03699786216020584, "probe_belief_target_nll": 4.622074604034424, "probe_belief_top1_on_support": 0.05078125, "mean_belief_support_size": 2.705078125 }, { "model": "shuffled", "length": 9, "query_type": "A_PLUS_B", "k": 9, "n": 512, "query_target_mass": 0.2400820553302765, "query_target_nll": 2.3775367736816406, "query_top1_on_support": 0.27734375, "mean_query_support_size": 2.4921875, "probe_belief_target_mass": 0.053612120449543, "probe_belief_target_nll": 4.361413955688477, "probe_belief_top1_on_support": 0.123046875, "mean_belief_support_size": 2.705078125 }, { "model": "shuffled", "length": 9, "query_type": "A_PLUS_B", "k": 12, "n": 512, "query_target_mass": 0.2400820553302765, "query_target_nll": 2.3775367736816406, "query_top1_on_support": 0.27734375, "mean_query_support_size": 2.4921875, "probe_belief_target_mass": 0.053612120449543, "probe_belief_target_nll": 4.361413955688477, "probe_belief_top1_on_support": 0.123046875, "mean_belief_support_size": 2.705078125 }, { "model": "shuffled", "length": 9, "query_type": "A_MINUS_B", "k": 0, "n": 512, "query_target_mass": 0.22265231609344482, "query_target_nll": 11.912864685058594, "query_top1_on_support": 0.22265625, "mean_query_support_size": 2.46875, "probe_belief_target_mass": 0.022297922521829605, "probe_belief_target_nll": 14.086868286132812, "probe_belief_top1_on_support": 0.017578125, "mean_belief_support_size": 2.6953125 }, { "model": "shuffled", "length": 9, "query_type": "A_MINUS_B", "k": 1, "n": 512, "query_target_mass": 0.23029755055904388, "query_target_nll": 7.923817157745361, "query_top1_on_support": 0.23046875, "mean_query_support_size": 2.46875, "probe_belief_target_mass": 0.024785548448562622, "probe_belief_target_nll": 5.595614433288574, "probe_belief_top1_on_support": 0.025390625, "mean_belief_support_size": 2.6953125 }, { "model": "shuffled", "length": 9, "query_type": "A_MINUS_B", "k": 2, "n": 512, "query_target_mass": 0.2611847519874573, "query_target_nll": 7.148374557495117, "query_top1_on_support": 0.2578125, "mean_query_support_size": 2.46875, "probe_belief_target_mass": 0.02475418895483017, "probe_belief_target_nll": 5.079769134521484, "probe_belief_top1_on_support": 0.04296875, "mean_belief_support_size": 2.6953125 }, { "model": "shuffled", "length": 9, "query_type": "A_MINUS_B", "k": 3, "n": 512, "query_target_mass": 0.24584251642227173, "query_target_nll": 5.884329795837402, "query_top1_on_support": 0.236328125, "mean_query_support_size": 2.46875, "probe_belief_target_mass": 0.02534281089901924, "probe_belief_target_nll": 4.928459644317627, "probe_belief_top1_on_support": 0.0390625, "mean_belief_support_size": 2.6953125 }, { "model": "shuffled", "length": 9, "query_type": "A_MINUS_B", "k": 6, "n": 512, "query_target_mass": 0.26017144322395325, "query_target_nll": 2.553288459777832, "query_top1_on_support": 0.26953125, "mean_query_support_size": 2.46875, "probe_belief_target_mass": 0.03736865147948265, "probe_belief_target_nll": 4.584179401397705, "probe_belief_top1_on_support": 0.080078125, "mean_belief_support_size": 2.6953125 }, { "model": "shuffled", "length": 9, "query_type": "A_MINUS_B", "k": 9, "n": 512, "query_target_mass": 0.28169873356819153, "query_target_nll": 2.2698683738708496, "query_top1_on_support": 0.314453125, "mean_query_support_size": 2.46875, "probe_belief_target_mass": 0.05709461122751236, "probe_belief_target_nll": 4.332185745239258, "probe_belief_top1_on_support": 0.130859375, "mean_belief_support_size": 2.6953125 }, { "model": "shuffled", "length": 9, "query_type": "A_MINUS_B", "k": 12, "n": 512, "query_target_mass": 0.28169873356819153, "query_target_nll": 2.2698683738708496, "query_top1_on_support": 0.314453125, "mean_query_support_size": 2.46875, "probe_belief_target_mass": 0.05709461122751236, "probe_belief_target_nll": 4.332185745239258, "probe_belief_top1_on_support": 0.130859375, "mean_belief_support_size": 2.6953125 }, { "model": "shuffled", "length": 12, "query_type": "A", "k": 0, "n": 512, "query_target_mass": 0.16397671401500702, "query_target_nll": 2.7022688388824463, "query_top1_on_support": 0.146484375, "mean_query_support_size": 1.888671875, "probe_belief_target_mass": 0.01671004854142666, "probe_belief_target_nll": 14.314071655273438, "probe_belief_top1_on_support": 0.005859375, "mean_belief_support_size": 2.05078125 }, { "model": "shuffled", "length": 12, "query_type": "A", "k": 1, "n": 512, "query_target_mass": 0.17241635918617249, "query_target_nll": 2.4179906845092773, "query_top1_on_support": 0.181640625, "mean_query_support_size": 1.888671875, "probe_belief_target_mass": 0.018306691199541092, "probe_belief_target_nll": 5.592731475830078, "probe_belief_top1_on_support": 0.015625, "mean_belief_support_size": 2.05078125 }, { "model": "shuffled", "length": 12, "query_type": "A", "k": 2, "n": 512, "query_target_mass": 0.17137941718101501, "query_target_nll": 2.4341936111450195, "query_top1_on_support": 0.1640625, "mean_query_support_size": 1.888671875, "probe_belief_target_mass": 0.01904396340250969, "probe_belief_target_nll": 5.052694320678711, "probe_belief_top1_on_support": 0.01953125, "mean_belief_support_size": 2.05078125 }, { "model": "shuffled", "length": 12, "query_type": "A", "k": 3, "n": 512, "query_target_mass": 0.17062760889530182, "query_target_nll": 2.447411060333252, "query_top1_on_support": 0.181640625, "mean_query_support_size": 1.888671875, "probe_belief_target_mass": 0.018101124092936516, "probe_belief_target_nll": 4.994239807128906, "probe_belief_top1_on_support": 0.017578125, "mean_belief_support_size": 2.05078125 }, { "model": "shuffled", "length": 12, "query_type": "A", "k": 6, "n": 512, "query_target_mass": 0.17917264997959137, "query_target_nll": 2.39338755607605, "query_top1_on_support": 0.1875, "mean_query_support_size": 1.888671875, "probe_belief_target_mass": 0.022243347018957138, "probe_belief_target_nll": 4.774113655090332, "probe_belief_top1_on_support": 0.0234375, "mean_belief_support_size": 2.05078125 }, { "model": "shuffled", "length": 12, "query_type": "A", "k": 9, "n": 512, "query_target_mass": 0.19853776693344116, "query_target_nll": 2.304701328277588, "query_top1_on_support": 0.244140625, "mean_query_support_size": 1.888671875, "probe_belief_target_mass": 0.028126992285251617, "probe_belief_target_nll": 4.60906982421875, "probe_belief_top1_on_support": 0.060546875, "mean_belief_support_size": 2.05078125 }, { "model": "shuffled", "length": 12, "query_type": "A", "k": 12, "n": 512, "query_target_mass": 0.20902091264724731, "query_target_nll": 2.2529079914093018, "query_top1_on_support": 0.27734375, "mean_query_support_size": 1.888671875, "probe_belief_target_mass": 0.032513491809368134, "probe_belief_target_nll": 4.54926872253418, "probe_belief_top1_on_support": 0.06640625, "mean_belief_support_size": 2.05078125 }, { "model": "shuffled", "length": 12, "query_type": "B", "k": 0, "n": 512, "query_target_mass": 0.15985728800296783, "query_target_nll": 2.701371669769287, "query_top1_on_support": 0.146484375, "mean_query_support_size": 1.822265625, "probe_belief_target_mass": 0.01694609597325325, "probe_belief_target_nll": 14.285931587219238, "probe_belief_top1_on_support": 0.015625, "mean_belief_support_size": 1.966796875 }, { "model": "shuffled", "length": 12, "query_type": "B", "k": 1, "n": 512, "query_target_mass": 0.166494220495224, "query_target_nll": 2.4157891273498535, "query_top1_on_support": 0.1875, "mean_query_support_size": 1.822265625, "probe_belief_target_mass": 0.017175618559122086, "probe_belief_target_nll": 5.593259334564209, "probe_belief_top1_on_support": 0.0078125, "mean_belief_support_size": 1.966796875 }, { "model": "shuffled", "length": 12, "query_type": "B", "k": 2, "n": 512, "query_target_mass": 0.16686947643756866, "query_target_nll": 2.4129903316497803, "query_top1_on_support": 0.17578125, "mean_query_support_size": 1.822265625, "probe_belief_target_mass": 0.01672542281448841, "probe_belief_target_nll": 5.116477012634277, "probe_belief_top1_on_support": 0.013671875, "mean_belief_support_size": 1.966796875 }, { "model": "shuffled", "length": 12, "query_type": "B", "k": 3, "n": 512, "query_target_mass": 0.16498301923274994, "query_target_nll": 2.4201269149780273, "query_top1_on_support": 0.169921875, "mean_query_support_size": 1.822265625, "probe_belief_target_mass": 0.017270389944314957, "probe_belief_target_nll": 5.010287284851074, "probe_belief_top1_on_support": 0.015625, "mean_belief_support_size": 1.966796875 }, { "model": "shuffled", "length": 12, "query_type": "B", "k": 6, "n": 512, "query_target_mass": 0.16785690188407898, "query_target_nll": 2.406816005706787, "query_top1_on_support": 0.169921875, "mean_query_support_size": 1.822265625, "probe_belief_target_mass": 0.01951572112739086, "probe_belief_target_nll": 4.824010372161865, "probe_belief_top1_on_support": 0.02734375, "mean_belief_support_size": 1.966796875 }, { "model": "shuffled", "length": 12, "query_type": "B", "k": 9, "n": 512, "query_target_mass": 0.17495833337306976, "query_target_nll": 2.376816749572754, "query_top1_on_support": 0.2109375, "mean_query_support_size": 1.822265625, "probe_belief_target_mass": 0.02568317949771881, "probe_belief_target_nll": 4.662356376647949, "probe_belief_top1_on_support": 0.041015625, "mean_belief_support_size": 1.966796875 }, { "model": "shuffled", "length": 12, "query_type": "B", "k": 12, "n": 512, "query_target_mass": 0.20393772423267365, "query_target_nll": 2.2669858932495117, "query_top1_on_support": 0.25, "mean_query_support_size": 1.822265625, "probe_belief_target_mass": 0.03005887195467949, "probe_belief_target_nll": 4.527053356170654, "probe_belief_top1_on_support": 0.068359375, "mean_belief_support_size": 1.966796875 }, { "model": "shuffled", "length": 12, "query_type": "A_PLUS_B", "k": 0, "n": 512, "query_target_mass": 0.17947699129581451, "query_target_nll": 2.5999796390533447, "query_top1_on_support": 0.171875, "mean_query_support_size": 1.974609375, "probe_belief_target_mass": 0.0172453373670578, "probe_belief_target_nll": 14.275390625, "probe_belief_top1_on_support": 0.015625, "mean_belief_support_size": 2.1328125 }, { "model": "shuffled", "length": 12, "query_type": "A_PLUS_B", "k": 1, "n": 512, "query_target_mass": 0.1786961704492569, "query_target_nll": 2.429314374923706, "query_top1_on_support": 0.17578125, "mean_query_support_size": 1.974609375, "probe_belief_target_mass": 0.01796809956431389, "probe_belief_target_nll": 5.586801528930664, "probe_belief_top1_on_support": 0.01953125, "mean_belief_support_size": 2.1328125 }, { "model": "shuffled", "length": 12, "query_type": "A_PLUS_B", "k": 2, "n": 512, "query_target_mass": 0.18119296431541443, "query_target_nll": 2.424450635910034, "query_top1_on_support": 0.177734375, "mean_query_support_size": 1.974609375, "probe_belief_target_mass": 0.019232457503676414, "probe_belief_target_nll": 5.053633213043213, "probe_belief_top1_on_support": 0.01953125, "mean_belief_support_size": 2.1328125 }, { "model": "shuffled", "length": 12, "query_type": "A_PLUS_B", "k": 3, "n": 512, "query_target_mass": 0.180486798286438, "query_target_nll": 2.430018424987793, "query_top1_on_support": 0.189453125, "mean_query_support_size": 1.974609375, "probe_belief_target_mass": 0.019028764218091965, "probe_belief_target_nll": 5.004889011383057, "probe_belief_top1_on_support": 0.0234375, "mean_belief_support_size": 2.1328125 }, { "model": "shuffled", "length": 12, "query_type": "A_PLUS_B", "k": 6, "n": 512, "query_target_mass": 0.18167009949684143, "query_target_nll": 2.399545907974243, "query_top1_on_support": 0.18359375, "mean_query_support_size": 1.974609375, "probe_belief_target_mass": 0.0202375166118145, "probe_belief_target_nll": 4.7864179611206055, "probe_belief_top1_on_support": 0.0234375, "mean_belief_support_size": 2.1328125 }, { "model": "shuffled", "length": 12, "query_type": "A_PLUS_B", "k": 9, "n": 512, "query_target_mass": 0.1840789020061493, "query_target_nll": 2.384106159210205, "query_top1_on_support": 0.1875, "mean_query_support_size": 1.974609375, "probe_belief_target_mass": 0.02460474520921707, "probe_belief_target_nll": 4.7002644538879395, "probe_belief_top1_on_support": 0.05859375, "mean_belief_support_size": 2.1328125 }, { "model": "shuffled", "length": 12, "query_type": "A_PLUS_B", "k": 12, "n": 512, "query_target_mass": 0.18727163970470428, "query_target_nll": 2.3824429512023926, "query_top1_on_support": 0.21484375, "mean_query_support_size": 1.974609375, "probe_belief_target_mass": 0.031488463282585144, "probe_belief_target_nll": 4.547966003417969, "probe_belief_top1_on_support": 0.08984375, "mean_belief_support_size": 2.1328125 }, { "model": "shuffled", "length": 12, "query_type": "A_MINUS_B", "k": 0, "n": 512, "query_target_mass": 0.16407039761543274, "query_target_nll": 12.162185668945312, "query_top1_on_support": 0.1640625, "mean_query_support_size": 1.939453125, "probe_belief_target_mass": 0.015687257051467896, "probe_belief_target_nll": 14.476585388183594, "probe_belief_top1_on_support": 0.017578125, "mean_belief_support_size": 1.99609375 }, { "model": "shuffled", "length": 12, "query_type": "A_MINUS_B", "k": 1, "n": 512, "query_target_mass": 0.16481223702430725, "query_target_nll": 8.272133827209473, "query_top1_on_support": 0.1640625, "mean_query_support_size": 1.939453125, "probe_belief_target_mass": 0.016030624508857727, "probe_belief_target_nll": 5.654322147369385, "probe_belief_top1_on_support": 0.015625, "mean_belief_support_size": 1.99609375 }, { "model": "shuffled", "length": 12, "query_type": "A_MINUS_B", "k": 2, "n": 512, "query_target_mass": 0.15303701162338257, "query_target_nll": 7.806595802307129, "query_top1_on_support": 0.15625, "mean_query_support_size": 1.939453125, "probe_belief_target_mass": 0.015943489968776703, "probe_belief_target_nll": 5.175835609436035, "probe_belief_top1_on_support": 0.017578125, "mean_belief_support_size": 1.99609375 }, { "model": "shuffled", "length": 12, "query_type": "A_MINUS_B", "k": 3, "n": 512, "query_target_mass": 0.19075752794742584, "query_target_nll": 6.888829231262207, "query_top1_on_support": 0.205078125, "mean_query_support_size": 1.939453125, "probe_belief_target_mass": 0.01686137728393078, "probe_belief_target_nll": 5.041329383850098, "probe_belief_top1_on_support": 0.01953125, "mean_belief_support_size": 1.99609375 }, { "model": "shuffled", "length": 12, "query_type": "A_MINUS_B", "k": 6, "n": 512, "query_target_mass": 0.1786540150642395, "query_target_nll": 3.3544788360595703, "query_top1_on_support": 0.201171875, "mean_query_support_size": 1.939453125, "probe_belief_target_mass": 0.01854195073246956, "probe_belief_target_nll": 4.824018478393555, "probe_belief_top1_on_support": 0.033203125, "mean_belief_support_size": 1.99609375 }, { "model": "shuffled", "length": 12, "query_type": "A_MINUS_B", "k": 9, "n": 512, "query_target_mass": 0.19260016083717346, "query_target_nll": 2.3989498615264893, "query_top1_on_support": 0.208984375, "mean_query_support_size": 1.939453125, "probe_belief_target_mass": 0.02308942936360836, "probe_belief_target_nll": 4.6968464851379395, "probe_belief_top1_on_support": 0.03125, "mean_belief_support_size": 1.99609375 }, { "model": "shuffled", "length": 12, "query_type": "A_MINUS_B", "k": 12, "n": 512, "query_target_mass": 0.19031649827957153, "query_target_nll": 2.3819379806518555, "query_top1_on_support": 0.216796875, "mean_query_support_size": 1.939453125, "probe_belief_target_mass": 0.026661714538931847, "probe_belief_target_nll": 4.624006271362305, "probe_belief_top1_on_support": 0.0546875, "mean_belief_support_size": 1.99609375 } ] } ], "checkpoints": [ "large_artifacts/dense_latent_query_executor/checkpoints/control_shuffled_mod11/checkpoint_final.pt" ] }