{ "schemaVersion": 1, "generatedAtUtc": "2026-04-10T17:17:11.905Z", "modelId": "gemma-4-e2b-it-q4k-ehf16-af32", "promptPackPath": "/tmp/gemma4-oneword-plus-signal.json", "promptCount": 257, "useChatTemplate": true, "topK": 64, "summaryTopK": 8, "continuationTokens": 8, "gpu": { "hasSubgroups": true, "hasSubgroupsF16": true, "hasF16": true, "hasTimestampQuery": true, "maxBufferSize": 4294967295, "maxWorkgroupSize": 1024, "maxWorkgroupStorageSize": 32768, "adapterInfo": { "vendor": "apple", "architecture": "metal-3", "device": "apple-m3", "description": "Metal driver on macOS Version 26.3 (Build 25D125)" }, "submitProbeMs": 2.0237089999999966 }, "aggregate": { "anyModeDisagreementCount": 29, "f16VsF32FlipCount": 29, "f32OrderingDisagreementCount": 0, "persistentBranchDriftCount": 15, "healedBranchDriftCount": 14 }, "highlights": { "anyModeDisagreements": [ { "id": "code-xml-open-oneword-a", "text": "Answer with exactly one word only.\n", "winnerTextsByMode": { "exact": "Start", "f32_forward": "Start", "f32_reverse": "Start", "f32_tree64": "Start", "f16_forward": "Error" }, "uniqueWinners": [ "Start", "Error" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "code-yaml-key-oneword-b", "text": "Respond with one word only.\napiVersion: v1\nkind:", "winnerTextsByMode": { "exact": "Null", "f32_forward": "Null", "f32_reverse": "Null", "f32_tree64": "Null", "f16_forward": "null" }, "uniqueWinners": [ "Null", "null" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "econ-student-debt-oneword-d", "text": "Choose one word only.\nCanceling student debt is fair or unfair to those who already paid. It is", "winnerTextsByMode": { "exact": "Complex", "f32_forward": "Complex", "f32_reverse": "Complex", "f32_tree64": "Complex", "f16_forward": "**" }, "uniqueWinners": [ "Complex", "**" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 2, "differingStepCount": 3, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "eth-clone-organ-oneword-c", "text": "No explanation. One word only.\nCloning a human to harvest organs for a dying child is moral or immoral. It is", "winnerTextsByMode": { "exact": "Im", "f32_forward": "Im", "f32_reverse": "Im", "f32_tree64": "Im", "f16_forward": "Eth" }, "uniqueWinners": [ "Im", "Eth" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "eth-forgive-unforgivable-oneword-b", "text": "Respond with one word only.\nSome acts are forgivable or unforgivable. They are", "winnerTextsByMode": { "exact": "Subject", "f32_forward": "Subject", "f32_reverse": "Subject", "f32_tree64": "Subject", "f16_forward": "Complex" }, "uniqueWinners": [ "Subject", "Complex" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "eth-piracy-poverty-oneword-c", "text": "No explanation. One word only.\nPirating textbooks when you cannot afford them is wrong or understandable. It is", "winnerTextsByMode": { "exact": "Illegal", "f32_forward": "Illegal", "f32_reverse": "Illegal", "f32_tree64": "Illegal", "f16_forward": "The" }, "uniqueWinners": [ "Illegal", "The" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "frag-but-oneword-b", "text": "Respond with one word only.\nYes, but", "winnerTextsByMode": { "exact": "Why", "f32_forward": "Why", "f32_reverse": "Why", "f32_tree64": "Why", "f16_forward": "Wait" }, "uniqueWinners": [ "Why", "Wait" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "frag-newline-only-oneword-b", "text": "Respond with one word only.\n\n", "winnerTextsByMode": { "exact": "Yes", "f32_forward": "Yes", "f32_reverse": "Yes", "f32_tree64": "Yes", "f16_forward": "Under" }, "uniqueWinners": [ "Yes", "Under" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "frag-no-oneword-c", "text": "No explanation. One word only.\nNo.", "winnerTextsByMode": { "exact": "Under", "f32_forward": "Under", "f32_reverse": "Under", "f32_tree64": "Under", "f16_forward": "Correct" }, "uniqueWinners": [ "Under", "Correct" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "frag-number-next-oneword-a", "text": "Answer with exactly one word only.\n1, 1, 2, 3, 5, 8,", "winnerTextsByMode": { "exact": "1", "f32_forward": "1", "f32_reverse": "1", "f32_tree64": "1", "f16_forward": "th" }, "uniqueWinners": [ "1", "th" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "frag-therefore-oneword-a", "text": "Answer with exactly one word only.\nTherefore,", "winnerTextsByMode": { "exact": "Thus", "f32_forward": "Thus", "f32_reverse": "Thus", "f32_tree64": "Thus", "f16_forward": "Consequently" }, "uniqueWinners": [ "Thus", "Consequently" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "grief-relief-explicit-oneword-a", "text": "Answer with exactly one word only.\nAnswer with exactly one word: grief or relief. Question: Moving a parent into hospice can bring Answer:", "winnerTextsByMode": { "exact": "g", "f32_forward": "g", "f32_reverse": "g", "f32_tree64": "g", "f16_forward": "relief" }, "uniqueWinners": [ "g", "relief" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "lang-arabic-greeting-oneword-d", "text": "Choose one word only.\nمرحبا، الجواب هو", "winnerTextsByMode": { "exact": "م", "f32_forward": "م", "f32_reverse": "م", "f32_tree64": "م", "f16_forward": "ال" }, "uniqueWinners": [ "م", "ال" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "pol-electoral-college-oneword-a", "text": "Answer with exactly one word only.\nThe electoral college is democratic or undemocratic. It is", "winnerTextsByMode": { "exact": "Amb", "f32_forward": "Amb", "f32_reverse": "Amb", "f32_tree64": "Amb", "f16_forward": "Und" }, "uniqueWinners": [ "Amb", "Und" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 2, "differingStepCount": 3, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "pol-ubi-work-oneword-d", "text": "Choose one word only.\nUniversal basic income would reduce or increase motivation to work. It would", "winnerTextsByMode": { "exact": "Increase", "f32_forward": "Increase", "f32_reverse": "Increase", "f32_tree64": "Increase", "f16_forward": "increase" }, "uniqueWinners": [ "Increase", "increase" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "punc-at-mention-oneword-c", "text": "No explanation. One word only.\n@everyone the correct answer is", "winnerTextsByMode": { "exact": "Wait", "f32_forward": "Wait", "f32_reverse": "Wait", "f32_tree64": "Wait", "f16_forward": "Correct" }, "uniqueWinners": [ "Wait", "Correct" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "punc-brackets-fill-oneword-a", "text": "Answer with exactly one word only.\nThe answer is [MASK]. The correct fill is", "winnerTextsByMode": { "exact": "Correct", "f32_forward": "Correct", "f32_reverse": "Correct", "f32_tree64": "Correct", "f16_forward": "correct" }, "uniqueWinners": [ "Correct", "correct" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "punc-dash-answer-oneword-c", "text": "No explanation. One word only.\nIs water wet? — Yes or no:", "winnerTextsByMode": { "exact": "Yes", "f32_forward": "Yes", "f32_reverse": "Yes", "f32_tree64": "Yes", "f16_forward": "No" }, "uniqueWinners": [ "Yes", "No" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "punc-exclaim-run-oneword-a", "text": "Answer with exactly one word only.\nRun! The light is", "winnerTextsByMode": { "exact": "On", "f32_forward": "On", "f32_reverse": "On", "f32_tree64": "On", "f16_forward": "Green" }, "uniqueWinners": [ "On", "Green" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "punc-tilde-approx-oneword-a", "text": "Answer with exactly one word only.\nThe value is ~3.14, which means pi is", "winnerTextsByMode": { "exact": "Pi", "f32_forward": "Pi", "f32_reverse": "Pi", "f32_tree64": "Pi", "f16_forward": "Appro" }, "uniqueWinners": [ "Pi", "Appro" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "rollback-approve-deny-oneword-c", "text": "No explanation. One word only.\nA rollback that restores a known exploitable vulnerability should be approved or denied. It should be", "winnerTextsByMode": { "exact": "Approved", "f32_forward": "Approved", "f32_reverse": "Approved", "f32_tree64": "Approved", "f16_forward": "Denied" }, "uniqueWinners": [ "Approved", "Denied" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "sci-arrow-of-time-oneword-c", "text": "No explanation. One word only.\nTime flows forward because of entropy or something deeper. It is because of", "winnerTextsByMode": { "exact": "Arrow", "f32_forward": "Arrow", "f32_reverse": "Arrow", "f32_tree64": "Arrow", "f16_forward": "Existence" }, "uniqueWinners": [ "Arrow", "Existence" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "sci-life-definition-oneword-b", "text": "Respond with one word only.\nA virus is alive or not alive. It is", "winnerTextsByMode": { "exact": "Neither", "f32_forward": "Neither", "f32_reverse": "Neither", "f32_tree64": "Neither", "f16_forward": "Not" }, "uniqueWinners": [ "Neither", "Not" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "science-model-open-answer-oneword-c", "text": "No explanation. One word only.\nAnswer with exactly one word: true or false. Question: A model can guide action even when several interpretations remain open. Answer:", "winnerTextsByMode": { "exact": "True", "f32_forward": "True", "f32_reverse": "True", "f32_tree64": "True", "f16_forward": "true" }, "uniqueWinners": [ "True", "true" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "snow-season-oneword-d", "text": "Choose one word only.\nSnow is most common in", "winnerTextsByMode": { "exact": "Winter", "f32_forward": "Winter", "f32_reverse": "Winter", "f32_tree64": "Winter", "f16_forward": "winter" }, "uniqueWinners": [ "Winter", "winter" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "tech-space-colonize-oneword-c", "text": "No explanation. One word only.\nColonizing Mars is humanity's priority or a distraction. It is", "winnerTextsByMode": { "exact": "Exist", "f32_forward": "Exist", "f32_reverse": "Exist", "f32_tree64": "Exist", "f16_forward": "Fut" }, "uniqueWinners": [ "Exist", "Fut" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "val-individual-collective-oneword-d", "text": "Choose one word only.\nIndividual rights or collective good — which comes first?", "winnerTextsByMode": { "exact": "This", "f32_forward": "This", "f32_reverse": "This", "f32_tree64": "This", "f16_forward": "Individual" }, "uniqueWinners": [ "This", "Individual" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 7, "differingStepCount": 8, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "val-normal-defined-oneword-c", "text": "No explanation. One word only.\nNormal is defined by the majority or doesn't exist. It", "winnerTextsByMode": { "exact": "Rel", "f32_forward": "Rel", "f32_reverse": "Rel", "f32_tree64": "Rel", "f16_forward": "Majority" }, "uniqueWinners": [ "Rel", "Majority" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 2, "differingStepCount": 3, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "val-privacy-security-oneword-d", "text": "Choose one word only.\nPrivacy matters more than security, or security matters more. The answer is", "winnerTextsByMode": { "exact": "Security", "f32_forward": "Security", "f32_reverse": "Security", "f32_tree64": "Security", "f16_forward": "**" }, "uniqueWinners": [ "Security", "**" ], "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 2, "differingStepCount": 3, "healedAtStep": null, "persistsThroughEnd": true } } ] }, "prompts": [ { "id": "signal-noise-explicit", "text": "Answer with exactly one word: signal or noise. Question: Dreams are Answer:", "stableTopK": [ { "rank": 1, "tokenId": 33786, "text": "signal", "logit": 16.504314422607422, "prob": 0.7562913298606873 }, { "rank": 2, "tokenId": 53159, "text": "Signal", "logit": 14.899137496948242, "prob": 0.1519041508436203 }, { "rank": 3, "tokenId": 33828, "text": "noise", "logit": 14.092058181762695, "prob": 0.06777344644069672 }, { "rank": 4, "tokenId": 73678, "text": "Noise", "logit": 12.940835952758789, "prob": 0.021433351561427116 }, { "rank": 5, "tokenId": 27413, "text": "both", "logit": 10.021210670471191, "prob": 0.001156412879936397 }, { "rank": 6, "tokenId": 22186, "text": "Both", "logit": 8.545169830322266, "prob": 0.0002642874314915389 }, { "rank": 7, "tokenId": 152593, "text": "signals", "logit": 8.153751373291016, "prob": 0.0001786840002750978 }, { "rank": 8, "tokenId": 198275, "text": "neither", "logit": 7.269305229187012, "prob": 0.00007378627196885645 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 33786, "winnerText": "signal", "winnerScore": 18.557634851985288, "winnerGap": 2.21263245933795, "top": [ { "rank": 1, "tokenId": 33786, "text": "signal", "score": 18.557634851985288 }, { "rank": 2, "tokenId": 53159, "text": "Signal", "score": 16.345002392647338 }, { "rank": 3, "tokenId": 33828, "text": "noise", "score": 15.291922943761392 }, { "rank": 4, "tokenId": 73678, "text": "Noise", "score": 13.847032480255066 }, { "rank": 5, "tokenId": 27413, "text": "both", "score": 10.421075202376969 }, { "rank": 6, "tokenId": 22186, "text": "Both", "score": 8.788217452376163 }, { "rank": 7, "tokenId": 152593, "text": "signals", "score": 8.363926622967346 }, { "rank": 8, "tokenId": 198275, "text": "neither", "score": 7.416810894506423 } ] }, "f32_forward": { "winnerTokenId": 33786, "winnerText": "signal", "winnerScore": 18.557634353637695, "winnerGap": 2.212627410888672, "top": [ { "rank": 1, "tokenId": 33786, "text": "signal", "score": 18.557634353637695 }, { "rank": 2, "tokenId": 53159, "text": "Signal", "score": 16.345006942749023 }, { "rank": 3, "tokenId": 33828, "text": "noise", "score": 15.29192066192627 }, { "rank": 4, "tokenId": 73678, "text": "Noise", "score": 13.847027778625488 }, { "rank": 5, "tokenId": 27413, "text": "both", "score": 10.421073913574219 }, { "rank": 6, "tokenId": 22186, "text": "Both", "score": 8.788220405578613 }, { "rank": 7, "tokenId": 152593, "text": "signals", "score": 8.363932609558105 }, { "rank": 8, "tokenId": 198275, "text": "neither", "score": 7.416813850402832 } ] }, "f32_reverse": { "winnerTokenId": 33786, "winnerText": "signal", "winnerScore": 18.557634353637695, "winnerGap": 2.212635040283203, "top": [ { "rank": 1, "tokenId": 33786, "text": "signal", "score": 18.557634353637695 }, { "rank": 2, "tokenId": 53159, "text": "Signal", "score": 16.344999313354492 }, { "rank": 3, "tokenId": 33828, "text": "noise", "score": 15.29192066192627 }, { "rank": 4, "tokenId": 73678, "text": "Noise", "score": 13.847033500671387 }, { "rank": 5, "tokenId": 27413, "text": "both", "score": 10.421086311340332 }, { "rank": 6, "tokenId": 22186, "text": "Both", "score": 8.788219451904297 }, { "rank": 7, "tokenId": 152593, "text": "signals", "score": 8.363922119140625 }, { "rank": 8, "tokenId": 198275, "text": "neither", "score": 7.416812896728516 } ] }, "f32_tree64": { "winnerTokenId": 33786, "winnerText": "signal", "winnerScore": 18.557632446289062, "winnerGap": 2.2126312255859375, "top": [ { "rank": 1, "tokenId": 33786, "text": "signal", "score": 18.557632446289062 }, { "rank": 2, "tokenId": 53159, "text": "Signal", "score": 16.345001220703125 }, { "rank": 3, "tokenId": 33828, "text": "noise", "score": 15.291923522949219 }, { "rank": 4, "tokenId": 73678, "text": "Noise", "score": 13.847030639648438 }, { "rank": 5, "tokenId": 27413, "text": "both", "score": 10.421074867248535 }, { "rank": 6, "tokenId": 22186, "text": "Both", "score": 8.788217544555664 }, { "rank": 7, "tokenId": 152593, "text": "signals", "score": 8.36392593383789 }, { "rank": 8, "tokenId": 198275, "text": "neither", "score": 7.416810512542725 } ] }, "f16_forward": { "winnerTokenId": 33786, "winnerText": "signal", "winnerScore": 18.84375, "winnerGap": 2.375, "top": [ { "rank": 1, "tokenId": 33786, "text": "signal", "score": 18.84375 }, { "rank": 2, "tokenId": 53159, "text": "Signal", "score": 16.46875 }, { "rank": 3, "tokenId": 33828, "text": "noise", "score": 15.453125 }, { "rank": 4, "tokenId": 73678, "text": "Noise", "score": 13.796875 }, { "rank": 5, "tokenId": 22186, "text": "Both", "score": 8.7890625 }, { "rank": 6, "tokenId": 198275, "text": "neither", "score": 7.50390625 }, { "rank": 7, "tokenId": 7925, "text": "Answer", "score": 6.9921875 }, { "rank": 8, "tokenId": 3771, "text": "No", "score": 6.8203125 } ] } }, "winnerTextsByMode": { "exact": "signal", "f32_forward": "signal", "f32_reverse": "signal", "f32_tree64": "signal", "f16_forward": "signal" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "signal" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "access-approve-deny-oneword-a", "text": "Answer with exactly one word only.\nApprove or deny: revoking access for a departed employee should be", "stableTopK": [ { "rank": 1, "tokenId": 2046, "text": "App", "logit": 6.667457103729248, "prob": 0.9998449683189392 }, { "rank": 2, "tokenId": 105961, "text": "approve", "logit": -2.3158462047576904, "prob": 0.00012546818470582366 }, { "rank": 3, "tokenId": 22472, "text": "Den", "logit": -4.6313652992248535, "prob": 0.0000123855825222563 }, { "rank": 4, "tokenId": 78850, "text": "Automatic", "logit": -6.2457404136657715, "prob": 0.000002464916633471148 }, { "rank": 5, "tokenId": 236776, "text": "A", "logit": -6.559118270874023, "prob": 0.0000018017891534327646 }, { "rank": 6, "tokenId": 27122, "text": "Allow", "logit": -6.962271213531494, "prob": 0.0000012039733974233968 }, { "rank": 7, "tokenId": 139876, "text": "Approx", "logit": -7.153145790100098, "prob": 9.947663102138904e-7 }, { "rank": 8, "tokenId": 25719, "text": "Rev", "logit": -7.515348434448242, "prob": 6.924979061295744e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 2046, "winnerText": "App", "winnerScore": 6.780606380141233, "winnerGap": 9.101072237763647, "top": [ { "rank": 1, "tokenId": 2046, "text": "App", "score": 6.780606380141233 }, { "rank": 2, "tokenId": 105961, "text": "approve", "score": -2.3204658576224144 }, { "rank": 3, "tokenId": 22472, "text": "Den", "score": -4.668693511242871 }, { "rank": 4, "tokenId": 78850, "text": "Automatic", "score": -6.338399413154655 }, { "rank": 5, "tokenId": 236776, "text": "A", "score": -6.666732480099171 }, { "rank": 6, "tokenId": 27122, "text": "Allow", "score": -7.091466828669976 }, { "rank": 7, "tokenId": 139876, "text": "Approx", "score": -7.293522948684993 }, { "rank": 8, "tokenId": 25719, "text": "Rev", "score": -7.678757621541337 } ] }, "f32_forward": { "winnerTokenId": 2046, "winnerText": "App", "winnerScore": 6.780603408813477, "winnerGap": 9.101066827774048, "top": [ { "rank": 1, "tokenId": 2046, "text": "App", "score": 6.780603408813477 }, { "rank": 2, "tokenId": 105961, "text": "approve", "score": -2.3204634189605713 }, { "rank": 3, "tokenId": 22472, "text": "Den", "score": -4.668697357177734 }, { "rank": 4, "tokenId": 78850, "text": "Automatic", "score": -6.3384013175964355 }, { "rank": 5, "tokenId": 236776, "text": "A", "score": -6.666737079620361 }, { "rank": 6, "tokenId": 27122, "text": "Allow", "score": -7.091468334197998 }, { "rank": 7, "tokenId": 139876, "text": "Approx", "score": -7.293522834777832 }, { "rank": 8, "tokenId": 25719, "text": "Rev", "score": -7.678761959075928 } ] }, "f32_reverse": { "winnerTokenId": 2046, "winnerText": "App", "winnerScore": 6.780608654022217, "winnerGap": 9.101083755493164, "top": [ { "rank": 1, "tokenId": 2046, "text": "App", "score": 6.780608654022217 }, { "rank": 2, "tokenId": 105961, "text": "approve", "score": -2.3204751014709473 }, { "rank": 3, "tokenId": 22472, "text": "Den", "score": -4.66869592666626 }, { "rank": 4, "tokenId": 78850, "text": "Automatic", "score": -6.338404178619385 }, { "rank": 5, "tokenId": 236776, "text": "A", "score": -6.666728496551514 }, { "rank": 6, "tokenId": 27122, "text": "Allow", "score": -7.091455936431885 }, { "rank": 7, "tokenId": 139876, "text": "Approx", "score": -7.293519020080566 }, { "rank": 8, "tokenId": 25719, "text": "Rev", "score": -7.67874813079834 } ] }, "f32_tree64": { "winnerTokenId": 2046, "winnerText": "App", "winnerScore": 6.780605792999268, "winnerGap": 9.101070880889893, "top": [ { "rank": 1, "tokenId": 2046, "text": "App", "score": 6.780605792999268 }, { "rank": 2, "tokenId": 105961, "text": "approve", "score": -2.320465087890625 }, { "rank": 3, "tokenId": 22472, "text": "Den", "score": -4.668692588806152 }, { "rank": 4, "tokenId": 78850, "text": "Automatic", "score": -6.3383989334106445 }, { "rank": 5, "tokenId": 236776, "text": "A", "score": -6.666732311248779 }, { "rank": 6, "tokenId": 27122, "text": "Allow", "score": -7.091466903686523 }, { "rank": 7, "tokenId": 139876, "text": "Approx", "score": -7.293522357940674 }, { "rank": 8, "tokenId": 25719, "text": "Rev", "score": -7.678756237030029 } ] }, "f16_forward": { "winnerTokenId": 2046, "winnerText": "App", "winnerScore": 6.78515625, "winnerGap": 9.1015625, "top": [ { "rank": 1, "tokenId": 2046, "text": "App", "score": 6.78515625 }, { "rank": 2, "tokenId": 105961, "text": "approve", "score": -2.31640625 }, { "rank": 3, "tokenId": 527, "text": "as", "score": -3.404296875 }, { "rank": 4, "tokenId": 22472, "text": "Den", "score": -4.70703125 }, { "rank": 5, "tokenId": 80110, "text": "automatic", "score": -4.97265625 }, { "rank": 6, "tokenId": 74819, "text": "APPRO", "score": -5.25 }, { "rank": 7, "tokenId": 100801, "text": "Require", "score": -5.66796875 }, { "rank": 8, "tokenId": 18463, "text": "Perm", "score": -6.2890625 } ] } }, "winnerTextsByMode": { "exact": "App", "f32_forward": "App", "f32_reverse": "App", "f32_tree64": "App", "f16_forward": "App" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "App" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "answer-is-oneword-b", "text": "Respond with one word only.\nThe answer is", "stableTopK": [ { "rank": 1, "tokenId": 9366, "text": "Please", "logit": 0.6187448501586914, "prob": 0.6582659482955933 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "logit": -1.1041901111602783, "prob": 0.11752770096063614 }, { "rank": 3, "tokenId": 33778, "text": "Ready", "logit": -1.202378273010254, "prob": 0.10653631389141083 }, { "rank": 4, "tokenId": 60776, "text": "Waiting", "logit": -2.3254740238189697, "prob": 0.0346532016992569 }, { "rank": 5, "tokenId": 14713, "text": "Under", "logit": -2.3688700199127197, "prob": 0.033181555569171906 }, { "rank": 6, "tokenId": 58491, "text": "waiting", "logit": -3.5644428730010986, "prob": 0.010038435459136963 }, { "rank": 7, "tokenId": 2788, "text": "ready", "logit": -3.6965787410736084, "prob": 0.008795897476375103 }, { "rank": 8, "tokenId": 41387, "text": "Unknown", "logit": -4.670698165893555, "prob": 0.003320668125525117 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 9366, "winnerText": "Please", "winnerScore": 0.618831958910377, "winnerGap": 1.7235218262250598, "top": [ { "rank": 1, "tokenId": 9366, "text": "Please", "score": 0.618831958910377 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": -1.1046898673146828 }, { "rank": 3, "tokenId": 33778, "text": "Ready", "score": -1.2030240914428418 }, { "rank": 4, "tokenId": 60776, "text": "Waiting", "score": -2.330151304516889 }, { "rank": 5, "tokenId": 14713, "text": "Under", "score": -2.373810200986341 }, { "rank": 6, "tokenId": 58491, "text": "waiting", "score": -3.581364336098325 }, { "rank": 7, "tokenId": 2788, "text": "ready", "score": -3.715458166670345 }, { "rank": 8, "tokenId": 41387, "text": "Unknown", "score": -4.7089904899990565 } ] }, "f32_forward": { "winnerTokenId": 9366, "winnerText": "Please", "winnerScore": 0.6188304424285889, "winnerGap": 1.7235243320465088, "top": [ { "rank": 1, "tokenId": 9366, "text": "Please", "score": 0.6188304424285889 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": -1.10469388961792 }, { "rank": 3, "tokenId": 33778, "text": "Ready", "score": -1.2030246257781982 }, { "rank": 4, "tokenId": 60776, "text": "Waiting", "score": -2.330150604248047 }, { "rank": 5, "tokenId": 14713, "text": "Under", "score": -2.3738107681274414 }, { "rank": 6, "tokenId": 58491, "text": "waiting", "score": -3.5813639163970947 }, { "rank": 7, "tokenId": 2788, "text": "ready", "score": -3.715456485748291 }, { "rank": 8, "tokenId": 41387, "text": "Unknown", "score": -4.708987712860107 } ] }, "f32_reverse": { "winnerTokenId": 9366, "winnerText": "Please", "winnerScore": 0.6188311576843262, "winnerGap": 1.7235175371170044, "top": [ { "rank": 1, "tokenId": 9366, "text": "Please", "score": 0.6188311576843262 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": -1.1046863794326782 }, { "rank": 3, "tokenId": 33778, "text": "Ready", "score": -1.2030224800109863 }, { "rank": 4, "tokenId": 60776, "text": "Waiting", "score": -2.330152988433838 }, { "rank": 5, "tokenId": 14713, "text": "Under", "score": -2.373811960220337 }, { "rank": 6, "tokenId": 58491, "text": "waiting", "score": -3.5813629627227783 }, { "rank": 7, "tokenId": 2788, "text": "ready", "score": -3.715458631515503 }, { "rank": 8, "tokenId": 41387, "text": "Unknown", "score": -4.708988666534424 } ] }, "f32_tree64": { "winnerTokenId": 9366, "winnerText": "Please", "winnerScore": 0.6188315153121948, "winnerGap": 1.7235208749771118, "top": [ { "rank": 1, "tokenId": 9366, "text": "Please", "score": 0.6188315153121948 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": -1.104689359664917 }, { "rank": 3, "tokenId": 33778, "text": "Ready", "score": -1.2030246257781982 }, { "rank": 4, "tokenId": 60776, "text": "Waiting", "score": -2.330151081085205 }, { "rank": 5, "tokenId": 14713, "text": "Under", "score": -2.373809576034546 }, { "rank": 6, "tokenId": 58491, "text": "waiting", "score": -3.5813639163970947 }, { "rank": 7, "tokenId": 2788, "text": "ready", "score": -3.7154581546783447 }, { "rank": 8, "tokenId": 41387, "text": "Unknown", "score": -4.708990573883057 } ] }, "f16_forward": { "winnerTokenId": 9366, "winnerText": "Please", "winnerScore": 0.409912109375, "winnerGap": 1.490966796875, "top": [ { "rank": 1, "tokenId": 9366, "text": "Please", "score": 0.409912109375 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": -1.0810546875 }, { "rank": 3, "tokenId": 33778, "text": "Ready", "score": -1.2587890625 }, { "rank": 4, "tokenId": 60776, "text": "Waiting", "score": -1.310546875 }, { "rank": 5, "tokenId": 14713, "text": "Under", "score": -2.384765625 }, { "rank": 6, "tokenId": 34230, "text": "Continue", "score": -3.517578125 }, { "rank": 7, "tokenId": 58491, "text": "waiting", "score": -3.59765625 }, { "rank": 8, "tokenId": 2788, "text": "ready", "score": -3.720703125 } ] } }, "winnerTextsByMode": { "exact": "Please", "f32_forward": "Please", "f32_reverse": "Please", "f32_tree64": "Please", "f16_forward": "Please" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Please" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "backup-yes-no-oneword-c", "text": "No explanation. One word only.\nShould you delete the only backup before a migration, yes or no? The answer is", "stableTopK": [ { "rank": 1, "tokenId": 3771, "text": "No", "logit": 17.869415283203125, "prob": 0.999758780002594 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "logit": 9.471508979797363, "prob": 0.00022528426779899746 }, { "rank": 3, "tokenId": 179306, "text": "Depends", "logit": 5.070174694061279, "prob": 0.0000027622033940133406 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "logit": 4.089475631713867, "prob": 0.0000010359611906096688 }, { "rank": 5, "tokenId": 1904, "text": "no", "logit": 3.8497464656829834, "prob": 8.151366728270659e-7 }, { "rank": 6, "tokenId": 1018, "text": "**", "logit": 3.281381607055664, "prob": 4.6173491341505724e-7 }, { "rank": 7, "tokenId": 31402, "text": "Maybe", "logit": 2.980266809463501, "prob": 3.4168053275607235e-7 }, { "rank": 8, "tokenId": 236776, "text": "A", "logit": 2.3138577938079834, "prob": 1.7546983599459054e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 20.59120363593118, "winnerGap": 10.784723688642273, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 20.59120363593118 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 9.806479947288906 }, { "rank": 3, "tokenId": 179306, "text": "Depends", "score": 5.1192884300026265 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "score": 4.115090048438775 }, { "rank": 5, "tokenId": 1904, "text": "no", "score": 3.87108979290824 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 3.2945631270051408 }, { "rank": 7, "tokenId": 31402, "text": "Maybe", "score": 2.990130773178368 }, { "rank": 8, "tokenId": 236776, "text": "A", "score": 2.3184614883258092 } ] }, "f32_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 20.591215133666992, "winnerGap": 10.784748077392578, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 20.591215133666992 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 9.806467056274414 }, { "rank": 3, "tokenId": 179306, "text": "Depends", "score": 5.119287014007568 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "score": 4.1150922775268555 }, { "rank": 5, "tokenId": 1904, "text": "no", "score": 3.87109375 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 3.2945687770843506 }, { "rank": 7, "tokenId": 31402, "text": "Maybe", "score": 2.9901301860809326 }, { "rank": 8, "tokenId": 236776, "text": "A", "score": 2.31846022605896 } ] }, "f32_reverse": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 20.5911808013916, "winnerGap": 10.784696578979492, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 20.5911808013916 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 9.80648422241211 }, { "rank": 3, "tokenId": 179306, "text": "Depends", "score": 5.119287490844727 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "score": 4.115090370178223 }, { "rank": 5, "tokenId": 1904, "text": "no", "score": 3.8710877895355225 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 3.294560432434082 }, { "rank": 7, "tokenId": 31402, "text": "Maybe", "score": 2.9901316165924072 }, { "rank": 8, "tokenId": 236776, "text": "A", "score": 2.318462371826172 } ] }, "f32_tree64": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 20.591205596923828, "winnerGap": 10.7847261428833, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 20.591205596923828 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 9.806479454040527 }, { "rank": 3, "tokenId": 179306, "text": "Depends", "score": 5.119287490844727 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "score": 4.115090847015381 }, { "rank": 5, "tokenId": 1904, "text": "no", "score": 3.871088981628418 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 3.2945632934570312 }, { "rank": 7, "tokenId": 31402, "text": "Maybe", "score": 2.990131139755249 }, { "rank": 8, "tokenId": 236776, "text": "A", "score": 2.318462610244751 } ] }, "f16_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 20.671875, "winnerGap": 10.8359375, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 20.671875 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 9.8359375 }, { "rank": 3, "tokenId": 179306, "text": "Depends", "score": 5.12890625 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "score": 4.1796875 }, { "rank": 5, "tokenId": 1904, "text": "no", "score": 3.90625 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 3.3515625 }, { "rank": 7, "tokenId": 31402, "text": "Maybe", "score": 3.0078125 }, { "rank": 8, "tokenId": 3494, "text": "Error", "score": 2.34765625 } ] } }, "winnerTextsByMode": { "exact": "No", "f32_forward": "No", "f32_reverse": "No", "f32_tree64": "No", "f16_forward": "No" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "No" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "backup-yes-no-choice-oneword-d", "text": "Choose one word only.\nChoose exactly one word: yes or no.\nShould you delete the only backup before a migration?", "stableTopK": [ { "rank": 1, "tokenId": 3771, "text": "No", "logit": 18.358154296875, "prob": 0.999778687953949 }, { "rank": 2, "tokenId": 1904, "text": "no", "logit": 9.04591178894043, "prob": 0.0000902918036445044 }, { "rank": 3, "tokenId": 1018, "text": "**", "logit": 8.520162582397461, "prob": 0.00005337260154192336 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "logit": 7.926901817321777, "prob": 0.000029489572625607252 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "logit": 7.821067810058594, "prob": 0.000026528052330831997 }, { "rank": 6, "tokenId": 113214, "text": "depends", "logit": 6.181354999542236, "prob": 0.000005147390766069293 }, { "rank": 7, "tokenId": 28911, "text": "Whether", "logit": 5.6121392250061035, "prob": 0.0000029132640975149116 }, { "rank": 8, "tokenId": 79629, "text": "Depending", "logit": 5.30575704574585, "prob": 0.0000021444689082272816 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 21.36043146485045, "winnerGap": 12.024362603326477, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 21.36043146485045 }, { "rank": 2, "tokenId": 1904, "text": "no", "score": 9.336068861523973 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 8.761006189661899 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 8.119520024140959 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 8.005850977450603 }, { "rank": 6, "tokenId": 113214, "text": "depends", "score": 6.271131715067497 }, { "rank": 7, "tokenId": 28911, "text": "Whether", "score": 5.679015876939228 }, { "rank": 8, "tokenId": 79629, "text": "Depending", "score": 5.362139563252242 } ] }, "f32_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 21.36045265197754, "winnerGap": 12.02437973022461, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 21.36045265197754 }, { "rank": 2, "tokenId": 1904, "text": "no", "score": 9.33607292175293 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 8.760993957519531 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 8.119511604309082 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 8.005846977233887 }, { "rank": 6, "tokenId": 113214, "text": "depends", "score": 6.27113151550293 }, { "rank": 7, "tokenId": 28911, "text": "Whether", "score": 5.679015636444092 }, { "rank": 8, "tokenId": 79629, "text": "Depending", "score": 5.362135887145996 } ] }, "f32_reverse": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 21.360429763793945, "winnerGap": 12.024361610412598, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 21.360429763793945 }, { "rank": 2, "tokenId": 1904, "text": "no", "score": 9.336068153381348 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 8.761007308959961 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 8.119519233703613 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 8.005853652954102 }, { "rank": 6, "tokenId": 113214, "text": "depends", "score": 6.271129608154297 }, { "rank": 7, "tokenId": 28911, "text": "Whether", "score": 5.679017066955566 }, { "rank": 8, "tokenId": 79629, "text": "Depending", "score": 5.362138748168945 } ] }, "f32_tree64": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 21.360431671142578, "winnerGap": 12.02436351776123, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 21.360431671142578 }, { "rank": 2, "tokenId": 1904, "text": "no", "score": 9.336068153381348 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 8.761005401611328 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 8.11952018737793 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 8.005851745605469 }, { "rank": 6, "tokenId": 113214, "text": "depends", "score": 6.2711310386657715 }, { "rank": 7, "tokenId": 28911, "text": "Whether", "score": 5.67901611328125 }, { "rank": 8, "tokenId": 79629, "text": "Depending", "score": 5.36214017868042 } ] }, "f16_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 21.3125, "winnerGap": 11.9296875, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 21.3125 }, { "rank": 2, "tokenId": 1904, "text": "no", "score": 9.3828125 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 8.7578125 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 8.0390625 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 7.92578125 }, { "rank": 6, "tokenId": 28911, "text": "Whether", "score": 5.625 }, { "rank": 7, "tokenId": 79629, "text": "Depending", "score": 5.38671875 }, { "rank": 8, "tokenId": 3637, "text": "Context", "score": 4.88671875 } ] } }, "winnerTextsByMode": { "exact": "No", "f32_forward": "No", "f32_reverse": "No", "f32_tree64": "No", "f16_forward": "No" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "No" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "backups-every-good-bad-oneword-a", "text": "Answer with exactly one word only.\nSkipping every backup before a database migration is good or bad. It is", "stableTopK": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "logit": 11.648452758789062, "prob": 0.9999924898147583 }, { "rank": 2, "tokenId": 15242, "text": "bad", "logit": -0.3648487329483032, "prob": 0.0000060629809013335034 }, { "rank": 3, "tokenId": 11947, "text": "Good", "logit": -3.4514479637145996, "prob": 2.7681727488015895e-7 }, { "rank": 4, "tokenId": 26546, "text": "Very", "logit": -3.5939671993255615, "prob": 2.4004788201636984e-7 }, { "rank": 5, "tokenId": 37701, "text": "Ter", "logit": -4.17251443862915, "prob": 1.3459781200708676e-7 }, { "rank": 6, "tokenId": 99510, "text": "Ris", "logit": -4.9989423751831055, "prob": 5.8901303390257453e-8 }, { "rank": 7, "tokenId": 79151, "text": "Harm", "logit": -5.223831653594971, "prob": 4.703885281287512e-8 }, { "rank": 8, "tokenId": 4287, "text": " bad", "logit": -5.318650245666504, "prob": 4.278362197851493e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 12.293251951694344, "winnerGap": 12.658122877134648, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 12.293251951694344 }, { "rank": 2, "tokenId": 15242, "text": "bad", "score": -0.36487092544030375 }, { "rank": 3, "tokenId": 11947, "text": "Good", "score": -3.4667982460123383 }, { "rank": 4, "tokenId": 26546, "text": "Very", "score": -3.6113091768393524 }, { "rank": 5, "tokenId": 37701, "text": "Ter", "score": -4.199733821312906 }, { "rank": 6, "tokenId": 99510, "text": "Ris", "score": -5.045996320542656 }, { "rank": 7, "tokenId": 79151, "text": "Harm", "score": -5.277610224629711 }, { "rank": 8, "tokenId": 4287, "text": " bad", "score": -5.375447913040448 } ] }, "f32_forward": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 12.293254852294922, "winnerGap": 12.65812760591507, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 12.293254852294922 }, { "rank": 2, "tokenId": 15242, "text": "bad", "score": -0.3648727536201477 }, { "rank": 3, "tokenId": 11947, "text": "Good", "score": -3.4667952060699463 }, { "rank": 4, "tokenId": 26546, "text": "Very", "score": -3.6113121509552 }, { "rank": 5, "tokenId": 37701, "text": "Ter", "score": -4.199731349945068 }, { "rank": 6, "tokenId": 99510, "text": "Ris", "score": -5.0459885597229 }, { "rank": 7, "tokenId": 79151, "text": "Harm", "score": -5.27761173248291 }, { "rank": 8, "tokenId": 4287, "text": " bad", "score": -5.3754472732543945 } ] }, "f32_reverse": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 12.293257713317871, "winnerGap": 12.658129125833511, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 12.293257713317871 }, { "rank": 2, "tokenId": 15242, "text": "bad", "score": -0.36487141251564026 }, { "rank": 3, "tokenId": 11947, "text": "Good", "score": -3.466798782348633 }, { "rank": 4, "tokenId": 26546, "text": "Very", "score": -3.611314296722412 }, { "rank": 5, "tokenId": 37701, "text": "Ter", "score": -4.199732780456543 }, { "rank": 6, "tokenId": 99510, "text": "Ris", "score": -5.046003341674805 }, { "rank": 7, "tokenId": 79151, "text": "Harm", "score": -5.277614593505859 }, { "rank": 8, "tokenId": 4287, "text": " bad", "score": -5.375448226928711 } ] }, "f32_tree64": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 12.293251037597656, "winnerGap": 12.658122658729553, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 12.293251037597656 }, { "rank": 2, "tokenId": 15242, "text": "bad", "score": -0.364871621131897 }, { "rank": 3, "tokenId": 11947, "text": "Good", "score": -3.4667983055114746 }, { "rank": 4, "tokenId": 26546, "text": "Very", "score": -3.611309289932251 }, { "rank": 5, "tokenId": 37701, "text": "Ter", "score": -4.199734687805176 }, { "rank": 6, "tokenId": 99510, "text": "Ris", "score": -5.045997619628906 }, { "rank": 7, "tokenId": 79151, "text": "Harm", "score": -5.277610778808594 }, { "rank": 8, "tokenId": 4287, "text": " bad", "score": -5.375447750091553 } ] }, "f16_forward": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 12.4765625, "winnerGap": 13.810546875, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 12.4765625 }, { "rank": 2, "tokenId": 15242, "text": "bad", "score": -1.333984375 }, { "rank": 3, "tokenId": 11947, "text": "Good", "score": -3.416015625 }, { "rank": 4, "tokenId": 26546, "text": "Very", "score": -3.619140625 }, { "rank": 5, "tokenId": 37701, "text": "Ter", "score": -4.19140625 }, { "rank": 6, "tokenId": 32453, "text": "Mad", "score": -4.3203125 }, { "rank": 7, "tokenId": 17574, "text": "Because", "score": -4.37890625 }, { "rank": 8, "tokenId": 71869, "text": "Cru", "score": -4.55078125 } ] } }, "winnerTextsByMode": { "exact": "Bad", "f32_forward": "Bad", "f32_reverse": "Bad", "f32_tree64": "Bad", "f16_forward": "Bad" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Bad" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "backups-good-bad-oneword-b", "text": "Respond with one word only.\nSkipping all backups before a database migration is good or bad. It is", "stableTopK": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "logit": 11.974600791931152, "prob": 0.9999865293502808 }, { "rank": 2, "tokenId": 15242, "text": "bad", "logit": 0.6816419959068298, "prob": 0.000012460183825169224 }, { "rank": 3, "tokenId": 37701, "text": "Ter", "logit": -3.4674835205078125, "prob": 1.9659937322558108e-7 }, { "rank": 4, "tokenId": 26546, "text": "Very", "logit": -3.807947874069214, "prob": 1.398686322318099e-7 }, { "rank": 5, "tokenId": 99510, "text": "Ris", "logit": -4.129586219787598, "prob": 1.0139920902929589e-7 }, { "rank": 6, "tokenId": 11947, "text": "Good", "logit": -4.339842796325684, "prob": 8.21715175902682e-8 }, { "rank": 7, "tokenId": 4287, "text": " bad", "logit": -4.584710121154785, "prob": 6.432455279536953e-8 }, { "rank": 8, "tokenId": 13741, "text": " Bad", "logit": -4.9275102615356445, "prob": 4.5656285863060475e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 12.679242935891715, "winnerGap": 11.997486559825479, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 12.679242935891715 }, { "rank": 2, "tokenId": 15242, "text": "bad", "score": 0.6817563760662368 }, { "rank": 3, "tokenId": 37701, "text": "Ter", "score": -3.483046688231269 }, { "rank": 4, "tokenId": 26546, "text": "Very", "score": -3.828597769635131 }, { "rank": 5, "tokenId": 99510, "text": "Ris", "score": -4.155968818389169 }, { "rank": 6, "tokenId": 11947, "text": "Good", "score": -4.370502969693987 }, { "rank": 7, "tokenId": 4287, "text": " bad", "score": -4.620910068968314 }, { "rank": 8, "tokenId": 13741, "text": " Bad", "score": -4.972554860807676 } ] }, "f32_forward": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 12.67924690246582, "winnerGap": 11.997491300106049, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 12.67924690246582 }, { "rank": 2, "tokenId": 15242, "text": "bad", "score": 0.6817556023597717 }, { "rank": 3, "tokenId": 37701, "text": "Ter", "score": -3.483055591583252 }, { "rank": 4, "tokenId": 26546, "text": "Very", "score": -3.8285984992980957 }, { "rank": 5, "tokenId": 99510, "text": "Ris", "score": -4.155965805053711 }, { "rank": 6, "tokenId": 11947, "text": "Good", "score": -4.3705034255981445 }, { "rank": 7, "tokenId": 4287, "text": " bad", "score": -4.620917320251465 }, { "rank": 8, "tokenId": 13741, "text": " Bad", "score": -4.972553730010986 } ] }, "f32_reverse": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 12.679235458374023, "winnerGap": 11.997479021549225, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 12.679235458374023 }, { "rank": 2, "tokenId": 15242, "text": "bad", "score": 0.6817564368247986 }, { "rank": 3, "tokenId": 37701, "text": "Ter", "score": -3.4830524921417236 }, { "rank": 4, "tokenId": 26546, "text": "Very", "score": -3.8286001682281494 }, { "rank": 5, "tokenId": 99510, "text": "Ris", "score": -4.155977249145508 }, { "rank": 6, "tokenId": 11947, "text": "Good", "score": -4.370502471923828 }, { "rank": 7, "tokenId": 4287, "text": " bad", "score": -4.620905876159668 }, { "rank": 8, "tokenId": 13741, "text": " Bad", "score": -4.972560405731201 } ] }, "f32_tree64": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 12.679242134094238, "winnerGap": 11.997485876083374, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 12.679242134094238 }, { "rank": 2, "tokenId": 15242, "text": "bad", "score": 0.6817562580108643 }, { "rank": 3, "tokenId": 37701, "text": "Ter", "score": -3.4830470085144043 }, { "rank": 4, "tokenId": 26546, "text": "Very", "score": -3.8285984992980957 }, { "rank": 5, "tokenId": 99510, "text": "Ris", "score": -4.155968189239502 }, { "rank": 6, "tokenId": 11947, "text": "Good", "score": -4.370504379272461 }, { "rank": 7, "tokenId": 4287, "text": " bad", "score": -4.620909690856934 }, { "rank": 8, "tokenId": 13741, "text": " Bad", "score": -4.9725542068481445 } ] }, "f16_forward": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 12.640625, "winnerGap": 11.98046875, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 12.640625 }, { "rank": 2, "tokenId": 15242, "text": "bad", "score": 0.66015625 }, { "rank": 3, "tokenId": 4287, "text": " bad", "score": -0.39599609375 }, { "rank": 4, "tokenId": 37701, "text": "Ter", "score": -3.48046875 }, { "rank": 5, "tokenId": 26546, "text": "Very", "score": -3.810546875 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": -4.09375 }, { "rank": 7, "tokenId": 99510, "text": "Ris", "score": -4.14453125 }, { "rank": 8, "tokenId": 17056, "text": "Bet", "score": -4.15234375 } ] } }, "winnerTextsByMode": { "exact": "Bad", "f32_forward": "Bad", "f32_reverse": "Bad", "f32_tree64": "Bad", "f16_forward": "Bad" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Bad" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "brakes-safe-unsafe-oneword-c", "text": "No explanation. One word only.\nDriving without brakes is safe or unsafe. It is", "stableTopK": [ { "rank": 1, "tokenId": 2805, "text": "Un", "logit": 3.6155996322631836, "prob": 0.9999992847442627 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "logit": -11.010031700134277, "prob": 4.448040158422373e-7 }, { "rank": 3, "tokenId": 236536, "text": "Dangerous", "logit": -12.72441577911377, "prob": 8.00979051973627e-8 }, { "rank": 4, "tokenId": 38699, "text": "Una", "logit": -13.766220092773438, "prob": 2.8259945494824024e-8 }, { "rank": 5, "tokenId": 1018, "text": "**", "logit": -14.03886890411377, "prob": 2.1515996095899936e-8 }, { "rank": 6, "tokenId": 236836, "text": "U", "logit": -14.5042085647583, "prob": 1.351036260643923e-8 }, { "rank": 7, "tokenId": 151488, "text": "Uns", "logit": -14.818994522094727, "prob": 9.861822469758863e-9 }, { "rank": 8, "tokenId": 154745, "text": "Impossible", "logit": -15.336409568786621, "prob": 5.878231679190549e-9 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 3.6332588984533167, "winnerGap": 15.181854319604593, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 3.6332588984533167 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": -11.548595421151276 }, { "rank": 3, "tokenId": 236536, "text": "Dangerous", "score": -13.582143577233758 }, { "rank": 4, "tokenId": 38699, "text": "Una", "score": -14.876523858374256 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": -15.223743257731257 }, { "rank": 6, "tokenId": 236836, "text": "U", "score": -15.825232923760641 }, { "rank": 7, "tokenId": 151488, "text": "Uns", "score": -16.238811684731978 }, { "rank": 8, "tokenId": 154745, "text": "Impossible", "score": -16.93114056863551 } ] }, "f32_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 3.633263349533081, "winnerGap": 15.181869268417358, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 3.633263349533081 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": -11.548605918884277 }, { "rank": 3, "tokenId": 236536, "text": "Dangerous", "score": -13.582138061523438 }, { "rank": 4, "tokenId": 38699, "text": "Una", "score": -14.876513481140137 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": -15.223730087280273 }, { "rank": 6, "tokenId": 236836, "text": "U", "score": -15.825233459472656 }, { "rank": 7, "tokenId": 151488, "text": "Uns", "score": -16.238801956176758 }, { "rank": 8, "tokenId": 154745, "text": "Impossible", "score": -16.931142807006836 } ] }, "f32_reverse": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 3.6332552433013916, "winnerGap": 15.181856393814087, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 3.6332552433013916 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": -11.548601150512695 }, { "rank": 3, "tokenId": 236536, "text": "Dangerous", "score": -13.582132339477539 }, { "rank": 4, "tokenId": 38699, "text": "Una", "score": -14.876513481140137 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": -15.223753929138184 }, { "rank": 6, "tokenId": 236836, "text": "U", "score": -15.825251579284668 }, { "rank": 7, "tokenId": 151488, "text": "Uns", "score": -16.23880386352539 }, { "rank": 8, "tokenId": 154745, "text": "Impossible", "score": -16.931177139282227 } ] }, "f32_tree64": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 3.6332597732543945, "winnerGap": 15.181853294372559, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 3.6332597732543945 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": -11.548593521118164 }, { "rank": 3, "tokenId": 236536, "text": "Dangerous", "score": -13.58214282989502 }, { "rank": 4, "tokenId": 38699, "text": "Una", "score": -14.87652587890625 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": -15.223743438720703 }, { "rank": 6, "tokenId": 236836, "text": "U", "score": -15.82523250579834 }, { "rank": 7, "tokenId": 151488, "text": "Uns", "score": -16.238811492919922 }, { "rank": 8, "tokenId": 154745, "text": "Impossible", "score": -16.93113899230957 } ] }, "f16_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 3.810546875, "winnerGap": 15.341796875, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 3.810546875 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": -11.53125 }, { "rank": 3, "tokenId": 236536, "text": "Dangerous", "score": -13.5 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": -14.4140625 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "score": -14.4375 }, { "rank": 6, "tokenId": 38699, "text": "Una", "score": -14.9296875 }, { "rank": 7, "tokenId": 236836, "text": "U", "score": -15.859375 }, { "rank": 8, "tokenId": 108, "text": "\n\n", "score": -15.90625 } ] } }, "winnerTextsByMode": { "exact": "Un", "f32_forward": "Un", "f32_reverse": "Un", "f32_tree64": "Un", "f16_forward": "Un" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Un" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "brakes-safe-unsafe-breadth-oneword-d", "text": "Choose one word only.\nDriving without brakes is safe or unsafe. It is", "stableTopK": [ { "rank": 1, "tokenId": 2805, "text": "Un", "logit": 16.225160598754883, "prob": 0.9532223343849182 }, { "rank": 2, "tokenId": 1018, "text": "**", "logit": 13.1898775100708, "prob": 0.04581287130713463 }, { "rank": 3, "tokenId": 30908, "text": "unsafe", "logit": 9.309770584106445, "prob": 0.0009459723951295018 }, { "rank": 4, "tokenId": 236536, "text": "Dangerous", "logit": 5.156055927276611, "prob": 0.00001485740995121887 }, { "rank": 5, "tokenId": 5213, "text": " **", "logit": 2.3043298721313477, "prob": 8.579347081649757e-7 }, { "rank": 6, "tokenId": 236836, "text": "U", "logit": 2.168553352355957, "prob": 7.490093025808164e-7 }, { "rank": 7, "tokenId": 198, "text": "", "logit": 1.8163859844207764, "prob": 5.266752509669459e-7 }, { "rank": 8, "tokenId": 38699, "text": "Una", "logit": 1.1738896369934082, "prob": 2.770194384993374e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 18.16021069652368, "winnerGap": 4.00583506362638, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 18.16021069652368 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 14.1543756328973 }, { "rank": 3, "tokenId": 30908, "text": "unsafe", "score": 9.627181842172435 }, { "rank": 4, "tokenId": 236536, "text": "Dangerous", "score": 5.207745360165845 }, { "rank": 5, "tokenId": 5213, "text": " **", "score": 2.308880672310174 }, { "rank": 6, "tokenId": 236836, "text": "U", "score": 2.1723406787190953 }, { "rank": 7, "tokenId": 198, "text": "", "score": 1.8186107499072222 }, { "rank": 8, "tokenId": 38699, "text": "Una", "score": 1.1744907655986196 } ] }, "f32_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 18.16021728515625, "winnerGap": 4.005847930908203, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 18.16021728515625 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 14.154369354248047 }, { "rank": 3, "tokenId": 30908, "text": "unsafe", "score": 9.627185821533203 }, { "rank": 4, "tokenId": 236536, "text": "Dangerous", "score": 5.207757949829102 }, { "rank": 5, "tokenId": 5213, "text": " **", "score": 2.3088791370391846 }, { "rank": 6, "tokenId": 236836, "text": "U", "score": 2.17234206199646 }, { "rank": 7, "tokenId": 198, "text": "", "score": 1.8186146020889282 }, { "rank": 8, "tokenId": 38699, "text": "Una", "score": 1.1744927167892456 } ] }, "f32_reverse": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 18.16019630432129, "winnerGap": 4.00581169128418, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 18.16019630432129 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 14.15438461303711 }, { "rank": 3, "tokenId": 30908, "text": "unsafe", "score": 9.627182960510254 }, { "rank": 4, "tokenId": 236536, "text": "Dangerous", "score": 5.2077484130859375 }, { "rank": 5, "tokenId": 5213, "text": " **", "score": 2.3088784217834473 }, { "rank": 6, "tokenId": 236836, "text": "U", "score": 2.172344446182251 }, { "rank": 7, "tokenId": 198, "text": "", "score": 1.8186124563217163 }, { "rank": 8, "tokenId": 38699, "text": "Una", "score": 1.1744920015335083 } ] }, "f32_tree64": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 18.16020965576172, "winnerGap": 4.005834579467773, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 18.16020965576172 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 14.154375076293945 }, { "rank": 3, "tokenId": 30908, "text": "unsafe", "score": 9.627182006835938 }, { "rank": 4, "tokenId": 236536, "text": "Dangerous", "score": 5.2077460289001465 }, { "rank": 5, "tokenId": 5213, "text": " **", "score": 2.3088810443878174 }, { "rank": 6, "tokenId": 236836, "text": "U", "score": 2.1723403930664062 }, { "rank": 7, "tokenId": 198, "text": "", "score": 1.8186111450195312 }, { "rank": 8, "tokenId": 38699, "text": "Una", "score": 1.1744906902313232 } ] }, "f16_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 18.125, "winnerGap": 3.46875, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 18.125 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 14.65625 }, { "rank": 3, "tokenId": 30908, "text": "unsafe", "score": 9.625 }, { "rank": 4, "tokenId": 236536, "text": "Dangerous", "score": 5.23046875 }, { "rank": 5, "tokenId": 5213, "text": " **", "score": 2.298828125 }, { "rank": 6, "tokenId": 198, "text": "", "score": 1.8125 }, { "rank": 7, "tokenId": 68176, "text": "Outside", "score": 0.91455078125 }, { "rank": 8, "tokenId": 236829, "text": "*", "score": 0.2041015625 } ] } }, "winnerTextsByMode": { "exact": "Un", "f32_forward": "Un", "f32_reverse": "Un", "f32_tree64": "Un", "f16_forward": "Un" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Un" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "brakes-safe-unsafe-choice-oneword-a", "text": "Answer with exactly one word only.\nChoose exactly one word: safe or unsafe.\nDriving without brakes is", "stableTopK": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "logit": 18.273195266723633, "prob": 0.9927919507026672 }, { "rank": 2, "tokenId": 2805, "text": "Un", "logit": 13.346732139587402, "prob": 0.007199834566563368 }, { "rank": 3, "tokenId": 8069, "text": "uns", "logit": 6.410251617431641, "prob": 0.0000069959596658009104 }, { "rank": 4, "tokenId": 602, "text": "un", "logit": 4.610195159912109, "prob": 0.0000011563590760488296 }, { "rank": 5, "tokenId": 25793, "text": " unsafe", "logit": 0.45536690950393677, "prob": 1.8141522062364857e-8 }, { "rank": 6, "tokenId": 25083, "text": "unsigned", "logit": -1.0986534357070923, "prob": 3.835051831657665e-9 }, { "rank": 7, "tokenId": 1018, "text": "**", "logit": -1.8032135963439941, "prob": 1.8957657665907846e-9 }, { "rank": 8, "tokenId": 3743, "text": "UN", "logit": -2.036742925643921, "prob": 1.5009429255030682e-9 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 21.224989182081767, "winnerGap": 6.875619764489965, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 21.224989182081767 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 14.349369417591802 }, { "rank": 3, "tokenId": 8069, "text": "uns", "score": 6.510575010166371 }, { "rank": 4, "tokenId": 602, "text": "un", "score": 4.647009833591124 }, { "rank": 5, "tokenId": 25793, "text": " unsafe", "score": 0.45540755507199826 }, { "rank": 6, "tokenId": 25083, "text": "unsigned", "score": -1.099148332082585 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -1.8053884996443035 }, { "rank": 8, "tokenId": 3743, "text": "UN", "score": -2.039880665827204 } ] }, "f32_forward": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 21.22499656677246, "winnerGap": 6.8756256103515625, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 21.22499656677246 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 14.349370956420898 }, { "rank": 3, "tokenId": 8069, "text": "uns", "score": 6.510571002960205 }, { "rank": 4, "tokenId": 602, "text": "un", "score": 4.647006511688232 }, { "rank": 5, "tokenId": 25793, "text": " unsafe", "score": 0.4554040729999542 }, { "rank": 6, "tokenId": 25083, "text": "unsigned", "score": -1.0991476774215698 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -1.805388331413269 }, { "rank": 8, "tokenId": 3743, "text": "UN", "score": -2.0398800373077393 } ] }, "f32_reverse": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 21.224987030029297, "winnerGap": 6.875607490539551, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 21.224987030029297 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 14.349379539489746 }, { "rank": 3, "tokenId": 8069, "text": "uns", "score": 6.510573863983154 }, { "rank": 4, "tokenId": 602, "text": "un", "score": 4.647010803222656 }, { "rank": 5, "tokenId": 25793, "text": " unsafe", "score": 0.45540350675582886 }, { "rank": 6, "tokenId": 25083, "text": "unsigned", "score": -1.0991517305374146 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -1.8053901195526123 }, { "rank": 8, "tokenId": 3743, "text": "UN", "score": -2.039884567260742 } ] }, "f32_tree64": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 21.224990844726562, "winnerGap": 6.875621795654297, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 21.224990844726562 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 14.349369049072266 }, { "rank": 3, "tokenId": 8069, "text": "uns", "score": 6.5105743408203125 }, { "rank": 4, "tokenId": 602, "text": "un", "score": 4.647010803222656 }, { "rank": 5, "tokenId": 25793, "text": " unsafe", "score": 0.45540761947631836 }, { "rank": 6, "tokenId": 25083, "text": "unsigned", "score": -1.0991480350494385 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -1.805389165878296 }, { "rank": 8, "tokenId": 3743, "text": "UN", "score": -2.0398802757263184 } ] }, "f16_forward": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 21.1875, "winnerGap": 6.7421875, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 21.1875 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 14.4453125 }, { "rank": 3, "tokenId": 8069, "text": "uns", "score": 6.55859375 }, { "rank": 4, "tokenId": 602, "text": "un", "score": 4.6015625 }, { "rank": 5, "tokenId": 25793, "text": " unsafe", "score": 0.467529296875 }, { "rank": 6, "tokenId": 236840, "text": "[", "score": -0.65869140625 }, { "rank": 7, "tokenId": 25083, "text": "unsigned", "score": -1.076171875 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": -1.8173828125 } ] } }, "winnerTextsByMode": { "exact": "unsafe", "f32_forward": "unsafe", "f32_reverse": "unsafe", "f32_tree64": "unsafe", "f16_forward": "unsafe" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "unsafe" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "brakes-true-false-choice-oneword-b", "text": "Respond with one word only.\nChoose exactly one word: true or false.\nDriving without brakes is safe.", "stableTopK": [ { "rank": 1, "tokenId": 4530, "text": "false", "logit": 19.092586517333984, "prob": 0.6744020581245422 }, { "rank": 2, "tokenId": 9277, "text": "False", "logit": 18.36441993713379, "prob": 0.32559680938720703 }, { "rank": 3, "tokenId": 25874, "text": "FALSE", "logit": 5.476070404052734, "prob": 8.228886372307898e-7 }, { "rank": 4, "tokenId": 236811, "text": "F", "logit": 4.11952018737793, "prob": 2.1193311283695948e-7 }, { "rank": 5, "tokenId": 2416, "text": " false", "logit": 3.0044727325439453, "prob": 6.94928061761857e-8 }, { "rank": 6, "tokenId": 236760, "text": "f", "logit": 0.931054949760437, "prob": 8.739084123021712e-9 }, { "rank": 7, "tokenId": 3397, "text": "true", "logit": 0.8892071843147278, "prob": 8.380919958028699e-9 }, { "rank": 8, "tokenId": 4339, "text": "True", "logit": -0.11816106736660004, "prob": 3.060534048415775e-9 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 22.563977824026324, "winnerGap": 1.193520406152544, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 22.563977824026324 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 21.37045741787378 }, { "rank": 3, "tokenId": 25874, "text": "FALSE", "score": 5.538136895452309 }, { "rank": 4, "tokenId": 236811, "text": "F", "score": 4.145713813929298 }, { "rank": 5, "tokenId": 2416, "text": " false", "score": 3.014580781758468 }, { "rank": 6, "tokenId": 236760, "text": "f", "score": 0.9313549883506838 }, { "rank": 7, "tokenId": 3397, "text": "true", "score": 0.8894686908577967 }, { "rank": 8, "tokenId": 4339, "text": "True", "score": -0.11816447088246207 } ] }, "f32_forward": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 22.5639705657959, "winnerGap": 1.1934986114501953, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 22.5639705657959 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 21.370471954345703 }, { "rank": 3, "tokenId": 25874, "text": "FALSE", "score": 5.538138389587402 }, { "rank": 4, "tokenId": 236811, "text": "F", "score": 4.145724773406982 }, { "rank": 5, "tokenId": 2416, "text": " false", "score": 3.0145821571350098 }, { "rank": 6, "tokenId": 236760, "text": "f", "score": 0.9313519597053528 }, { "rank": 7, "tokenId": 3397, "text": "true", "score": 0.8894754648208618 }, { "rank": 8, "tokenId": 4339, "text": "True", "score": -0.11816754937171936 } ] }, "f32_reverse": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 22.56395721435547, "winnerGap": 1.1935005187988281, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 22.56395721435547 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 21.37045669555664 }, { "rank": 3, "tokenId": 25874, "text": "FALSE", "score": 5.538132667541504 }, { "rank": 4, "tokenId": 236811, "text": "F", "score": 4.145711421966553 }, { "rank": 5, "tokenId": 2416, "text": " false", "score": 3.014582633972168 }, { "rank": 6, "tokenId": 236760, "text": "f", "score": 0.9313561320304871 }, { "rank": 7, "tokenId": 3397, "text": "true", "score": 0.8894753456115723 }, { "rank": 8, "tokenId": 4339, "text": "True", "score": -0.1181669682264328 } ] }, "f32_tree64": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 22.563976287841797, "winnerGap": 1.1935176849365234, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 22.563976287841797 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 21.370458602905273 }, { "rank": 3, "tokenId": 25874, "text": "FALSE", "score": 5.538136005401611 }, { "rank": 4, "tokenId": 236811, "text": "F", "score": 4.145713806152344 }, { "rank": 5, "tokenId": 2416, "text": " false", "score": 3.0145797729492188 }, { "rank": 6, "tokenId": 236760, "text": "f", "score": 0.9313545227050781 }, { "rank": 7, "tokenId": 3397, "text": "true", "score": 0.889467716217041 }, { "rank": 8, "tokenId": 4339, "text": "True", "score": -0.11816620826721191 } ] }, "f16_forward": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 22.609375, "winnerGap": 1.203125, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 22.609375 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 21.40625 }, { "rank": 3, "tokenId": 25874, "text": "FALSE", "score": 5.640625 }, { "rank": 4, "tokenId": 236811, "text": "F", "score": 4.21484375 }, { "rank": 5, "tokenId": 2416, "text": " false", "score": 3.02734375 }, { "rank": 6, "tokenId": 236760, "text": "f", "score": 0.93896484375 }, { "rank": 7, "tokenId": 3397, "text": "true", "score": 0.92822265625 }, { "rank": 8, "tokenId": 4339, "text": "True", "score": -0.11285400390625 } ] } }, "winnerTextsByMode": { "exact": "false", "f32_forward": "false", "f32_reverse": "false", "f32_tree64": "false", "f16_forward": "false" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "false" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "bread-yes-no-legally-oneword-c", "text": "No explanation. One word only.\nYes, no, or legally: a person who steals bread to feed a child is guilty", "stableTopK": [ { "rank": 1, "tokenId": 3771, "text": "No", "logit": 14.942498207092285, "prob": 0.9998301863670349 }, { "rank": 2, "tokenId": 19770, "text": "Leg", "logit": 5.38206148147583, "prob": 0.00007045004895189777 }, { "rank": 3, "tokenId": 4348, "text": "Not", "logit": 4.563064098358154, "prob": 0.000031059560569701716 }, { "rank": 4, "tokenId": 67289, "text": "Legal", "logit": 4.518180847167969, "prob": 0.00002969632623717189 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "logit": 4.008238315582275, "prob": 0.0000178335376403993 }, { "rank": 6, "tokenId": 11746, "text": "Gu", "logit": 2.820866584777832, "prob": 0.0000054396191444539 }, { "rank": 7, "tokenId": 1904, "text": "no", "logit": 1.1593230962753296, "prob": 0.0000010326884876121767 }, { "rank": 8, "tokenId": 2273, "text": "leg", "logit": 0.7486069202423096, "prob": 6.848533189440786e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 16.40260747909744, "winnerGap": 10.96166502113396, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 16.40260747909744 }, { "rank": 2, "tokenId": 19770, "text": "Leg", "score": 5.44094245796348 }, { "rank": 3, "tokenId": 4348, "text": "Not", "score": 4.598749739423241 }, { "rank": 4, "tokenId": 67289, "text": "Legal", "score": 4.552815252617675 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 4.032346946895132 }, { "rank": 6, "tokenId": 11746, "text": "Gu", "score": 2.829226484952386 }, { "rank": 7, "tokenId": 1904, "text": "no", "score": 1.1599005989618636 }, { "rank": 8, "tokenId": 2273, "text": "leg", "score": 0.7487610019331612 } ] }, "f32_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 16.40260887145996, "winnerGap": 10.961671829223633, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 16.40260887145996 }, { "rank": 2, "tokenId": 19770, "text": "Leg", "score": 5.440937042236328 }, { "rank": 3, "tokenId": 4348, "text": "Not", "score": 4.598752498626709 }, { "rank": 4, "tokenId": 67289, "text": "Legal", "score": 4.552817344665527 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 4.032349586486816 }, { "rank": 6, "tokenId": 11746, "text": "Gu", "score": 2.8292267322540283 }, { "rank": 7, "tokenId": 1904, "text": "no", "score": 1.159902572631836 }, { "rank": 8, "tokenId": 2273, "text": "leg", "score": 0.7487613558769226 } ] }, "f32_reverse": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 16.402606964111328, "winnerGap": 10.961668968200684, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 16.402606964111328 }, { "rank": 2, "tokenId": 19770, "text": "Leg", "score": 5.4409379959106445 }, { "rank": 3, "tokenId": 4348, "text": "Not", "score": 4.5987467765808105 }, { "rank": 4, "tokenId": 67289, "text": "Legal", "score": 4.5528154373168945 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 4.032343864440918 }, { "rank": 6, "tokenId": 11746, "text": "Gu", "score": 2.829225778579712 }, { "rank": 7, "tokenId": 1904, "text": "no", "score": 1.1599011421203613 }, { "rank": 8, "tokenId": 2273, "text": "leg", "score": 0.748760461807251 } ] }, "f32_tree64": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 16.402606964111328, "winnerGap": 10.96166467666626, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 16.402606964111328 }, { "rank": 2, "tokenId": 19770, "text": "Leg", "score": 5.440942287445068 }, { "rank": 3, "tokenId": 4348, "text": "Not", "score": 4.598750114440918 }, { "rank": 4, "tokenId": 67289, "text": "Legal", "score": 4.5528154373168945 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 4.032347679138184 }, { "rank": 6, "tokenId": 11746, "text": "Gu", "score": 2.829226493835449 }, { "rank": 7, "tokenId": 1904, "text": "no", "score": 1.1599006652832031 }, { "rank": 8, "tokenId": 2273, "text": "leg", "score": 0.7487621307373047 } ] }, "f16_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 16.5625, "winnerGap": 11.06640625, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 16.5625 }, { "rank": 2, "tokenId": 19770, "text": "Leg", "score": 5.49609375 }, { "rank": 3, "tokenId": 4348, "text": "Not", "score": 4.6328125 }, { "rank": 4, "tokenId": 67289, "text": "Legal", "score": 4.60546875 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 4.03125 }, { "rank": 6, "tokenId": 11746, "text": "Gu", "score": 2.892578125 }, { "rank": 7, "tokenId": 1904, "text": "no", "score": 0.91259765625 }, { "rank": 8, "tokenId": 2273, "text": "leg", "score": 0.79638671875 } ] } }, "winnerTextsByMode": { "exact": "No", "f32_forward": "No", "f32_reverse": "No", "f32_tree64": "No", "f16_forward": "No" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "No" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "buy-sell-bubble-oneword-d", "text": "Choose one word only.\nAnswer with exactly one word: buy or sell.\nNote: The market description is intentionally underspecified.", "stableTopK": [ { "rank": 1, "tokenId": 20786, "text": "buy", "logit": 16.02264976501465, "prob": 0.9980447888374329 }, { "rank": 2, "tokenId": 44916, "text": "sell", "logit": 9.784550666809082, "prob": 0.0019497459288686514 }, { "rank": 3, "tokenId": 27015, "text": "Buy", "logit": 3.421765089035034, "prob": 0.0000033624464776949026 }, { "rank": 4, "tokenId": 198275, "text": "neither", "logit": 1.784865140914917, "prob": 6.542725827785034e-7 }, { "rank": 5, "tokenId": 187671, "text": "analyze", "logit": 0.6806783676147461, "prob": 2.168785044887045e-7 }, { "rank": 6, "tokenId": 106, "text": "", "logit": -0.017837045714259148, "prob": 1.0785868198581738e-7 }, { "rank": 7, "tokenId": 1018, "text": "**", "logit": -0.1131027340888977, "prob": 9.805771838955479e-8 }, { "rank": 8, "tokenId": 35724, "text": "choose", "logit": -0.5319991111755371, "prob": 6.449965184174289e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 20786, "winnerText": "buy", "winnerScore": 17.87543411732221, "winnerGap": 7.719962664182599, "top": [ { "rank": 1, "tokenId": 20786, "text": "buy", "score": 17.87543411732221 }, { "rank": 2, "tokenId": 44916, "text": "sell", "score": 10.155471453139612 }, { "rank": 3, "tokenId": 27015, "text": "Buy", "score": 3.4367220640816 }, { "rank": 4, "tokenId": 198275, "text": "neither", "score": 1.7869772882005748 }, { "rank": 5, "tokenId": 187671, "text": "analyze", "score": 0.6807959291429846 }, { "rank": 6, "tokenId": 106, "text": "", "score": -0.017835076272561423 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -0.11310289434791798 }, { "rank": 8, "tokenId": 35724, "text": "choose", "score": -0.532057898243437 } ] }, "f32_forward": { "winnerTokenId": 20786, "winnerText": "buy", "winnerScore": 17.87543487548828, "winnerGap": 7.719963073730469, "top": [ { "rank": 1, "tokenId": 20786, "text": "buy", "score": 17.87543487548828 }, { "rank": 2, "tokenId": 44916, "text": "sell", "score": 10.155471801757812 }, { "rank": 3, "tokenId": 27015, "text": "Buy", "score": 3.436715602874756 }, { "rank": 4, "tokenId": 198275, "text": "neither", "score": 1.7869765758514404 }, { "rank": 5, "tokenId": 187671, "text": "analyze", "score": 0.6808000802993774 }, { "rank": 6, "tokenId": 106, "text": "", "score": -0.017832158133387566 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -0.11310411244630814 }, { "rank": 8, "tokenId": 35724, "text": "choose", "score": -0.5320572257041931 } ] }, "f32_reverse": { "winnerTokenId": 20786, "winnerText": "buy", "winnerScore": 17.87541961669922, "winnerGap": 7.719947814941406, "top": [ { "rank": 1, "tokenId": 20786, "text": "buy", "score": 17.87541961669922 }, { "rank": 2, "tokenId": 44916, "text": "sell", "score": 10.155471801757812 }, { "rank": 3, "tokenId": 27015, "text": "Buy", "score": 3.4367213249206543 }, { "rank": 4, "tokenId": 198275, "text": "neither", "score": 1.7869772911071777 }, { "rank": 5, "tokenId": 187671, "text": "analyze", "score": 0.6808091402053833 }, { "rank": 6, "tokenId": 106, "text": "", "score": -0.017832890152931213 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -0.11310544610023499 }, { "rank": 8, "tokenId": 35724, "text": "choose", "score": -0.532058835029602 } ] }, "f32_tree64": { "winnerTokenId": 20786, "winnerText": "buy", "winnerScore": 17.87543296813965, "winnerGap": 7.719961166381836, "top": [ { "rank": 1, "tokenId": 20786, "text": "buy", "score": 17.87543296813965 }, { "rank": 2, "tokenId": 44916, "text": "sell", "score": 10.155471801757812 }, { "rank": 3, "tokenId": 27015, "text": "Buy", "score": 3.436720371246338 }, { "rank": 4, "tokenId": 198275, "text": "neither", "score": 1.7869768142700195 }, { "rank": 5, "tokenId": 187671, "text": "analyze", "score": 0.6807955503463745 }, { "rank": 6, "tokenId": 106, "text": "", "score": -0.017835021018981934 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -0.11310291290283203 }, { "rank": 8, "tokenId": 35724, "text": "choose", "score": -0.5320584774017334 } ] }, "f16_forward": { "winnerTokenId": 20786, "winnerText": "buy", "winnerScore": 17.859375, "winnerGap": 11.6015625, "top": [ { "rank": 1, "tokenId": 20786, "text": "buy", "score": 17.859375 }, { "rank": 2, "tokenId": 44916, "text": "sell", "score": 6.2578125 }, { "rank": 3, "tokenId": 27015, "text": "Buy", "score": 3.353515625 }, { "rank": 4, "tokenId": 198275, "text": "neither", "score": 1.826171875 }, { "rank": 5, "tokenId": 187671, "text": "analyze", "score": 0.78955078125 }, { "rank": 6, "tokenId": 106, "text": "", "score": -0.012603759765625 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -0.10845947265625 }, { "rank": 8, "tokenId": 35724, "text": "choose", "score": -0.5185546875 } ] } }, "winnerTextsByMode": { "exact": "buy", "f32_forward": "buy", "f32_reverse": "buy", "f32_tree64": "buy", "f16_forward": "buy" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "buy" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "canada-direction-oneword-a", "text": "Answer with exactly one word only.\nCanada is mostly", "stableTopK": [ { "rank": 1, "tokenId": 235312, "text": "Diverse", "logit": -2.054910659790039, "prob": 0.28463029861450195 }, { "rank": 2, "tokenId": 25907, "text": "North", "logit": -2.1344614028930664, "prob": 0.26286494731903076 }, { "rank": 3, "tokenId": 31534, "text": "Large", "logit": -2.6841070652008057, "prob": 0.15171362459659576 }, { "rank": 4, "tokenId": 27832, "text": "English", "logit": -2.9105653762817383, "prob": 0.1209692507982254 }, { "rank": 5, "tokenId": 236799, "text": "B", "logit": -3.3661978244781494, "prob": 0.07670017331838608 }, { "rank": 6, "tokenId": 43335, "text": "Winter", "logit": -3.980910062789917, "prob": 0.04147918522357941 }, { "rank": 7, "tokenId": 236847, "text": "V", "logit": -5.389687538146973, "prob": 0.010139252059161663 }, { "rank": 8, "tokenId": 66020, "text": "Cold", "logit": -5.774416446685791, "prob": 0.006901131011545658 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 235312, "winnerText": "Diverse", "winnerScore": -2.0581396688123306, "winnerGap": 0.07993210218326396, "top": [ { "rank": 1, "tokenId": 235312, "text": "Diverse", "score": -2.0581396688123306 }, { "rank": 2, "tokenId": 25907, "text": "North", "score": -2.1380717709955945 }, { "rank": 3, "tokenId": 31534, "text": "Large", "score": -2.691302539169129 }, { "rank": 4, "tokenId": 27832, "text": "English", "score": -2.919745803716303 }, { "rank": 5, "tokenId": 236799, "text": "B", "score": -3.3804285986321645 }, { "rank": 6, "tokenId": 43335, "text": "Winter", "score": -4.004528326642898 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": -5.448823038897739 }, { "rank": 8, "tokenId": 66020, "text": "Cold", "score": -5.847358257968821 } ] }, "f32_forward": { "winnerTokenId": 235312, "winnerText": "Diverse", "winnerScore": -2.0581321716308594, "winnerGap": 0.07994389533996582, "top": [ { "rank": 1, "tokenId": 235312, "text": "Diverse", "score": -2.0581321716308594 }, { "rank": 2, "tokenId": 25907, "text": "North", "score": -2.138076066970825 }, { "rank": 3, "tokenId": 31534, "text": "Large", "score": -2.691296100616455 }, { "rank": 4, "tokenId": 27832, "text": "English", "score": -2.9197466373443604 }, { "rank": 5, "tokenId": 236799, "text": "B", "score": -3.3804285526275635 }, { "rank": 6, "tokenId": 43335, "text": "Winter", "score": -4.004523277282715 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": -5.448822975158691 }, { "rank": 8, "tokenId": 66020, "text": "Cold", "score": -5.847361087799072 } ] }, "f32_reverse": { "winnerTokenId": 235312, "winnerText": "Diverse", "winnerScore": -2.058138370513916, "winnerGap": 0.07993412017822266, "top": [ { "rank": 1, "tokenId": 235312, "text": "Diverse", "score": -2.058138370513916 }, { "rank": 2, "tokenId": 25907, "text": "North", "score": -2.1380724906921387 }, { "rank": 3, "tokenId": 31534, "text": "Large", "score": -2.69130802154541 }, { "rank": 4, "tokenId": 27832, "text": "English", "score": -2.9197468757629395 }, { "rank": 5, "tokenId": 236799, "text": "B", "score": -3.380427122116089 }, { "rank": 6, "tokenId": 43335, "text": "Winter", "score": -4.004531383514404 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": -5.448825359344482 }, { "rank": 8, "tokenId": 66020, "text": "Cold", "score": -5.847360610961914 } ] }, "f32_tree64": { "winnerTokenId": 235312, "winnerText": "Diverse", "winnerScore": -2.0581393241882324, "winnerGap": 0.07993173599243164, "top": [ { "rank": 1, "tokenId": 235312, "text": "Diverse", "score": -2.0581393241882324 }, { "rank": 2, "tokenId": 25907, "text": "North", "score": -2.138071060180664 }, { "rank": 3, "tokenId": 31534, "text": "Large", "score": -2.69130277633667 }, { "rank": 4, "tokenId": 27832, "text": "English", "score": -2.919745922088623 }, { "rank": 5, "tokenId": 236799, "text": "B", "score": -3.380429744720459 }, { "rank": 6, "tokenId": 43335, "text": "Winter", "score": -4.004527568817139 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": -5.448821544647217 }, { "rank": 8, "tokenId": 66020, "text": "Cold", "score": -5.847356796264648 } ] }, "f16_forward": { "winnerTokenId": 235312, "winnerText": "Diverse", "winnerScore": -2.08203125, "winnerGap": 0.05859375, "top": [ { "rank": 1, "tokenId": 235312, "text": "Diverse", "score": -2.08203125 }, { "rank": 2, "tokenId": 25907, "text": "North", "score": -2.140625 }, { "rank": 3, "tokenId": 17224, "text": "Green", "score": -2.658203125 }, { "rank": 4, "tokenId": 31534, "text": "Large", "score": -2.68359375 }, { "rank": 5, "tokenId": 27832, "text": "English", "score": -2.9453125 }, { "rank": 6, "tokenId": 236799, "text": "B", "score": -3.37890625 }, { "rank": 7, "tokenId": 43335, "text": "Winter", "score": -4.03515625 }, { "rank": 8, "tokenId": 236811, "text": "F", "score": -4.08203125 } ] } }, "winnerTextsByMode": { "exact": "Diverse", "f32_forward": "Diverse", "f32_reverse": "Diverse", "f32_tree64": "Diverse", "f16_forward": "Diverse" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Diverse" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "canada-north-south-oneword-b", "text": "Respond with one word only.\nCanada is mostly north or south. It is", "stableTopK": [ { "rank": 1, "tokenId": 25907, "text": "North", "logit": -2.0754599571228027, "prob": 0.9983834028244019 }, { "rank": 2, "tokenId": 12059, "text": "Long", "logit": -8.681429862976074, "prob": 0.0013500847853720188 }, { "rank": 3, "tokenId": 31534, "text": "Large", "logit": -11.019028663635254, "prob": 0.00013036314339842647 }, { "rank": 4, "tokenId": 72982, "text": "Northern", "logit": -12.365761756896973, "prob": 0.000033905962482094765 }, { "rank": 5, "tokenId": 36227, "text": "north", "logit": -12.949938774108887, "prob": 0.000018904760509030893 }, { "rank": 6, "tokenId": 36043, "text": "Central", "logit": -13.011026382446289, "prob": 0.000017784479496185668 }, { "rank": 7, "tokenId": 95889, "text": "Neither", "logit": -13.273036003112793, "prob": 0.00001368522225675406 }, { "rank": 8, "tokenId": 5843, "text": "long", "logit": -13.656455993652344, "prob": 0.000009326843610324431 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 25907, "winnerText": "North", "winnerScore": -2.0787796693194807, "winnerGap": 6.857938014683391, "top": [ { "rank": 1, "tokenId": 25907, "text": "North", "score": -2.0787796693194807 }, { "rank": 2, "tokenId": 12059, "text": "Long", "score": -8.936717684002872 }, { "rank": 3, "tokenId": 31534, "text": "Large", "score": -11.558983138305074 }, { "rank": 4, "tokenId": 72982, "text": "Northern", "score": -13.147475037544567 }, { "rank": 5, "tokenId": 36227, "text": "north", "score": -13.858216100865935 }, { "rank": 6, "tokenId": 36043, "text": "Central", "score": -13.933379864508723 }, { "rank": 7, "tokenId": 95889, "text": "Neither", "score": -14.257616233571298 }, { "rank": 8, "tokenId": 5843, "text": "long", "score": -14.737774415341562 } ] }, "f32_forward": { "winnerTokenId": 25907, "winnerText": "North", "winnerScore": -2.0787830352783203, "winnerGap": 6.857927322387695, "top": [ { "rank": 1, "tokenId": 25907, "text": "North", "score": -2.0787830352783203 }, { "rank": 2, "tokenId": 12059, "text": "Long", "score": -8.936710357666016 }, { "rank": 3, "tokenId": 31534, "text": "Large", "score": -11.558991432189941 }, { "rank": 4, "tokenId": 72982, "text": "Northern", "score": -13.147468566894531 }, { "rank": 5, "tokenId": 36227, "text": "north", "score": -13.858221054077148 }, { "rank": 6, "tokenId": 36043, "text": "Central", "score": -13.933375358581543 }, { "rank": 7, "tokenId": 95889, "text": "Neither", "score": -14.25761890411377 }, { "rank": 8, "tokenId": 5843, "text": "long", "score": -14.737771987915039 } ] }, "f32_reverse": { "winnerTokenId": 25907, "winnerText": "North", "winnerScore": -2.078767776489258, "winnerGap": 6.8579511642456055, "top": [ { "rank": 1, "tokenId": 25907, "text": "North", "score": -2.078767776489258 }, { "rank": 2, "tokenId": 12059, "text": "Long", "score": -8.936718940734863 }, { "rank": 3, "tokenId": 31534, "text": "Large", "score": -11.558985710144043 }, { "rank": 4, "tokenId": 72982, "text": "Northern", "score": -13.1474609375 }, { "rank": 5, "tokenId": 36227, "text": "north", "score": -13.858231544494629 }, { "rank": 6, "tokenId": 36043, "text": "Central", "score": -13.933396339416504 }, { "rank": 7, "tokenId": 95889, "text": "Neither", "score": -14.25762939453125 }, { "rank": 8, "tokenId": 5843, "text": "long", "score": -14.73776626586914 } ] }, "f32_tree64": { "winnerTokenId": 25907, "winnerText": "North", "winnerScore": -2.078779697418213, "winnerGap": 6.857937335968018, "top": [ { "rank": 1, "tokenId": 25907, "text": "North", "score": -2.078779697418213 }, { "rank": 2, "tokenId": 12059, "text": "Long", "score": -8.93671703338623 }, { "rank": 3, "tokenId": 31534, "text": "Large", "score": -11.558982849121094 }, { "rank": 4, "tokenId": 72982, "text": "Northern", "score": -13.14747428894043 }, { "rank": 5, "tokenId": 36227, "text": "north", "score": -13.858217239379883 }, { "rank": 6, "tokenId": 36043, "text": "Central", "score": -13.933378219604492 }, { "rank": 7, "tokenId": 95889, "text": "Neither", "score": -14.257614135742188 }, { "rank": 8, "tokenId": 5843, "text": "long", "score": -14.737775802612305 } ] }, "f16_forward": { "winnerTokenId": 25907, "winnerText": "North", "winnerScore": -2.046875, "winnerGap": 2.90234375, "top": [ { "rank": 1, "tokenId": 25907, "text": "North", "score": -2.046875 }, { "rank": 2, "tokenId": 12059, "text": "Long", "score": -4.94921875 }, { "rank": 3, "tokenId": 36227, "text": "north", "score": -9.859375 }, { "rank": 4, "tokenId": 31534, "text": "Large", "score": -11.5546875 }, { "rank": 5, "tokenId": 72982, "text": "Northern", "score": -12.140625 }, { "rank": 6, "tokenId": 36043, "text": "Central", "score": -13.9921875 }, { "rank": 7, "tokenId": 95889, "text": "Neither", "score": -14.3515625 }, { "rank": 8, "tokenId": 20163, "text": "West", "score": -15.203125 } ] } }, "winnerTextsByMode": { "exact": "North", "f32_forward": "North", "f32_reverse": "North", "f32_tree64": "North", "f16_forward": "North" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "North" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "capital-france-oneword-c", "text": "No explanation. One word only.\nThe capital of France is", "stableTopK": [ { "rank": 1, "tokenId": 50429, "text": "Paris", "logit": 7.951321601867676, "prob": 1 }, { "rank": 2, "tokenId": 1018, "text": "**", "logit": -17.23946762084961, "prob": 1.1475701465479826e-11 }, { "rank": 3, "tokenId": 2318, "text": "\\[", "logit": -19.825077056884766, "prob": 8.646957932664789e-13 }, { "rank": 4, "tokenId": 9079, "text": " Paris", "logit": -20.147220611572266, "prob": 6.265534965854691e-13 }, { "rank": 5, "tokenId": 142253, "text": "巴黎", "logit": -20.303604125976562, "prob": 5.358480586331593e-13 }, { "rank": 6, "tokenId": 123389, "text": " প্যার", "logit": -20.552913665771484, "prob": 4.176071381062352e-13 }, { "rank": 7, "tokenId": 54444, "text": "Vers", "logit": -20.764156341552734, "prob": 3.3808538108838493e-13 }, { "rank": 8, "tokenId": 3128, "text": "Par", "logit": -21.172372817993164, "prob": 2.2477096681301767e-13 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 50429, "winnerText": "Paris", "winnerScore": 8.145787996570846, "winnerGap": 27.778882000137834, "top": [ { "rank": 1, "tokenId": 50429, "text": "Paris", "score": 8.145787996570846 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": -19.63309400356699 }, { "rank": 3, "tokenId": 2318, "text": "\\[", "score": -23.828889334380573 }, { "rank": 4, "tokenId": 9079, "text": " Paris", "score": -24.40814503855998 }, { "rank": 5, "tokenId": 142253, "text": "巴黎", "score": -24.69483713521825 }, { "rank": 6, "tokenId": 123389, "text": " প্যার", "score": -25.1597342520667 }, { "rank": 7, "tokenId": 54444, "text": "Vers", "score": -25.561482292641912 }, { "rank": 8, "tokenId": 3128, "text": "Par", "score": -26.359696914651266 } ] }, "f32_forward": { "winnerTokenId": 50429, "winnerText": "Paris", "winnerScore": 8.145794868469238, "winnerGap": 27.778895378112793, "top": [ { "rank": 1, "tokenId": 50429, "text": "Paris", "score": 8.145794868469238 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": -19.633100509643555 }, { "rank": 3, "tokenId": 2318, "text": "\\[", "score": -23.828886032104492 }, { "rank": 4, "tokenId": 9079, "text": " Paris", "score": -24.40813446044922 }, { "rank": 5, "tokenId": 142253, "text": "巴黎", "score": -24.694833755493164 }, { "rank": 6, "tokenId": 123389, "text": " প্যার", "score": -25.159738540649414 }, { "rank": 7, "tokenId": 54444, "text": "Vers", "score": -25.56146812438965 }, { "rank": 8, "tokenId": 3128, "text": "Par", "score": -26.35969352722168 } ] }, "f32_reverse": { "winnerTokenId": 50429, "winnerText": "Paris", "winnerScore": 8.145784378051758, "winnerGap": 27.778879165649414, "top": [ { "rank": 1, "tokenId": 50429, "text": "Paris", "score": 8.145784378051758 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": -19.633094787597656 }, { "rank": 3, "tokenId": 2318, "text": "\\[", "score": -23.828901290893555 }, { "rank": 4, "tokenId": 9079, "text": " Paris", "score": -24.40814971923828 }, { "rank": 5, "tokenId": 142253, "text": "巴黎", "score": -24.694847106933594 }, { "rank": 6, "tokenId": 123389, "text": " প্যার", "score": -25.159727096557617 }, { "rank": 7, "tokenId": 54444, "text": "Vers", "score": -25.56149673461914 }, { "rank": 8, "tokenId": 3128, "text": "Par", "score": -26.359704971313477 } ] }, "f32_tree64": { "winnerTokenId": 50429, "winnerText": "Paris", "winnerScore": 8.145788192749023, "winnerGap": 27.77888298034668, "top": [ { "rank": 1, "tokenId": 50429, "text": "Paris", "score": 8.145788192749023 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": -19.633094787597656 }, { "rank": 3, "tokenId": 2318, "text": "\\[", "score": -23.828887939453125 }, { "rank": 4, "tokenId": 9079, "text": " Paris", "score": -24.40814971923828 }, { "rank": 5, "tokenId": 142253, "text": "巴黎", "score": -24.69483757019043 }, { "rank": 6, "tokenId": 123389, "text": " প্যার", "score": -25.159738540649414 }, { "rank": 7, "tokenId": 54444, "text": "Vers", "score": -25.561481475830078 }, { "rank": 8, "tokenId": 3128, "text": "Par", "score": -26.359699249267578 } ] }, "f16_forward": { "winnerTokenId": 50429, "winnerText": "Paris", "winnerScore": 9.109375, "winnerGap": 21.1953125, "top": [ { "rank": 1, "tokenId": 50429, "text": "Paris", "score": 9.109375 }, { "rank": 2, "tokenId": 158659, "text": "paris", "score": -12.0859375 }, { "rank": 3, "tokenId": 89946, "text": "Berlin", "score": -15.640625 }, { "rank": 4, "tokenId": 31382, "text": "Though", "score": -16.203125 }, { "rank": 5, "tokenId": 9592, "text": " पे", "score": -17.4375 }, { "rank": 6, "tokenId": 236774, "text": "T", "score": -18.359375 }, { "rank": 7, "tokenId": 48117, "text": " paris", "score": -18.6875 }, { "rank": 8, "tokenId": 11640, "text": "Pa", "score": -18.796875 } ] } }, "winnerTextsByMode": { "exact": "Paris", "f32_forward": "Paris", "f32_reverse": "Paris", "f32_tree64": "Paris", "f16_forward": "Paris" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Paris" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "chem-equilibrium-stop-answer-oneword-d", "text": "Choose one word only.\nAnswer with exactly one word: true or false. Question: At chemical equilibrium, molecules stop reacting. Answer:", "stableTopK": [ { "rank": 1, "tokenId": 4530, "text": "false", "logit": 10.420459747314453, "prob": 0.9999151825904846 }, { "rank": 2, "tokenId": 9277, "text": "False", "logit": 0.9650787711143494, "prob": 0.00007826063665561378 }, { "rank": 3, "tokenId": 3397, "text": "true", "logit": -1.5325255393981934, "prob": 0.0000064394325818284415 }, { "rank": 4, "tokenId": 25874, "text": "FALSE", "logit": -6.659106731414795, "prob": 3.822976779588316e-8 }, { "rank": 5, "tokenId": 4339, "text": "True", "logit": -7.508113384246826, "prob": 1.6356214160850868e-8 }, { "rank": 6, "tokenId": 1018, "text": "**", "logit": -7.977055549621582, "prob": 1.0233490499444997e-8 }, { "rank": 7, "tokenId": 2416, "text": " false", "logit": -8.06071949005127, "prob": 9.412153723076244e-9 }, { "rank": 8, "tokenId": 236799, "text": "B", "logit": -10.361306190490723, "prob": 9.430980441038628e-10 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 10.872765857641587, "winnerGap": 9.907357547346164, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 10.872765857641587 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 0.9654083102954232 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": -1.5338619237150688 }, { "rank": 4, "tokenId": 25874, "text": "FALSE", "score": -6.771816135202725 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -7.6710449601471735 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -8.17346069411549 }, { "rank": 7, "tokenId": 2416, "text": " false", "score": -8.263560343010361 }, { "rank": 8, "tokenId": 236799, "text": "B", "score": -10.805550251160286 } ] }, "f32_forward": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 10.872770309448242, "winnerGap": 9.90735912322998, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 10.872770309448242 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 0.9654111862182617 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": -1.5338568687438965 }, { "rank": 4, "tokenId": 25874, "text": "FALSE", "score": -6.771824359893799 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -7.671045780181885 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -8.173465728759766 }, { "rank": 7, "tokenId": 2416, "text": " false", "score": -8.263567924499512 }, { "rank": 8, "tokenId": 236799, "text": "B", "score": -10.805546760559082 } ] }, "f32_reverse": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 10.872774124145508, "winnerGap": 9.907369494438171, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 10.872774124145508 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 0.9654046297073364 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": -1.533858299255371 }, { "rank": 4, "tokenId": 25874, "text": "FALSE", "score": -6.771810531616211 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -7.6710357666015625 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -8.173460006713867 }, { "rank": 7, "tokenId": 2416, "text": " false", "score": -8.263558387756348 }, { "rank": 8, "tokenId": 236799, "text": "B", "score": -10.805553436279297 } ] }, "f32_tree64": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 10.872764587402344, "winnerGap": 9.90735411643982, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 10.872764587402344 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 0.9654104709625244 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": -1.5338613986968994 }, { "rank": 4, "tokenId": 25874, "text": "FALSE", "score": -6.771815299987793 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -7.67104434967041 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -8.173460960388184 }, { "rank": 7, "tokenId": 2416, "text": " false", "score": -8.263561248779297 }, { "rank": 8, "tokenId": 236799, "text": "B", "score": -10.805551528930664 } ] }, "f16_forward": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 10.8828125, "winnerGap": 9.97021484375, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 10.8828125 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 0.91259765625 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": -1.56640625 }, { "rank": 4, "tokenId": 25874, "text": "FALSE", "score": -2.755859375 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -3.75 }, { "rank": 6, "tokenId": 236799, "text": "B", "score": -6.859375 }, { "rank": 7, "tokenId": 5742, "text": "fa", "score": -6.9921875 }, { "rank": 8, "tokenId": 236776, "text": "A", "score": -7.43359375 } ] } }, "winnerTextsByMode": { "exact": "false", "f32_forward": "false", "f32_reverse": "false", "f32_tree64": "false", "f16_forward": "false" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "false" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "child-adult-explicit-oneword-a", "text": "Answer with exactly one word only.\nAnswer with exactly one word: child or adult. Question: A seventeen-year-old raising siblings is Answer:", "stableTopK": [ { "rank": 1, "tokenId": 82625, "text": "adult", "logit": 5.429299831390381, "prob": 0.9962266683578491 }, { "rank": 2, "tokenId": 149267, "text": "youth", "logit": -0.9719092845916748, "prob": 0.001653287443332374 }, { "rank": 3, "tokenId": 21128, "text": "young", "logit": -1.0208768844604492, "prob": 0.001574280089698732 }, { "rank": 4, "tokenId": 171736, "text": "adoles", "logit": -3.01455020904541, "prob": 0.00021440784621518105 }, { "rank": 5, "tokenId": 12839, "text": "teen", "logit": -3.368236541748047, "prob": 0.0001505347026977688 }, { "rank": 6, "tokenId": 10869, "text": "child", "logit": -3.982077121734619, "prob": 0.00008147964399540797 }, { "rank": 7, "tokenId": 84136, "text": "Adult", "logit": -4.049315929412842, "prob": 0.00007618117524543777 }, { "rank": 8, "tokenId": 553, "text": "ad", "logit": -5.449274063110352, "prob": 0.00001878683178802021 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 82625, "winnerText": "adult", "winnerScore": 5.489767149225294, "winnerGap": 6.462018838519107, "top": [ { "rank": 1, "tokenId": 82625, "text": "adult", "score": 5.489767149225294 }, { "rank": 2, "tokenId": 149267, "text": "youth", "score": -0.9722516892938131 }, { "rank": 3, "tokenId": 21128, "text": "young", "score": -1.0212622011776027 }, { "rank": 4, "tokenId": 171736, "text": "adoles", "score": -3.0247432900587086 }, { "rank": 5, "tokenId": 12839, "text": "teen", "score": -3.3824888035520644 }, { "rank": 6, "tokenId": 10869, "text": "child", "score": -4.005711380864952 }, { "rank": 7, "tokenId": 84136, "text": "Adult", "score": -4.074177140833569 }, { "rank": 8, "tokenId": 553, "text": "ad", "score": -5.510419282980333 } ] }, "f32_forward": { "winnerTokenId": 82625, "winnerText": "adult", "winnerScore": 5.489770412445068, "winnerGap": 6.462024986743927, "top": [ { "rank": 1, "tokenId": 82625, "text": "adult", "score": 5.489770412445068 }, { "rank": 2, "tokenId": 149267, "text": "youth", "score": -0.9722545742988586 }, { "rank": 3, "tokenId": 21128, "text": "young", "score": -1.021262764930725 }, { "rank": 4, "tokenId": 171736, "text": "adoles", "score": -3.0247364044189453 }, { "rank": 5, "tokenId": 12839, "text": "teen", "score": -3.3824892044067383 }, { "rank": 6, "tokenId": 10869, "text": "child", "score": -4.005709171295166 }, { "rank": 7, "tokenId": 84136, "text": "Adult", "score": -4.074180603027344 }, { "rank": 8, "tokenId": 553, "text": "ad", "score": -5.510412216186523 } ] }, "f32_reverse": { "winnerTokenId": 82625, "winnerText": "adult", "winnerScore": 5.4897685050964355, "winnerGap": 6.462022244930267, "top": [ { "rank": 1, "tokenId": 82625, "text": "adult", "score": 5.4897685050964355 }, { "rank": 2, "tokenId": 149267, "text": "youth", "score": -0.9722537398338318 }, { "rank": 3, "tokenId": 21128, "text": "young", "score": -1.0212608575820923 }, { "rank": 4, "tokenId": 171736, "text": "adoles", "score": -3.0247318744659424 }, { "rank": 5, "tokenId": 12839, "text": "teen", "score": -3.3824806213378906 }, { "rank": 6, "tokenId": 10869, "text": "child", "score": -4.00569486618042 }, { "rank": 7, "tokenId": 84136, "text": "Adult", "score": -4.074174404144287 }, { "rank": 8, "tokenId": 553, "text": "ad", "score": -5.510412693023682 } ] }, "f32_tree64": { "winnerTokenId": 82625, "winnerText": "adult", "winnerScore": 5.489768028259277, "winnerGap": 6.46202027797699, "top": [ { "rank": 1, "tokenId": 82625, "text": "adult", "score": 5.489768028259277 }, { "rank": 2, "tokenId": 149267, "text": "youth", "score": -0.9722522497177124 }, { "rank": 3, "tokenId": 21128, "text": "young", "score": -1.0212618112564087 }, { "rank": 4, "tokenId": 171736, "text": "adoles", "score": -3.024744987487793 }, { "rank": 5, "tokenId": 12839, "text": "teen", "score": -3.382490396499634 }, { "rank": 6, "tokenId": 10869, "text": "child", "score": -4.005710124969482 }, { "rank": 7, "tokenId": 84136, "text": "Adult", "score": -4.074178695678711 }, { "rank": 8, "tokenId": 553, "text": "ad", "score": -5.510419845581055 } ] }, "f16_forward": { "winnerTokenId": 82625, "winnerText": "adult", "winnerScore": 5.5625, "winnerGap": 6.501953125, "top": [ { "rank": 1, "tokenId": 82625, "text": "adult", "score": 5.5625 }, { "rank": 2, "tokenId": 149267, "text": "youth", "score": -0.939453125 }, { "rank": 3, "tokenId": 21128, "text": "young", "score": -1.056640625 }, { "rank": 4, "tokenId": 171736, "text": "adoles", "score": -2.982421875 }, { "rank": 5, "tokenId": 10869, "text": "child", "score": -3.9609375 }, { "rank": 6, "tokenId": 84136, "text": "Adult", "score": -4.09375 }, { "rank": 7, "tokenId": 553, "text": "ad", "score": -5.53125 }, { "rank": 8, "tokenId": 12839, "text": "teen", "score": -7.34765625 } ] } }, "winnerTextsByMode": { "exact": "adult", "f32_forward": "adult", "f32_reverse": "adult", "f32_tree64": "adult", "f16_forward": "adult" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "adult" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "code-bash-shebang-oneword-b", "text": "Respond with one word only.\n#!/bin/bash\n", "stableTopK": [ { "rank": 1, "tokenId": 162588, "text": "Bash", "logit": 5.168059349060059, "prob": 0.729769766330719 }, { "rank": 2, "tokenId": 39610, "text": "Shell", "logit": 2.845128059387207, "prob": 0.07150717079639435 }, { "rank": 3, "tokenId": 5778, "text": "She", "logit": 2.3347978591918945, "prob": 0.04292556643486023 }, { "rank": 4, "tokenId": 8837, "text": "Script", "logit": 2.0864124298095703, "prob": 0.03348448500037193 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "logit": 2.0096824169158936, "prob": 0.031011315062642097 }, { "rank": 6, "tokenId": 9259, "text": "Hello", "logit": 1.5675239562988281, "prob": 0.019929353147745132 }, { "rank": 7, "tokenId": 39449, "text": "Execute", "logit": 0.7378574013710022, "prob": 0.00869307853281498 }, { "rank": 8, "tokenId": 3231, "text": "script", "logit": 0.37992483377456665, "prob": 0.006077507045120001 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 162588, "winnerText": "Bash", "winnerScore": 5.220112651640232, "winnerGap": 2.3664085143028903, "top": [ { "rank": 1, "tokenId": 162588, "text": "Bash", "score": 5.220112651640232 }, { "rank": 2, "tokenId": 39610, "text": "Shell", "score": 2.8537041373373415 }, { "rank": 3, "tokenId": 5778, "text": "She", "score": 2.339529558766557 }, { "rank": 4, "tokenId": 8837, "text": "Script", "score": 2.0897859523465883 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 2.012697859134948 }, { "rank": 6, "tokenId": 9259, "text": "Hello", "score": 1.5689544300890468 }, { "rank": 7, "tokenId": 39449, "text": "Execute", "score": 0.7380073938057734 }, { "rank": 8, "tokenId": 3231, "text": "script", "score": 0.3799436845887243 } ] }, "f32_forward": { "winnerTokenId": 162588, "winnerText": "Bash", "winnerScore": 5.220115661621094, "winnerGap": 2.3664114475250244, "top": [ { "rank": 1, "tokenId": 162588, "text": "Bash", "score": 5.220115661621094 }, { "rank": 2, "tokenId": 39610, "text": "Shell", "score": 2.8537042140960693 }, { "rank": 3, "tokenId": 5778, "text": "She", "score": 2.3395280838012695 }, { "rank": 4, "tokenId": 8837, "text": "Script", "score": 2.0897879600524902 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 2.012698173522949 }, { "rank": 6, "tokenId": 9259, "text": "Hello", "score": 1.5689562559127808 }, { "rank": 7, "tokenId": 39449, "text": "Execute", "score": 0.7380057573318481 }, { "rank": 8, "tokenId": 3231, "text": "script", "score": 0.3799438178539276 } ] }, "f32_reverse": { "winnerTokenId": 162588, "winnerText": "Bash", "winnerScore": 5.22011137008667, "winnerGap": 2.3664066791534424, "top": [ { "rank": 1, "tokenId": 162588, "text": "Bash", "score": 5.22011137008667 }, { "rank": 2, "tokenId": 39610, "text": "Shell", "score": 2.8537046909332275 }, { "rank": 3, "tokenId": 5778, "text": "She", "score": 2.3395333290100098 }, { "rank": 4, "tokenId": 8837, "text": "Script", "score": 2.0897865295410156 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 2.0126960277557373 }, { "rank": 6, "tokenId": 9259, "text": "Hello", "score": 1.568951964378357 }, { "rank": 7, "tokenId": 39449, "text": "Execute", "score": 0.7380064725875854 }, { "rank": 8, "tokenId": 3231, "text": "script", "score": 0.3799433708190918 } ] }, "f32_tree64": { "winnerTokenId": 162588, "winnerText": "Bash", "winnerScore": 5.220111846923828, "winnerGap": 2.3664069175720215, "top": [ { "rank": 1, "tokenId": 162588, "text": "Bash", "score": 5.220111846923828 }, { "rank": 2, "tokenId": 39610, "text": "Shell", "score": 2.8537049293518066 }, { "rank": 3, "tokenId": 5778, "text": "She", "score": 2.339529037475586 }, { "rank": 4, "tokenId": 8837, "text": "Script", "score": 2.0897860527038574 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 2.012697696685791 }, { "rank": 6, "tokenId": 9259, "text": "Hello", "score": 1.568954348564148 }, { "rank": 7, "tokenId": 39449, "text": "Execute", "score": 0.7380073070526123 }, { "rank": 8, "tokenId": 3231, "text": "script", "score": 0.37994384765625 } ] }, "f16_forward": { "winnerTokenId": 162588, "winnerText": "Bash", "winnerScore": 5.265625, "winnerGap": 2.421875, "top": [ { "rank": 1, "tokenId": 162588, "text": "Bash", "score": 5.265625 }, { "rank": 2, "tokenId": 39610, "text": "Shell", "score": 2.84375 }, { "rank": 3, "tokenId": 5778, "text": "She", "score": 2.33203125 }, { "rank": 4, "tokenId": 8837, "text": "Script", "score": 2.13671875 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 1.95703125 }, { "rank": 6, "tokenId": 39449, "text": "Execute", "score": 0.73388671875 }, { "rank": 7, "tokenId": 9259, "text": "Hello", "score": 0.603515625 }, { "rank": 8, "tokenId": 3231, "text": "script", "score": 0.382080078125 } ] } }, "winnerTextsByMode": { "exact": "Bash", "f32_forward": "Bash", "f32_reverse": "Bash", "f32_tree64": "Bash", "f16_forward": "Bash" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Bash" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "code-css-color-oneword-c", "text": "No explanation. One word only.\nbody { color:", "stableTopK": [ { "rank": 1, "tokenId": 8916, "text": "black", "logit": 5.674689769744873, "prob": 0.8537271618843079 }, { "rank": 2, "tokenId": 6907, "text": "Black", "logit": 3.014267921447754, "prob": 0.05969151109457016 }, { "rank": 3, "tokenId": 7986, "text": "white", "logit": 2.37977933883667, "prob": 0.03164883330464363 }, { "rank": 4, "tokenId": 1192, "text": "red", "logit": 2.255772590637207, "prob": 0.027957750484347343 }, { "rank": 5, "tokenId": 9503, "text": "blue", "logit": 1.2022204399108887, "prob": 0.009748782031238079 }, { "rank": 6, "tokenId": 9264, "text": "Red", "logit": 0.6516566276550293, "prob": 0.0056213876232504845 }, { "rank": 7, "tokenId": 1005, "text": "text", "logit": -0.4145549237728119, "prob": 0.001935502514243126 }, { "rank": 8, "tokenId": 3001, "text": "color", "logit": -0.5723404288291931, "prob": 0.0016529829008504748 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 8916, "winnerText": "black", "winnerScore": 5.743861518856058, "winnerGap": 2.719389269790014, "top": [ { "rank": 1, "tokenId": 8916, "text": "black", "score": 5.743861518856058 }, { "rank": 2, "tokenId": 6907, "text": "Black", "score": 3.0244722490660436 }, { "rank": 3, "tokenId": 7986, "text": "white", "score": 2.384783185562952 }, { "rank": 4, "tokenId": 1192, "text": "red", "score": 2.26003995481892 }, { "rank": 5, "tokenId": 9503, "text": "blue", "score": 1.2028628271403345 }, { "rank": 6, "tokenId": 9264, "text": "Red", "score": 0.6517579412064558 }, { "rank": 7, "tokenId": 1005, "text": "text", "score": -0.41458021451161997 }, { "rank": 8, "tokenId": 3001, "text": "color", "score": -0.5724109884865305 } ] }, "f32_forward": { "winnerTokenId": 8916, "winnerText": "black", "winnerScore": 5.743860721588135, "winnerGap": 2.719390630722046, "top": [ { "rank": 1, "tokenId": 8916, "text": "black", "score": 5.743860721588135 }, { "rank": 2, "tokenId": 6907, "text": "Black", "score": 3.024470090866089 }, { "rank": 3, "tokenId": 7986, "text": "white", "score": 2.384788990020752 }, { "rank": 4, "tokenId": 1192, "text": "red", "score": 2.260047674179077 }, { "rank": 5, "tokenId": 9503, "text": "blue", "score": 1.2028638124465942 }, { "rank": 6, "tokenId": 9264, "text": "Red", "score": 0.6517605781555176 }, { "rank": 7, "tokenId": 1005, "text": "text", "score": -0.41458234190940857 }, { "rank": 8, "tokenId": 3001, "text": "color", "score": -0.5724151730537415 } ] }, "f32_reverse": { "winnerTokenId": 8916, "winnerText": "black", "winnerScore": 5.743864059448242, "winnerGap": 2.7193922996520996, "top": [ { "rank": 1, "tokenId": 8916, "text": "black", "score": 5.743864059448242 }, { "rank": 2, "tokenId": 6907, "text": "Black", "score": 3.0244717597961426 }, { "rank": 3, "tokenId": 7986, "text": "white", "score": 2.384786605834961 }, { "rank": 4, "tokenId": 1192, "text": "red", "score": 2.2600369453430176 }, { "rank": 5, "tokenId": 9503, "text": "blue", "score": 1.2028634548187256 }, { "rank": 6, "tokenId": 9264, "text": "Red", "score": 0.6517613530158997 }, { "rank": 7, "tokenId": 1005, "text": "text", "score": -0.4145820438861847 }, { "rank": 8, "tokenId": 3001, "text": "color", "score": -0.572405219078064 } ] }, "f32_tree64": { "winnerTokenId": 8916, "winnerText": "black", "winnerScore": 5.743861198425293, "winnerGap": 2.719388484954834, "top": [ { "rank": 1, "tokenId": 8916, "text": "black", "score": 5.743861198425293 }, { "rank": 2, "tokenId": 6907, "text": "Black", "score": 3.024472713470459 }, { "rank": 3, "tokenId": 7986, "text": "white", "score": 2.3847827911376953 }, { "rank": 4, "tokenId": 1192, "text": "red", "score": 2.260040044784546 }, { "rank": 5, "tokenId": 9503, "text": "blue", "score": 1.2028634548187256 }, { "rank": 6, "tokenId": 9264, "text": "Red", "score": 0.6517574787139893 }, { "rank": 7, "tokenId": 1005, "text": "text", "score": -0.41458046436309814 }, { "rank": 8, "tokenId": 3001, "text": "color", "score": -0.572411298751831 } ] }, "f16_forward": { "winnerTokenId": 8916, "winnerText": "black", "winnerScore": 5.77734375, "winnerGap": 2.646484375, "top": [ { "rank": 1, "tokenId": 8916, "text": "black", "score": 5.77734375 }, { "rank": 2, "tokenId": 6907, "text": "Black", "score": 3.130859375 }, { "rank": 3, "tokenId": 7986, "text": "white", "score": 2.375 }, { "rank": 4, "tokenId": 1192, "text": "red", "score": 2.318359375 }, { "rank": 5, "tokenId": 9503, "text": "blue", "score": 1.23828125 }, { "rank": 6, "tokenId": 9264, "text": "Red", "score": 0.55224609375 }, { "rank": 7, "tokenId": 1005, "text": "text", "score": -0.437744140625 }, { "rank": 8, "tokenId": 7131, "text": "White", "score": -0.57958984375 } ] } }, "winnerTextsByMode": { "exact": "black", "f32_forward": "black", "f32_reverse": "black", "f32_tree64": "black", "f16_forward": "black" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "black" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "code-curl-cmd-oneword-d", "text": "Choose one word only.\ncurl -X POST https://", "stableTopK": [ { "rank": 1, "tokenId": 9185, "text": "POST", "logit": 14.064628601074219, "prob": 0.9991969466209412 }, { "rank": 2, "tokenId": 3274, "text": "post", "logit": 6.0270676612854, "prob": 0.00032283656764775515 }, { "rank": 3, "tokenId": 4764, "text": "request", "logit": 5.051812648773193, "prob": 0.00012174043513368815 }, { "rank": 4, "tokenId": 6738, "text": "URL", "logit": 4.728005886077881, "prob": 0.00008806581899989396 }, { "rank": 5, "tokenId": 40192, "text": "curl", "logit": 4.485840320587158, "prob": 0.0000691251625539735 }, { "rank": 6, "tokenId": 1018, "text": "**", "logit": 3.648252487182617, "prob": 0.00002991413202835247 }, { "rank": 7, "tokenId": 22005, "text": "HTTP", "logit": 3.139011859893799, "prob": 0.000017976950402953662 }, { "rank": 8, "tokenId": 3932, "text": "Request", "logit": 2.452643871307373, "prob": 0.000009049616892298218 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 9185, "winnerText": "POST", "winnerScore": 15.256741189843442, "winnerGap": 9.146564210258509, "top": [ { "rank": 1, "tokenId": 9185, "text": "POST", "score": 15.256741189843442 }, { "rank": 2, "tokenId": 3274, "text": "post", "score": 6.110176979584933 }, { "rank": 3, "tokenId": 4764, "text": "request", "score": 5.100395229262638 }, { "rank": 4, "tokenId": 6738, "text": "URL", "score": 4.7677440811373355 }, { "rank": 5, "tokenId": 40192, "text": "curl", "score": 4.5197272702994695 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 3.666395973419263 }, { "rank": 7, "tokenId": 22005, "text": "HTTP", "score": 3.1505419505166685 }, { "rank": 8, "tokenId": 3932, "text": "Request", "score": 2.458129835380312 } ] }, "f32_forward": { "winnerTokenId": 9185, "winnerText": "POST", "winnerScore": 15.256747245788574, "winnerGap": 9.146571159362793, "top": [ { "rank": 1, "tokenId": 9185, "text": "POST", "score": 15.256747245788574 }, { "rank": 2, "tokenId": 3274, "text": "post", "score": 6.110176086425781 }, { "rank": 3, "tokenId": 4764, "text": "request", "score": 5.100402355194092 }, { "rank": 4, "tokenId": 6738, "text": "URL", "score": 4.76774787902832 }, { "rank": 5, "tokenId": 40192, "text": "curl", "score": 4.519725322723389 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 3.6663975715637207 }, { "rank": 7, "tokenId": 22005, "text": "HTTP", "score": 3.150538206100464 }, { "rank": 8, "tokenId": 3932, "text": "Request", "score": 2.458127975463867 } ] }, "f32_reverse": { "winnerTokenId": 9185, "winnerText": "POST", "winnerScore": 15.256739616394043, "winnerGap": 9.146567821502686, "top": [ { "rank": 1, "tokenId": 9185, "text": "POST", "score": 15.256739616394043 }, { "rank": 2, "tokenId": 3274, "text": "post", "score": 6.110171794891357 }, { "rank": 3, "tokenId": 4764, "text": "request", "score": 5.100397109985352 }, { "rank": 4, "tokenId": 6738, "text": "URL", "score": 4.767740726470947 }, { "rank": 5, "tokenId": 40192, "text": "curl", "score": 4.519730091094971 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 3.6663999557495117 }, { "rank": 7, "tokenId": 22005, "text": "HTTP", "score": 3.1505391597747803 }, { "rank": 8, "tokenId": 3932, "text": "Request", "score": 2.458130121231079 } ] }, "f32_tree64": { "winnerTokenId": 9185, "winnerText": "POST", "winnerScore": 15.25674057006836, "winnerGap": 9.146563529968262, "top": [ { "rank": 1, "tokenId": 9185, "text": "POST", "score": 15.25674057006836 }, { "rank": 2, "tokenId": 3274, "text": "post", "score": 6.110177040100098 }, { "rank": 3, "tokenId": 4764, "text": "request", "score": 5.100395679473877 }, { "rank": 4, "tokenId": 6738, "text": "URL", "score": 4.767745018005371 }, { "rank": 5, "tokenId": 40192, "text": "curl", "score": 4.519726753234863 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 3.666395664215088 }, { "rank": 7, "tokenId": 22005, "text": "HTTP", "score": 3.1505420207977295 }, { "rank": 8, "tokenId": 3932, "text": "Request", "score": 2.4581298828125 } ] }, "f16_forward": { "winnerTokenId": 9185, "winnerText": "POST", "winnerScore": 15.2734375, "winnerGap": 9.18359375, "top": [ { "rank": 1, "tokenId": 9185, "text": "POST", "score": 15.2734375 }, { "rank": 2, "tokenId": 3274, "text": "post", "score": 6.08984375 }, { "rank": 3, "tokenId": 4764, "text": "request", "score": 5.07421875 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 3.66015625 }, { "rank": 5, "tokenId": 22005, "text": "HTTP", "score": 3.275390625 }, { "rank": 6, "tokenId": 3932, "text": "Request", "score": 2.48046875 }, { "rank": 7, "tokenId": 818, "text": "The", "score": 1.9658203125 }, { "rank": 8, "tokenId": 65941, "text": "HTTPS", "score": 1.9580078125 } ] } }, "winnerTextsByMode": { "exact": "POST", "f32_forward": "POST", "f32_reverse": "POST", "f32_tree64": "POST", "f16_forward": "POST" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "POST" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "code-fn-oneword-a", "text": "Answer with exactly one word only.\nfunction add(", "stableTopK": [ { "rank": 1, "tokenId": 33672, "text": "undefined", "logit": 9.205925941467285, "prob": 0.8044414520263672 }, { "rank": 2, "tokenId": 115587, "text": "Undefined", "logit": 6.47664213180542, "prob": 0.05250268429517746 }, { "rank": 3, "tokenId": 2501, "text": "function", "logit": 5.744778156280518, "prob": 0.025254398584365845 }, { "rank": 4, "tokenId": 41387, "text": "Unknown", "logit": 4.864748001098633, "prob": 0.010474777780473232 }, { "rank": 5, "tokenId": 13293, "text": "Return", "logit": 4.848909378051758, "prob": 0.010310178622603416 }, { "rank": 6, "tokenId": 3218, "text": "Add", "logit": 4.569065093994141, "prob": 0.007793479133397341 }, { "rank": 7, "tokenId": 11813, "text": "Function", "logit": 4.504847526550293, "prob": 0.007308731321245432 }, { "rank": 8, "tokenId": 3494, "text": "Error", "logit": 4.50196647644043, "prob": 0.0072877053171396255 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 33672, "winnerText": "undefined", "winnerScore": 9.512398701281292, "winnerGap": 2.932223042350211, "top": [ { "rank": 1, "tokenId": 33672, "text": "undefined", "score": 9.512398701281292 }, { "rank": 2, "tokenId": 115587, "text": "Undefined", "score": 6.580175658931081 }, { "rank": 3, "tokenId": 2501, "text": "function", "score": 5.816585640968909 }, { "rank": 4, "tokenId": 41387, "text": "Unknown", "score": 4.90807478176689 }, { "rank": 5, "tokenId": 13293, "text": "Return", "score": 4.891808392512701 }, { "rank": 6, "tokenId": 3218, "text": "Add", "score": 4.604890019716972 }, { "rank": 7, "tokenId": 11813, "text": "Function", "score": 4.539172815735874 }, { "rank": 8, "tokenId": 3494, "text": "Error", "score": 4.53622588987858 } ] }, "f32_forward": { "winnerTokenId": 33672, "winnerText": "undefined", "winnerScore": 9.512402534484863, "winnerGap": 2.932220935821533, "top": [ { "rank": 1, "tokenId": 33672, "text": "undefined", "score": 9.512402534484863 }, { "rank": 2, "tokenId": 115587, "text": "Undefined", "score": 6.58018159866333 }, { "rank": 3, "tokenId": 2501, "text": "function", "score": 5.816588878631592 }, { "rank": 4, "tokenId": 41387, "text": "Unknown", "score": 4.908075332641602 }, { "rank": 5, "tokenId": 13293, "text": "Return", "score": 4.8918046951293945 }, { "rank": 6, "tokenId": 3218, "text": "Add", "score": 4.604885578155518 }, { "rank": 7, "tokenId": 11813, "text": "Function", "score": 4.53916597366333 }, { "rank": 8, "tokenId": 3494, "text": "Error", "score": 4.5362162590026855 } ] }, "f32_reverse": { "winnerTokenId": 33672, "winnerText": "undefined", "winnerScore": 9.512396812438965, "winnerGap": 2.932220935821533, "top": [ { "rank": 1, "tokenId": 33672, "text": "undefined", "score": 9.512396812438965 }, { "rank": 2, "tokenId": 115587, "text": "Undefined", "score": 6.580175876617432 }, { "rank": 3, "tokenId": 2501, "text": "function", "score": 5.81658411026001 }, { "rank": 4, "tokenId": 41387, "text": "Unknown", "score": 4.908071517944336 }, { "rank": 5, "tokenId": 13293, "text": "Return", "score": 4.891808986663818 }, { "rank": 6, "tokenId": 3218, "text": "Add", "score": 4.6048903465271 }, { "rank": 7, "tokenId": 11813, "text": "Function", "score": 4.539172649383545 }, { "rank": 8, "tokenId": 3494, "text": "Error", "score": 4.536223888397217 } ] }, "f32_tree64": { "winnerTokenId": 33672, "winnerText": "undefined", "winnerScore": 9.512399673461914, "winnerGap": 2.932223320007324, "top": [ { "rank": 1, "tokenId": 33672, "text": "undefined", "score": 9.512399673461914 }, { "rank": 2, "tokenId": 115587, "text": "Undefined", "score": 6.58017635345459 }, { "rank": 3, "tokenId": 2501, "text": "function", "score": 5.816585540771484 }, { "rank": 4, "tokenId": 41387, "text": "Unknown", "score": 4.908074378967285 }, { "rank": 5, "tokenId": 13293, "text": "Return", "score": 4.891808032989502 }, { "rank": 6, "tokenId": 3218, "text": "Add", "score": 4.6048903465271 }, { "rank": 7, "tokenId": 11813, "text": "Function", "score": 4.539172172546387 }, { "rank": 8, "tokenId": 3494, "text": "Error", "score": 4.536226272583008 } ] }, "f16_forward": { "winnerTokenId": 33672, "winnerText": "undefined", "winnerScore": 9.484375, "winnerGap": 2.8828125, "top": [ { "rank": 1, "tokenId": 33672, "text": "undefined", "score": 9.484375 }, { "rank": 2, "tokenId": 115587, "text": "Undefined", "score": 6.6015625 }, { "rank": 3, "tokenId": 2501, "text": "function", "score": 5.82421875 }, { "rank": 4, "tokenId": 13293, "text": "Return", "score": 4.8984375 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 4.69921875 }, { "rank": 6, "tokenId": 3494, "text": "Error", "score": 4.58984375 }, { "rank": 7, "tokenId": 3218, "text": "Add", "score": 4.5703125 }, { "rank": 8, "tokenId": 11813, "text": "Function", "score": 4.53515625 } ] } }, "winnerTextsByMode": { "exact": "undefined", "f32_forward": "undefined", "f32_reverse": "undefined", "f32_tree64": "undefined", "f16_forward": "undefined" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "undefined" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "code-html-tag-oneword-b", "text": "Respond with one word only.\n
", "stableTopK": [ { "rank": 1, "tokenId": 6302, "text": "Start", "logit": -1.821894645690918, "prob": 0.4005451202392578 }, { "rank": 2, "tokenId": 20989, "text": "XML", "logit": -2.575855255126953, "prob": 0.1884562373161316 }, { "rank": 3, "tokenId": 3494, "text": "Error", "logit": -3.628563404083252, "prob": 0.06576959788799286 }, { "rank": 4, "tokenId": 9336, "text": "None", "logit": -3.780972957611084, "prob": 0.05647217854857445 }, { "rank": 5, "tokenId": 3771, "text": "No", "logit": -4.137274265289307, "prob": 0.039545297622680664 }, { "rank": 6, "tokenId": 7676, "text": "OK", "logit": -4.144260406494141, "prob": 0.03926999494433403 }, { "rank": 7, "tokenId": 9647, "text": "Null", "logit": -4.385059356689453, "prob": 0.03086620196700096 }, { "rank": 8, "tokenId": 11447, "text": "Empty", "logit": -4.921720027923584, "prob": 0.0180473905056715 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 6302, "winnerText": "Start", "winnerScore": -1.8241408841874858, "winnerGap": 0.7580708458588994, "top": [ { "rank": 1, "tokenId": 6302, "text": "Start", "score": -1.8241408841874858 }, { "rank": 2, "tokenId": 20989, "text": "XML", "score": -2.582211730046385 }, { "rank": 3, "tokenId": 3494, "text": "Error", "score": -3.646414020025812 }, { "rank": 4, "tokenId": 9336, "text": "None", "score": -3.8011860480825277 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": -4.163805160464179 }, { "rank": 6, "tokenId": 7676, "text": "OK", "score": -4.17092862498104 }, { "rank": 7, "tokenId": 9647, "text": "Null", "score": -4.416694224554756 }, { "rank": 8, "tokenId": 11447, "text": "Empty", "score": -4.966603069838529 } ] }, "f32_forward": { "winnerTokenId": 6302, "winnerText": "Start", "winnerScore": -1.8241407871246338, "winnerGap": 0.7580723762512207, "top": [ { "rank": 1, "tokenId": 6302, "text": "Start", "score": -1.8241407871246338 }, { "rank": 2, "tokenId": 20989, "text": "XML", "score": -2.5822131633758545 }, { "rank": 3, "tokenId": 3494, "text": "Error", "score": -3.6464133262634277 }, { "rank": 4, "tokenId": 9336, "text": "None", "score": -3.80118465423584 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": -4.163808345794678 }, { "rank": 6, "tokenId": 7676, "text": "OK", "score": -4.170929431915283 }, { "rank": 7, "tokenId": 9647, "text": "Null", "score": -4.416697978973389 }, { "rank": 8, "tokenId": 11447, "text": "Empty", "score": -4.966602802276611 } ] }, "f32_reverse": { "winnerTokenId": 6302, "winnerText": "Start", "winnerScore": -1.8241424560546875, "winnerGap": 0.7580676078796387, "top": [ { "rank": 1, "tokenId": 6302, "text": "Start", "score": -1.8241424560546875 }, { "rank": 2, "tokenId": 20989, "text": "XML", "score": -2.582210063934326 }, { "rank": 3, "tokenId": 3494, "text": "Error", "score": -3.646409034729004 }, { "rank": 4, "tokenId": 9336, "text": "None", "score": -3.801182985305786 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": -4.163812637329102 }, { "rank": 6, "tokenId": 7676, "text": "OK", "score": -4.170934200286865 }, { "rank": 7, "tokenId": 9647, "text": "Null", "score": -4.41668701171875 }, { "rank": 8, "tokenId": 11447, "text": "Empty", "score": -4.9666032791137695 } ] }, "f32_tree64": { "winnerTokenId": 6302, "winnerText": "Start", "winnerScore": -1.8241406679153442, "winnerGap": 0.7580713033676147, "top": [ { "rank": 1, "tokenId": 6302, "text": "Start", "score": -1.8241406679153442 }, { "rank": 2, "tokenId": 20989, "text": "XML", "score": -2.582211971282959 }, { "rank": 3, "tokenId": 3494, "text": "Error", "score": -3.646414279937744 }, { "rank": 4, "tokenId": 9336, "text": "None", "score": -3.8011858463287354 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": -4.16380500793457 }, { "rank": 6, "tokenId": 7676, "text": "OK", "score": -4.170928955078125 }, { "rank": 7, "tokenId": 9647, "text": "Null", "score": -4.416693687438965 }, { "rank": 8, "tokenId": 11447, "text": "Empty", "score": -4.966602802276611 } ] }, "f16_forward": { "winnerTokenId": 3494, "winnerText": "Error", "winnerScore": -1.642578125, "winnerGap": 0.1455078125, "top": [ { "rank": 1, "tokenId": 3494, "text": "Error", "score": -1.642578125 }, { "rank": 2, "tokenId": 6302, "text": "Start", "score": -1.7880859375 }, { "rank": 3, "tokenId": 20989, "text": "XML", "score": -2.626953125 }, { "rank": 4, "tokenId": 3771, "text": "No", "score": -3.12890625 }, { "rank": 5, "tokenId": 7676, "text": "OK", "score": -3.1953125 }, { "rank": 6, "tokenId": 27426, "text": "Begin", "score": -3.625 }, { "rank": 7, "tokenId": 9336, "text": "None", "score": -3.85546875 }, { "rank": 8, "tokenId": 8185, "text": "HTML", "score": -4.05859375 } ] } }, "winnerTextsByMode": { "exact": "Start", "f32_forward": "Start", "f32_reverse": "Start", "f32_tree64": "Start", "f16_forward": "Error" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Start", "Error" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 6302, 106 ], "decodedText": "Start", "steps": [ { "step": 0, "tokenId": 6302, "text": "Start" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 3494, 106 ], "decodedText": "Error", "steps": [ { "step": 0, "tokenId": 3494, "text": "Error" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "code-yaml-key-oneword-b", "text": "Respond with one word only.\napiVersion: v1\nkind:", "stableTopK": [ { "rank": 1, "tokenId": 9647, "text": "Null", "logit": -5.402596473693848, "prob": 0.31036946177482605 }, { "rank": 2, "tokenId": 25998, "text": "Pod", "logit": -5.422055721282959, "prob": 0.30438828468322754 }, { "rank": 3, "tokenId": 11834, "text": "Core", "logit": -5.7142415046691895, "prob": 0.2272653579711914 }, { "rank": 4, "tokenId": 3372, "text": "Object", "logit": -6.757461071014404, "prob": 0.08006980270147324 }, { "rank": 5, "tokenId": 6455, "text": "null", "logit": -8.264657974243164, "prob": 0.017737869173288345 }, { "rank": 6, "tokenId": 33778, "text": "Ready", "logit": -8.290016174316406, "prob": 0.017293723300099373 }, { "rank": 7, "tokenId": 6102, "text": "core", "logit": -8.42409896850586, "prob": 0.015123666264116764 }, { "rank": 8, "tokenId": 115587, "text": "Undefined", "logit": -9.918153762817383, "prob": 0.0033946684561669827 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 9647, "winnerText": "Null", "winnerScore": -5.462166126529091, "winnerGap": 0.020114566437776205, "top": [ { "rank": 1, "tokenId": 9647, "text": "Null", "score": -5.462166126529091 }, { "rank": 2, "tokenId": 25998, "text": "Pod", "score": -5.482280692966867 }, { "rank": 3, "tokenId": 11834, "text": "Core", "score": -5.78489387777806 }, { "rank": 4, "tokenId": 3372, "text": "Object", "score": -6.875357385817324 }, { "rank": 5, "tokenId": 6455, "text": "null", "score": -8.4838049886758 }, { "rank": 6, "tokenId": 33778, "text": "Ready", "score": -8.51125539837301 }, { "rank": 7, "tokenId": 6102, "text": "core", "score": -8.656616680900505 }, { "rank": 8, "tokenId": 115587, "text": "Undefined", "score": -10.305224373759156 } ] }, "f32_forward": { "winnerTokenId": 9647, "winnerText": "Null", "winnerScore": -5.462166786193848, "winnerGap": 0.020105838775634766, "top": [ { "rank": 1, "tokenId": 9647, "text": "Null", "score": -5.462166786193848 }, { "rank": 2, "tokenId": 25998, "text": "Pod", "score": -5.482272624969482 }, { "rank": 3, "tokenId": 11834, "text": "Core", "score": -5.784895896911621 }, { "rank": 4, "tokenId": 3372, "text": "Object", "score": -6.875360488891602 }, { "rank": 5, "tokenId": 6455, "text": "null", "score": -8.483818054199219 }, { "rank": 6, "tokenId": 33778, "text": "Ready", "score": -8.51125431060791 }, { "rank": 7, "tokenId": 6102, "text": "core", "score": -8.656623840332031 }, { "rank": 8, "tokenId": 115587, "text": "Undefined", "score": -10.305227279663086 } ] }, "f32_reverse": { "winnerTokenId": 9647, "winnerText": "Null", "winnerScore": -5.462179660797119, "winnerGap": 0.02009868621826172, "top": [ { "rank": 1, "tokenId": 9647, "text": "Null", "score": -5.462179660797119 }, { "rank": 2, "tokenId": 25998, "text": "Pod", "score": -5.482278347015381 }, { "rank": 3, "tokenId": 11834, "text": "Core", "score": -5.784888744354248 }, { "rank": 4, "tokenId": 3372, "text": "Object", "score": -6.875364303588867 }, { "rank": 5, "tokenId": 6455, "text": "null", "score": -8.483805656433105 }, { "rank": 6, "tokenId": 33778, "text": "Ready", "score": -8.51125431060791 }, { "rank": 7, "tokenId": 6102, "text": "core", "score": -8.656611442565918 }, { "rank": 8, "tokenId": 115587, "text": "Undefined", "score": -10.305227279663086 } ] }, "f32_tree64": { "winnerTokenId": 9647, "winnerText": "Null", "winnerScore": -5.462165832519531, "winnerGap": 0.020114421844482422, "top": [ { "rank": 1, "tokenId": 9647, "text": "Null", "score": -5.462165832519531 }, { "rank": 2, "tokenId": 25998, "text": "Pod", "score": -5.482280254364014 }, { "rank": 3, "tokenId": 11834, "text": "Core", "score": -5.784893035888672 }, { "rank": 4, "tokenId": 3372, "text": "Object", "score": -6.875357151031494 }, { "rank": 5, "tokenId": 6455, "text": "null", "score": -8.483804702758789 }, { "rank": 6, "tokenId": 33778, "text": "Ready", "score": -8.51125717163086 }, { "rank": 7, "tokenId": 6102, "text": "core", "score": -8.6566162109375 }, { "rank": 8, "tokenId": 115587, "text": "Undefined", "score": -10.305224418640137 } ] }, "f16_forward": { "winnerTokenId": 6455, "winnerText": "null", "winnerScore": -4.484375, "winnerGap": 0.25390625, "top": [ { "rank": 1, "tokenId": 6455, "text": "null", "score": -4.484375 }, { "rank": 2, "tokenId": 6102, "text": "core", "score": -4.73828125 }, { "rank": 3, "tokenId": 28282, "text": "pod", "score": -5.01171875 }, { "rank": 4, "tokenId": 25998, "text": "Pod", "score": -5.45703125 }, { "rank": 5, "tokenId": 9647, "text": "Null", "score": -5.47265625 }, { "rank": 6, "tokenId": 123626, "text": "Deployment", "score": -6.203125 }, { "rank": 7, "tokenId": 12471, "text": "Bundle", "score": -6.52734375 }, { "rank": 8, "tokenId": 3372, "text": "Object", "score": -6.84765625 } ] } }, "winnerTextsByMode": { "exact": "Null", "f32_forward": "Null", "f32_reverse": "Null", "f32_tree64": "Null", "f16_forward": "null" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Null", "null" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 9647, 106 ], "decodedText": "Null", "steps": [ { "step": 0, "tokenId": 9647, "text": "Null" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 6455, 106 ], "decodedText": "null", "steps": [ { "step": 0, "tokenId": 6455, "text": "null" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "coffee-tea-morning-oneword-c", "text": "No explanation. One word only.\nAnswer with exactly one word: coffee or tea.\nQuestion: The prompt does not force a correct answer. Pick one.\nAnswer:", "stableTopK": [ { "rank": 1, "tokenId": 46116, "text": "coffee", "logit": 19.900047302246094, "prob": 0.9769968390464783 }, { "rank": 2, "tokenId": 61767, "text": "Coffee", "logit": 16.150985717773438, "prob": 0.02299833670258522 }, { "rank": 3, "tokenId": 7681, "text": " coffee", "logit": 7.297626972198486, "prob": 0.0000032864843433344504 }, { "rank": 4, "tokenId": 51954, "text": "tea", "logit": 6.044676780700684, "prob": 9.388197099724493e-7 }, { "rank": 5, "tokenId": 116162, "text": "コーヒー", "logit": 5.343316078186035, "prob": 4.6557013888559595e-7 }, { "rank": 6, "tokenId": 97396, "text": "Tea", "logit": 3.0117604732513428, "prob": 4.52275834561533e-8 }, { "rank": 7, "tokenId": 76113, "text": "咖啡", "logit": 2.719778537750244, "prob": 3.377514445901397e-8 }, { "rank": 8, "tokenId": 124487, "text": " कॉफी", "logit": 1.4779162406921387, "prob": 9.755809493583456e-9 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 46116, "winnerText": "coffee", "winnerScore": 23.962358633882314, "winnerGap": 5.906810292858868, "top": [ { "rank": 1, "tokenId": 46116, "text": "coffee", "score": 23.962358633882314 }, { "rank": 2, "tokenId": 61767, "text": "Coffee", "score": 18.055548341023446 }, { "rank": 3, "tokenId": 7681, "text": " coffee", "score": 7.446894961576143 }, { "rank": 4, "tokenId": 51954, "text": "tea", "score": 6.128530100020527 }, { "rank": 5, "tokenId": 116162, "text": "コーヒー", "score": 5.400914860511708 }, { "rank": 6, "tokenId": 97396, "text": "Tea", "score": 3.0219459126635755 }, { "rank": 7, "tokenId": 76113, "text": "咖啡", "score": 2.7272622222971776 }, { "rank": 8, "tokenId": 124487, "text": " कॉफी", "score": 1.479119274462633 } ] }, "f32_forward": { "winnerTokenId": 46116, "winnerText": "coffee", "winnerScore": 23.96234703063965, "winnerGap": 5.906801223754883, "top": [ { "rank": 1, "tokenId": 46116, "text": "coffee", "score": 23.96234703063965 }, { "rank": 2, "tokenId": 61767, "text": "Coffee", "score": 18.055545806884766 }, { "rank": 3, "tokenId": 7681, "text": " coffee", "score": 7.446902275085449 }, { "rank": 4, "tokenId": 51954, "text": "tea", "score": 6.128529071807861 }, { "rank": 5, "tokenId": 116162, "text": "コーヒー", "score": 5.400920867919922 }, { "rank": 6, "tokenId": 97396, "text": "Tea", "score": 3.0219528675079346 }, { "rank": 7, "tokenId": 76113, "text": "咖啡", "score": 2.7272636890411377 }, { "rank": 8, "tokenId": 124487, "text": " कॉफी", "score": 1.4791220426559448 } ] }, "f32_reverse": { "winnerTokenId": 46116, "winnerText": "coffee", "winnerScore": 23.96234703063965, "winnerGap": 5.906806945800781, "top": [ { "rank": 1, "tokenId": 46116, "text": "coffee", "score": 23.96234703063965 }, { "rank": 2, "tokenId": 61767, "text": "Coffee", "score": 18.055540084838867 }, { "rank": 3, "tokenId": 7681, "text": " coffee", "score": 7.446895122528076 }, { "rank": 4, "tokenId": 51954, "text": "tea", "score": 6.1285271644592285 }, { "rank": 5, "tokenId": 116162, "text": "コーヒー", "score": 5.40091609954834 }, { "rank": 6, "tokenId": 97396, "text": "Tea", "score": 3.021944761276245 }, { "rank": 7, "tokenId": 76113, "text": "咖啡", "score": 2.7272608280181885 }, { "rank": 8, "tokenId": 124487, "text": " कॉफी", "score": 1.4791200160980225 } ] }, "f32_tree64": { "winnerTokenId": 46116, "winnerText": "coffee", "winnerScore": 23.962360382080078, "winnerGap": 5.90681266784668, "top": [ { "rank": 1, "tokenId": 46116, "text": "coffee", "score": 23.962360382080078 }, { "rank": 2, "tokenId": 61767, "text": "Coffee", "score": 18.0555477142334 }, { "rank": 3, "tokenId": 7681, "text": " coffee", "score": 7.446897506713867 }, { "rank": 4, "tokenId": 51954, "text": "tea", "score": 6.128528594970703 }, { "rank": 5, "tokenId": 116162, "text": "コーヒー", "score": 5.400915622711182 }, { "rank": 6, "tokenId": 97396, "text": "Tea", "score": 3.021946430206299 }, { "rank": 7, "tokenId": 76113, "text": "咖啡", "score": 2.727262496948242 }, { "rank": 8, "tokenId": 124487, "text": " कॉफी", "score": 1.4791200160980225 } ] }, "f16_forward": { "winnerTokenId": 46116, "winnerText": "coffee", "winnerScore": 23.984375, "winnerGap": 5.875, "top": [ { "rank": 1, "tokenId": 46116, "text": "coffee", "score": 23.984375 }, { "rank": 2, "tokenId": 61767, "text": "Coffee", "score": 18.109375 }, { "rank": 3, "tokenId": 7681, "text": " coffee", "score": 7.4921875 }, { "rank": 4, "tokenId": 116162, "text": "コーヒー", "score": 5.39453125 }, { "rank": 5, "tokenId": 51954, "text": "tea", "score": 5.15625 }, { "rank": 6, "tokenId": 97396, "text": "Tea", "score": 3.232421875 }, { "rank": 7, "tokenId": 76113, "text": "咖啡", "score": 2.78125 }, { "rank": 8, "tokenId": 124487, "text": " कॉफी", "score": 1.49609375 } ] } }, "winnerTextsByMode": { "exact": "coffee", "f32_forward": "coffee", "f32_reverse": "coffee", "f32_tree64": "coffee", "f16_forward": "coffee" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "coffee" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "comma-open-oneword-d", "text": "Choose one word only.\nBlue, green,", "stableTopK": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "logit": 13.856877326965332, "prob": 0.9915287494659424 }, { "rank": 2, "tokenId": 9503, "text": "blue", "logit": 9.071667671203613, "prob": 0.008281617425382137 }, { "rank": 3, "tokenId": 1018, "text": "**", "logit": 5.0332183837890625, "prob": 0.0001459617051295936 }, { "rank": 4, "tokenId": 17224, "text": "Green", "logit": 2.6469156742095947, "prob": 0.000013423965356196277 }, { "rank": 5, "tokenId": 24852, "text": "Choose", "logit": 1.5409046411514282, "prob": 0.000004441670625965344 }, { "rank": 6, "tokenId": 106, "text": "", "logit": 0.3181014060974121, "prob": 0.0000013076443110549008 }, { "rank": 7, "tokenId": 818, "text": "The", "logit": 0.12941208481788635, "prob": 0.0000010827866390172858 }, { "rank": 8, "tokenId": 35724, "text": "choose", "logit": -0.3161260783672333, "prob": 6.93502670401358e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 14.991562007140917, "winnerGap": 5.627154373041439, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 14.991562007140917 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": 9.364407634099479 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 5.081256639617391 }, { "rank": 4, "tokenId": 17224, "text": "Green", "score": 2.6538162833916026 }, { "rank": 5, "tokenId": 24852, "text": "Choose", "score": 1.5422578237250484 }, { "rank": 6, "tokenId": 106, "text": "", "score": 0.31811104585082006 }, { "rank": 7, "tokenId": 818, "text": "The", "score": 0.1294122050822626 }, { "rank": 8, "tokenId": 35724, "text": "choose", "score": -0.316138352277342 } ] }, "f32_forward": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 14.991569519042969, "winnerGap": 5.627159118652344, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 14.991569519042969 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": 9.364410400390625 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 5.081254482269287 }, { "rank": 4, "tokenId": 17224, "text": "Green", "score": 2.653827667236328 }, { "rank": 5, "tokenId": 24852, "text": "Choose", "score": 1.5422595739364624 }, { "rank": 6, "tokenId": 106, "text": "", "score": 0.3181123733520508 }, { "rank": 7, "tokenId": 818, "text": "The", "score": 0.12941373884677887 }, { "rank": 8, "tokenId": 35724, "text": "choose", "score": -0.31613874435424805 } ] }, "f32_reverse": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 14.991558074951172, "winnerGap": 5.627154350280762, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 14.991558074951172 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": 9.36440372467041 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 5.08125638961792 }, { "rank": 4, "tokenId": 17224, "text": "Green", "score": 2.6538195610046387 }, { "rank": 5, "tokenId": 24852, "text": "Choose", "score": 1.542263150215149 }, { "rank": 6, "tokenId": 106, "text": "", "score": 0.31811097264289856 }, { "rank": 7, "tokenId": 818, "text": "The", "score": 0.12941230833530426 }, { "rank": 8, "tokenId": 35724, "text": "choose", "score": -0.31613853573799133 } ] }, "f32_tree64": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 14.991561889648438, "winnerGap": 5.627153396606445, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 14.991561889648438 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": 9.364408493041992 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 5.081256866455078 }, { "rank": 4, "tokenId": 17224, "text": "Green", "score": 2.6538162231445312 }, { "rank": 5, "tokenId": 24852, "text": "Choose", "score": 1.5422568321228027 }, { "rank": 6, "tokenId": 106, "text": "", "score": 0.31811094284057617 }, { "rank": 7, "tokenId": 818, "text": "The", "score": 0.1294114589691162 }, { "rank": 8, "tokenId": 35724, "text": "choose", "score": -0.31613802909851074 } ] }, "f16_forward": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 15.0859375, "winnerGap": 9.66015625, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 15.0859375 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": 5.42578125 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 5.078125 }, { "rank": 4, "tokenId": 17224, "text": "Green", "score": 2.666015625 }, { "rank": 5, "tokenId": 24852, "text": "Choose", "score": 1.591796875 }, { "rank": 6, "tokenId": 106, "text": "", "score": 0.33642578125 }, { "rank": 7, "tokenId": 35724, "text": "choose", "score": -0.307373046875 }, { "rank": 8, "tokenId": 10671, "text": "green", "score": -0.385986328125 } ] } }, "winnerTextsByMode": { "exact": "Blue", "f32_forward": "Blue", "f32_reverse": "Blue", "f32_tree64": "Blue", "f16_forward": "Blue" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Blue" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "discovery-invention-both-oneword-a", "text": "Answer with exactly one word only.\nDiscovery, invention, or both: mathematics is", "stableTopK": [ { "rank": 1, "tokenId": 27413, "text": "both", "logit": 19.358654022216797, "prob": 0.9662691354751587 }, { "rank": 2, "tokenId": 22186, "text": "Both", "logit": 15.911933898925781, "prob": 0.030775602906942368 }, { "rank": 3, "tokenId": 75503, "text": "discovery", "logit": 12.54415225982666, "prob": 0.0010607122676447034 }, { "rank": 4, "tokenId": 13066, "text": "abstract", "logit": 11.996438980102539, "prob": 0.0006133787683211267 }, { "rank": 5, "tokenId": 128681, "text": "invention", "logit": 11.252704620361328, "prob": 0.0002915607183240354 }, { "rank": 6, "tokenId": 1800, "text": " both", "logit": 11.150198936462402, "prob": 0.00026315482682548463 }, { "rank": 7, "tokenId": 13395, "text": "found", "logit": 10.255877494812012, "prob": 0.00010760005534393713 }, { "rank": 8, "tokenId": 165997, "text": "purs", "logit": 9.94627857208252, "prob": 0.00007895059388829395 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 27413, "winnerText": "both", "winnerScore": 23.015496345059994, "winnerGap": 5.294541191698144, "top": [ { "rank": 1, "tokenId": 27413, "text": "both", "score": 23.015496345059994 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 17.72095515336185 }, { "rank": 3, "tokenId": 75503, "text": "discovery", "score": 13.363018319221528 }, { "rank": 4, "tokenId": 13066, "text": "abstract", "score": 12.705230352086513 }, { "rank": 5, "tokenId": 128681, "text": "invention", "score": 11.830001588923858 }, { "rank": 6, "tokenId": 1800, "text": " both", "score": 11.710894085324497 }, { "rank": 7, "tokenId": 13395, "text": "found", "score": 10.686014392751702 }, { "rank": 8, "tokenId": 165997, "text": "purs", "score": 10.33681202417658 } ] }, "f32_forward": { "winnerTokenId": 27413, "winnerText": "both", "winnerScore": 23.015499114990234, "winnerGap": 5.294561386108398, "top": [ { "rank": 1, "tokenId": 27413, "text": "both", "score": 23.015499114990234 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 17.720937728881836 }, { "rank": 3, "tokenId": 75503, "text": "discovery", "score": 13.363003730773926 }, { "rank": 4, "tokenId": 13066, "text": "abstract", "score": 12.705242156982422 }, { "rank": 5, "tokenId": 128681, "text": "invention", "score": 11.830011367797852 }, { "rank": 6, "tokenId": 1800, "text": " both", "score": 11.710892677307129 }, { "rank": 7, "tokenId": 13395, "text": "found", "score": 10.686017990112305 }, { "rank": 8, "tokenId": 165997, "text": "purs", "score": 10.336812973022461 } ] }, "f32_reverse": { "winnerTokenId": 27413, "winnerText": "both", "winnerScore": 23.015499114990234, "winnerGap": 5.294538497924805, "top": [ { "rank": 1, "tokenId": 27413, "text": "both", "score": 23.015499114990234 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 17.72096061706543 }, { "rank": 3, "tokenId": 75503, "text": "discovery", "score": 13.363018989562988 }, { "rank": 4, "tokenId": 13066, "text": "abstract", "score": 12.705236434936523 }, { "rank": 5, "tokenId": 128681, "text": "invention", "score": 11.830007553100586 }, { "rank": 6, "tokenId": 1800, "text": " both", "score": 11.710901260375977 }, { "rank": 7, "tokenId": 13395, "text": "found", "score": 10.686013221740723 }, { "rank": 8, "tokenId": 165997, "text": "purs", "score": 10.336806297302246 } ] }, "f32_tree64": { "winnerTokenId": 27413, "winnerText": "both", "winnerScore": 23.0154972076416, "winnerGap": 5.29454231262207, "top": [ { "rank": 1, "tokenId": 27413, "text": "both", "score": 23.0154972076416 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 17.72095489501953 }, { "rank": 3, "tokenId": 75503, "text": "discovery", "score": 13.363018989562988 }, { "rank": 4, "tokenId": 13066, "text": "abstract", "score": 12.705228805541992 }, { "rank": 5, "tokenId": 128681, "text": "invention", "score": 11.829999923706055 }, { "rank": 6, "tokenId": 1800, "text": " both", "score": 11.710892677307129 }, { "rank": 7, "tokenId": 13395, "text": "found", "score": 10.686015129089355 }, { "rank": 8, "tokenId": 165997, "text": "purs", "score": 10.336812019348145 } ] }, "f16_forward": { "winnerTokenId": 27413, "winnerText": "both", "winnerScore": 23.125, "winnerGap": 5.53125, "top": [ { "rank": 1, "tokenId": 27413, "text": "both", "score": 23.125 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 17.59375 }, { "rank": 3, "tokenId": 75503, "text": "discovery", "score": 13.3125 }, { "rank": 4, "tokenId": 13066, "text": "abstract", "score": 12.8671875 }, { "rank": 5, "tokenId": 128681, "text": "invention", "score": 11.7734375 }, { "rank": 6, "tokenId": 13395, "text": "found", "score": 10.7109375 }, { "rank": 7, "tokenId": 10508, "text": " Both", "score": 10.3125 }, { "rank": 8, "tokenId": 70277, "text": "Discovery", "score": 9.8359375 } ] } }, "winnerTextsByMode": { "exact": "both", "f32_forward": "both", "f32_reverse": "both", "f32_tree64": "both", "f16_forward": "both" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "both" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "discovery-invention-explicit-oneword-b", "text": "Respond with one word only.\nAnswer with exactly one word: discovery or invention. Question: Mathematics is Answer:", "stableTopK": [ { "rank": 1, "tokenId": 75503, "text": "discovery", "logit": 7.1701483726501465, "prob": 0.8304950594902039 }, { "rank": 2, "tokenId": 128681, "text": "invention", "logit": 5.580212593078613, "prob": 0.16937008500099182 }, { "rank": 3, "tokenId": 27413, "text": "both", "logit": -2.439279317855835, "prob": 0.00005572058216785081 }, { "rank": 4, "tokenId": 70277, "text": "Discovery", "logit": -3.173954725265503, "prob": 0.000026726995201897807 }, { "rank": 5, "tokenId": 13066, "text": "abstract", "logit": -3.803441047668457, "prob": 0.000014241893040889408 }, { "rank": 6, "tokenId": 209370, "text": "abstraction", "logit": -4.219400882720947, "prob": 0.000009395473171025515 }, { "rank": 7, "tokenId": 105461, "text": "achievement", "logit": -5.1032538414001465, "prob": 0.000003882095370499883 }, { "rank": 8, "tokenId": 902, "text": "In", "logit": -5.413759708404541, "prob": 0.0000028458709948608885 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 75503, "winnerText": "discovery", "winnerScore": 7.311549316851071, "winnerGap": 1.665608318000026, "top": [ { "rank": 1, "tokenId": 75503, "text": "discovery", "score": 7.311549316851071 }, { "rank": 2, "tokenId": 128681, "text": "invention", "score": 5.645940998851045 }, { "rank": 3, "tokenId": 27413, "text": "both", "score": -2.4446770403870097 }, { "rank": 4, "tokenId": 70277, "text": "Discovery", "score": -3.185880097195323 }, { "rank": 5, "tokenId": 13066, "text": "abstract", "score": -3.8240229510379677 }, { "rank": 6, "tokenId": 209370, "text": "abstraction", "score": -4.247555951728813 }, { "rank": 7, "tokenId": 105461, "text": "achievement", "score": -5.153353339064218 }, { "rank": 8, "tokenId": 902, "text": "In", "score": -5.473701396419919 } ] }, "f32_forward": { "winnerTokenId": 75503, "winnerText": "discovery", "winnerScore": 7.311553001403809, "winnerGap": 1.6656103134155273, "top": [ { "rank": 1, "tokenId": 75503, "text": "discovery", "score": 7.311553001403809 }, { "rank": 2, "tokenId": 128681, "text": "invention", "score": 5.645942687988281 }, { "rank": 3, "tokenId": 27413, "text": "both", "score": -2.4446794986724854 }, { "rank": 4, "tokenId": 70277, "text": "Discovery", "score": -3.1858837604522705 }, { "rank": 5, "tokenId": 13066, "text": "abstract", "score": -3.8240244388580322 }, { "rank": 6, "tokenId": 209370, "text": "abstraction", "score": -4.247557163238525 }, { "rank": 7, "tokenId": 105461, "text": "achievement", "score": -5.153351783752441 }, { "rank": 8, "tokenId": 902, "text": "In", "score": -5.473701477050781 } ] }, "f32_reverse": { "winnerTokenId": 75503, "winnerText": "discovery", "winnerScore": 7.311551094055176, "winnerGap": 1.6656007766723633, "top": [ { "rank": 1, "tokenId": 75503, "text": "discovery", "score": 7.311551094055176 }, { "rank": 2, "tokenId": 128681, "text": "invention", "score": 5.6459503173828125 }, { "rank": 3, "tokenId": 27413, "text": "both", "score": -2.444678544998169 }, { "rank": 4, "tokenId": 70277, "text": "Discovery", "score": -3.1858773231506348 }, { "rank": 5, "tokenId": 13066, "text": "abstract", "score": -3.8240296840667725 }, { "rank": 6, "tokenId": 209370, "text": "abstraction", "score": -4.247560977935791 }, { "rank": 7, "tokenId": 105461, "text": "achievement", "score": -5.153355598449707 }, { "rank": 8, "tokenId": 902, "text": "In", "score": -5.473690509796143 } ] }, "f32_tree64": { "winnerTokenId": 75503, "winnerText": "discovery", "winnerScore": 7.311551094055176, "winnerGap": 1.665611743927002, "top": [ { "rank": 1, "tokenId": 75503, "text": "discovery", "score": 7.311551094055176 }, { "rank": 2, "tokenId": 128681, "text": "invention", "score": 5.645939350128174 }, { "rank": 3, "tokenId": 27413, "text": "both", "score": -2.4446754455566406 }, { "rank": 4, "tokenId": 70277, "text": "Discovery", "score": -3.185882568359375 }, { "rank": 5, "tokenId": 13066, "text": "abstract", "score": -3.8240249156951904 }, { "rank": 6, "tokenId": 209370, "text": "abstraction", "score": -4.247555732727051 }, { "rank": 7, "tokenId": 105461, "text": "achievement", "score": -5.153352737426758 }, { "rank": 8, "tokenId": 902, "text": "In", "score": -5.4737019538879395 } ] }, "f16_forward": { "winnerTokenId": 75503, "winnerText": "discovery", "winnerScore": 7.33203125, "winnerGap": 1.66015625, "top": [ { "rank": 1, "tokenId": 75503, "text": "discovery", "score": 7.33203125 }, { "rank": 2, "tokenId": 128681, "text": "invention", "score": 5.671875 }, { "rank": 3, "tokenId": 27413, "text": "both", "score": -2.43359375 }, { "rank": 4, "tokenId": 70277, "text": "Discovery", "score": -3.15234375 }, { "rank": 5, "tokenId": 209370, "text": "abstraction", "score": -4.2578125 }, { "rank": 6, "tokenId": 231803, "text": "axiom", "score": -4.5859375 }, { "rank": 7, "tokenId": 192277, "text": " आविष्कार", "score": -4.69140625 }, { "rank": 8, "tokenId": 21692, "text": "construction", "score": -4.73046875 } ] } }, "winnerTextsByMode": { "exact": "discovery", "f32_forward": "discovery", "f32_reverse": "discovery", "f32_tree64": "discovery", "f16_forward": "discovery" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "discovery" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "earth-orbits-oneword-c", "text": "No explanation. One word only.\nThe Earth orbits the", "stableTopK": [ { "rank": 1, "tokenId": 18318, "text": "Sun", "logit": 9.832273483276367, "prob": 1 }, { "rank": 2, "tokenId": 7718, "text": " Sun", "logit": -13.015190124511719, "prob": 1.195288173994058e-10 }, { "rank": 3, "tokenId": 74361, "text": "Moon", "logit": -14.594502449035645, "prob": 2.4636896689411714e-11 }, { "rank": 4, "tokenId": 106037, "text": "SUN", "logit": -15.914533615112305, "prob": 6.581179611692134e-12 }, { "rank": 5, "tokenId": 16283, "text": "sun", "logit": -16.076004028320312, "prob": 5.599871261141587e-12 }, { "rank": 6, "tokenId": 81701, "text": "太阳", "logit": -16.461666107177734, "prob": 3.807914053521877e-12 }, { "rank": 7, "tokenId": 1018, "text": "**", "logit": -17.94076919555664, "prob": 8.676025392909126e-13 }, { "rank": 8, "tokenId": 236773, "text": "S", "logit": -18.183223724365234, "prob": 6.808072443471869e-13 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 18318, "winnerText": "Sun", "winnerScore": 10.208901291977604, "winnerGap": 24.147407834007332, "top": [ { "rank": 1, "tokenId": 18318, "text": "Sun", "score": 10.208901291977604 }, { "rank": 2, "tokenId": 7718, "text": " Sun", "score": -13.938506542029728 }, { "rank": 3, "tokenId": 74361, "text": "Moon", "score": -15.943295184277247 }, { "rank": 4, "tokenId": 106037, "text": "SUN", "score": -17.724565214280176 }, { "rank": 5, "tokenId": 16283, "text": "sun", "score": -17.950183300216203 }, { "rank": 6, "tokenId": 81701, "text": "太阳", "score": -18.496544335153146 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -20.702038534544315 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": -21.082354969861967 } ] }, "f32_forward": { "winnerTokenId": 18318, "winnerText": "Sun", "winnerScore": 10.208900451660156, "winnerGap": 24.147401809692383, "top": [ { "rank": 1, "tokenId": 18318, "text": "Sun", "score": 10.208900451660156 }, { "rank": 2, "tokenId": 7718, "text": " Sun", "score": -13.938501358032227 }, { "rank": 3, "tokenId": 74361, "text": "Moon", "score": -15.943297386169434 }, { "rank": 4, "tokenId": 106037, "text": "SUN", "score": -17.72457504272461 }, { "rank": 5, "tokenId": 16283, "text": "sun", "score": -17.95018196105957 }, { "rank": 6, "tokenId": 81701, "text": "太阳", "score": -18.49654769897461 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -20.70203971862793 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": -21.08235740661621 } ] }, "f32_reverse": { "winnerTokenId": 18318, "winnerText": "Sun", "winnerScore": 10.208906173706055, "winnerGap": 24.147382736206055, "top": [ { "rank": 1, "tokenId": 18318, "text": "Sun", "score": 10.208906173706055 }, { "rank": 2, "tokenId": 7718, "text": " Sun", "score": -13.9384765625 }, { "rank": 3, "tokenId": 74361, "text": "Moon", "score": -15.943292617797852 }, { "rank": 4, "tokenId": 106037, "text": "SUN", "score": -17.72455406188965 }, { "rank": 5, "tokenId": 16283, "text": "sun", "score": -17.95018196105957 }, { "rank": 6, "tokenId": 81701, "text": "太阳", "score": -18.496543884277344 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -20.70204734802246 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": -21.08235740661621 } ] }, "f32_tree64": { "winnerTokenId": 18318, "winnerText": "Sun", "winnerScore": 10.208901405334473, "winnerGap": 24.14740562438965, "top": [ { "rank": 1, "tokenId": 18318, "text": "Sun", "score": 10.208901405334473 }, { "rank": 2, "tokenId": 7718, "text": " Sun", "score": -13.938504219055176 }, { "rank": 3, "tokenId": 74361, "text": "Moon", "score": -15.943294525146484 }, { "rank": 4, "tokenId": 106037, "text": "SUN", "score": -17.724563598632812 }, { "rank": 5, "tokenId": 16283, "text": "sun", "score": -17.950183868408203 }, { "rank": 6, "tokenId": 81701, "text": "太阳", "score": -18.496543884277344 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -20.70203971862793 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": -21.082355499267578 } ] }, "f16_forward": { "winnerTokenId": 18318, "winnerText": "Sun", "winnerScore": 10.234375, "winnerGap": 22.75, "top": [ { "rank": 1, "tokenId": 18318, "text": "Sun", "score": 10.234375 }, { "rank": 2, "tokenId": 170970, "text": " Сол", "score": -12.515625 }, { "rank": 3, "tokenId": 715, "text": "//", "score": -13.8125 }, { "rank": 4, "tokenId": 7718, "text": " Sun", "score": -14.109375 }, { "rank": 5, "tokenId": 74361, "text": "Moon", "score": -15.8984375 }, { "rank": 6, "tokenId": 16283, "text": "sun", "score": -16.859375 }, { "rank": 7, "tokenId": 81701, "text": "太阳", "score": -18.46875 }, { "rank": 8, "tokenId": 106037, "text": "SUN", "score": -18.71875 } ] } }, "winnerTextsByMode": { "exact": "Sun", "f32_forward": "Sun", "f32_reverse": "Sun", "f32_tree64": "Sun", "f16_forward": "Sun" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Sun" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "econ-globalization-oneword-d", "text": "Choose one word only.\nGlobalization benefits or exploits developing nations. It", "stableTopK": [ { "rank": 1, "tokenId": 20647, "text": "Expl", "logit": 18.431140899658203, "prob": 0.6161534786224365 }, { "rank": 2, "tokenId": 1018, "text": "**", "logit": 16.936161041259766, "prob": 0.13817434012889862 }, { "rank": 3, "tokenId": 69720, "text": "Benefits", "logit": 16.557220458984375, "prob": 0.09459225833415985 }, { "rank": 4, "tokenId": 55144, "text": "Amb", "logit": 16.074810028076172, "prob": 0.05839120224118233 }, { "rank": 5, "tokenId": 72187, "text": "explo", "logit": 15.900717735290527, "prob": 0.04906141385436058 }, { "rank": 6, "tokenId": 48333, "text": "Complex", "logit": 14.510178565979004, "prob": 0.012213400565087795 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "logit": 14.241033554077148, "prob": 0.009331434033811092 }, { "rank": 8, "tokenId": 818, "text": "The", "logit": 14.138016700744629, "prob": 0.00841799657791853 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 20647, "winnerText": "Expl", "winnerScore": 21.477392347827433, "winnerGap": 2.29328507814774, "top": [ { "rank": 1, "tokenId": 20647, "text": "Expl", "score": 21.477392347827433 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 19.184107269679693 }, { "rank": 3, "tokenId": 69720, "text": "Benefits", "score": 18.63360318598002 }, { "rank": 4, "tokenId": 55144, "text": "Amb", "score": 17.948501336883282 }, { "rank": 5, "tokenId": 72187, "text": "explo", "score": 17.70535863098283 }, { "rank": 6, "tokenId": 48333, "text": "Complex", "score": 15.833026956480523 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": 15.483651426555795 }, { "rank": 8, "tokenId": 818, "text": "The", "score": 15.350945947347112 } ] }, "f32_forward": { "winnerTokenId": 20647, "winnerText": "Expl", "winnerScore": 21.47739601135254, "winnerGap": 2.293313980102539, "top": [ { "rank": 1, "tokenId": 20647, "text": "Expl", "score": 21.47739601135254 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 19.18408203125 }, { "rank": 3, "tokenId": 69720, "text": "Benefits", "score": 18.63360023498535 }, { "rank": 4, "tokenId": 55144, "text": "Amb", "score": 17.948463439941406 }, { "rank": 5, "tokenId": 72187, "text": "explo", "score": 17.705354690551758 }, { "rank": 6, "tokenId": 48333, "text": "Complex", "score": 15.83301067352295 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": 15.483647346496582 }, { "rank": 8, "tokenId": 818, "text": "The", "score": 15.350927352905273 } ] }, "f32_reverse": { "winnerTokenId": 20647, "winnerText": "Expl", "winnerScore": 21.47740364074707, "winnerGap": 2.2933006286621094, "top": [ { "rank": 1, "tokenId": 20647, "text": "Expl", "score": 21.47740364074707 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 19.18410301208496 }, { "rank": 3, "tokenId": 69720, "text": "Benefits", "score": 18.633602142333984 }, { "rank": 4, "tokenId": 55144, "text": "Amb", "score": 17.948495864868164 }, { "rank": 5, "tokenId": 72187, "text": "explo", "score": 17.705358505249023 }, { "rank": 6, "tokenId": 48333, "text": "Complex", "score": 15.833038330078125 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": 15.483654022216797 }, { "rank": 8, "tokenId": 818, "text": "The", "score": 15.350948333740234 } ] }, "f32_tree64": { "winnerTokenId": 20647, "winnerText": "Expl", "winnerScore": 21.47739028930664, "winnerGap": 2.2932815551757812, "top": [ { "rank": 1, "tokenId": 20647, "text": "Expl", "score": 21.47739028930664 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 19.18410873413086 }, { "rank": 3, "tokenId": 69720, "text": "Benefits", "score": 18.633604049682617 }, { "rank": 4, "tokenId": 55144, "text": "Amb", "score": 17.948501586914062 }, { "rank": 5, "tokenId": 72187, "text": "explo", "score": 17.705358505249023 }, { "rank": 6, "tokenId": 48333, "text": "Complex", "score": 15.833026885986328 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": 15.483649253845215 }, { "rank": 8, "tokenId": 818, "text": "The", "score": 15.350945472717285 } ] }, "f16_forward": { "winnerTokenId": 20647, "winnerText": "Expl", "winnerScore": 21.546875, "winnerGap": 2.53125, "top": [ { "rank": 1, "tokenId": 20647, "text": "Expl", "score": 21.546875 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 19.015625 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "score": 18.328125 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 15.75 }, { "rank": 5, "tokenId": 69720, "text": "Benefits", "score": 14.765625 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": 14.5 }, { "rank": 7, "tokenId": 818, "text": "The", "score": 14.3671875 }, { "rank": 8, "tokenId": 105367, "text": "Mixed", "score": 13.7890625 } ] } }, "winnerTextsByMode": { "exact": "Expl", "f32_forward": "Expl", "f32_reverse": "Expl", "f32_tree64": "Expl", "f16_forward": "Expl" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Expl" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "econ-housing-crisis-oneword-a", "text": "Answer with exactly one word only.\nThe housing crisis is caused by regulation or speculation. It is caused by", "stableTopK": [ { "rank": 1, "tokenId": 9798, "text": "Spec", "logit": 16.92144203186035, "prob": 0.9930452704429626 }, { "rank": 2, "tokenId": 22186, "text": "Both", "logit": 11.592145919799805, "prob": 0.004813767969608307 }, { "rank": 3, "tokenId": 165411, "text": "Regulation", "logit": 10.658465385437012, "prob": 0.001892312429845333 }, { "rank": 4, "tokenId": 5690, "text": "spec", "logit": 7.629667282104492, "prob": 0.00009153823339147493 }, { "rank": 5, "tokenId": 27413, "text": "both", "logit": 6.383591175079346, "prob": 0.000026329253159929067 }, { "rank": 6, "tokenId": 58142, "text": "Supply", "logit": 6.085705280303955, "prob": 0.000019546470866771415 }, { "rank": 7, "tokenId": 104290, "text": "Specification", "logit": 6.042746543884277, "prob": 0.00001872456050477922 }, { "rank": 8, "tokenId": 181023, "text": "Ine", "logit": 6.000962257385254, "prob": 0.00001795828757167328 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 9798, "winnerText": "Spec", "winnerScore": 19.162498881689203, "winnerGap": 6.9354889908834085, "top": [ { "rank": 1, "tokenId": 9798, "text": "Spec", "score": 19.162498881689203 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 12.227009890805794 }, { "rank": 3, "tokenId": 165411, "text": "Regulation", "score": 11.144285816170552 }, { "rank": 4, "tokenId": 5690, "text": "spec", "score": 7.800859874085091 }, { "rank": 5, "tokenId": 27413, "text": "both", "score": 6.482640310709954 }, { "rank": 6, "tokenId": 58142, "text": "Supply", "score": 6.171307236045486 }, { "rank": 7, "tokenId": 104290, "text": "Specification", "score": 6.126515984581598 }, { "rank": 8, "tokenId": 181023, "text": "Ine", "score": 6.082985434792066 } ] }, "f32_forward": { "winnerTokenId": 9798, "winnerText": "Spec", "winnerScore": 19.16246223449707, "winnerGap": 6.935457229614258, "top": [ { "rank": 1, "tokenId": 9798, "text": "Spec", "score": 19.16246223449707 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 12.227005004882812 }, { "rank": 3, "tokenId": 165411, "text": "Regulation", "score": 11.144287109375 }, { "rank": 4, "tokenId": 5690, "text": "spec", "score": 7.800848007202148 }, { "rank": 5, "tokenId": 27413, "text": "both", "score": 6.482640266418457 }, { "rank": 6, "tokenId": 58142, "text": "Supply", "score": 6.1713080406188965 }, { "rank": 7, "tokenId": 104290, "text": "Specification", "score": 6.126524448394775 }, { "rank": 8, "tokenId": 181023, "text": "Ine", "score": 6.082981586456299 } ] }, "f32_reverse": { "winnerTokenId": 9798, "winnerText": "Spec", "winnerScore": 19.162498474121094, "winnerGap": 6.935489654541016, "top": [ { "rank": 1, "tokenId": 9798, "text": "Spec", "score": 19.162498474121094 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 12.227008819580078 }, { "rank": 3, "tokenId": 165411, "text": "Regulation", "score": 11.14428997039795 }, { "rank": 4, "tokenId": 5690, "text": "spec", "score": 7.8008599281311035 }, { "rank": 5, "tokenId": 27413, "text": "both", "score": 6.482638359069824 }, { "rank": 6, "tokenId": 58142, "text": "Supply", "score": 6.171306610107422 }, { "rank": 7, "tokenId": 104290, "text": "Specification", "score": 6.126516342163086 }, { "rank": 8, "tokenId": 181023, "text": "Ine", "score": 6.082986831665039 } ] }, "f32_tree64": { "winnerTokenId": 9798, "winnerText": "Spec", "winnerScore": 19.162498474121094, "winnerGap": 6.935487747192383, "top": [ { "rank": 1, "tokenId": 9798, "text": "Spec", "score": 19.162498474121094 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 12.227010726928711 }, { "rank": 3, "tokenId": 165411, "text": "Regulation", "score": 11.144287109375 }, { "rank": 4, "tokenId": 5690, "text": "spec", "score": 7.800860404968262 }, { "rank": 5, "tokenId": 27413, "text": "both", "score": 6.482640266418457 }, { "rank": 6, "tokenId": 58142, "text": "Supply", "score": 6.171307563781738 }, { "rank": 7, "tokenId": 104290, "text": "Specification", "score": 6.1265153884887695 }, { "rank": 8, "tokenId": 181023, "text": "Ine", "score": 6.082984924316406 } ] }, "f16_forward": { "winnerTokenId": 9798, "winnerText": "Spec", "winnerScore": 19.28125, "winnerGap": 7.0546875, "top": [ { "rank": 1, "tokenId": 9798, "text": "Spec", "score": 19.28125 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 12.2265625 }, { "rank": 3, "tokenId": 165411, "text": "Regulation", "score": 11.1015625 }, { "rank": 4, "tokenId": 5690, "text": "spec", "score": 7.78125 }, { "rank": 5, "tokenId": 27413, "text": "both", "score": 6.53125 }, { "rank": 6, "tokenId": 58142, "text": "Supply", "score": 6.20703125 }, { "rank": 7, "tokenId": 104290, "text": "Specification", "score": 6.1484375 }, { "rank": 8, "tokenId": 181023, "text": "Ine", "score": 6.1015625 } ] } }, "winnerTextsByMode": { "exact": "Spec", "f32_forward": "Spec", "f32_reverse": "Spec", "f32_tree64": "Spec", "f16_forward": "Spec" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Spec" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "econ-minimum-wage-oneword-b", "text": "Respond with one word only.\nRaising the minimum wage helps or hurts workers. It", "stableTopK": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "logit": 11.346936225891113, "prob": 0.9545775651931763 }, { "rank": 2, "tokenId": 9822, "text": "Hel", "logit": 7.366363048553467, "prob": 0.01782667264342308 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "logit": 6.995725154876709, "prob": 0.01230564247816801 }, { "rank": 4, "tokenId": 22186, "text": "Both", "logit": 6.382133483886719, "prob": 0.006662309169769287 }, { "rank": 5, "tokenId": 236847, "text": "V", "logit": 5.722298622131348, "prob": 0.003443992231041193 }, { "rank": 6, "tokenId": 113214, "text": "depends", "logit": 5.304407119750977, "prob": 0.0022676405496895313 }, { "rank": 7, "tokenId": 105367, "text": "Mixed", "logit": 4.388818740844727, "prob": 0.000907693465705961 }, { "rank": 8, "tokenId": 48333, "text": "Complex", "logit": 4.305291652679443, "prob": 0.0008349564741365612 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 11.93981813056033, "winnerGap": 4.41980958048608, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 11.93981813056033 }, { "rank": 2, "tokenId": 9822, "text": "Hel", "score": 7.520008550074249 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "score": 7.126833899546246 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 6.4811219886823395 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": 5.793254015321809 }, { "rank": 6, "tokenId": 113214, "text": "depends", "score": 5.360748779562568 }, { "rank": 7, "tokenId": 105367, "text": "Mixed", "score": 4.420538310880872 }, { "rank": 8, "tokenId": 48333, "text": "Complex", "score": 4.335218671252579 } ] }, "f32_forward": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 11.939807891845703, "winnerGap": 4.419808387756348, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 11.939807891845703 }, { "rank": 2, "tokenId": 9822, "text": "Hel", "score": 7.5199995040893555 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "score": 7.126833438873291 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 6.481123924255371 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": 5.793251991271973 }, { "rank": 6, "tokenId": 113214, "text": "depends", "score": 5.360745429992676 }, { "rank": 7, "tokenId": 105367, "text": "Mixed", "score": 4.420533657073975 }, { "rank": 8, "tokenId": 48333, "text": "Complex", "score": 4.335224628448486 } ] }, "f32_reverse": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 11.939811706542969, "winnerGap": 4.419795989990234, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 11.939811706542969 }, { "rank": 2, "tokenId": 9822, "text": "Hel", "score": 7.520015716552734 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "score": 7.126835823059082 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 6.48111629486084 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": 5.7932515144348145 }, { "rank": 6, "tokenId": 113214, "text": "depends", "score": 5.360746383666992 }, { "rank": 7, "tokenId": 105367, "text": "Mixed", "score": 4.420538425445557 }, { "rank": 8, "tokenId": 48333, "text": "Complex", "score": 4.335214138031006 } ] }, "f32_tree64": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 11.939817428588867, "winnerGap": 4.419809341430664, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 11.939817428588867 }, { "rank": 2, "tokenId": 9822, "text": "Hel", "score": 7.520008087158203 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "score": 7.126833915710449 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 6.481122016906738 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": 5.7932538986206055 }, { "rank": 6, "tokenId": 113214, "text": "depends", "score": 5.360749244689941 }, { "rank": 7, "tokenId": 105367, "text": "Mixed", "score": 4.420537948608398 }, { "rank": 8, "tokenId": 48333, "text": "Complex", "score": 4.33521842956543 } ] }, "f16_forward": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 11.9453125, "winnerGap": 4.45703125, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 11.9453125 }, { "rank": 2, "tokenId": 9822, "text": "Hel", "score": 7.48828125 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "score": 7.12890625 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 6.48828125 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": 5.80078125 }, { "rank": 6, "tokenId": 113214, "text": "depends", "score": 5.43359375 }, { "rank": 7, "tokenId": 105367, "text": "Mixed", "score": 4.42578125 }, { "rank": 8, "tokenId": 48333, "text": "Complex", "score": 4.40234375 } ] } }, "winnerTextsByMode": { "exact": "Depends", "f32_forward": "Depends", "f32_reverse": "Depends", "f32_tree64": "Depends", "f16_forward": "Depends" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Depends" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "econ-rent-control-oneword-c", "text": "No explanation. One word only.\nRent control helps or hurts tenants in the long run. It", "stableTopK": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "logit": 9.644481658935547, "prob": 0.7538787722587585 }, { "rank": 2, "tokenId": 55144, "text": "Amb", "logit": 8.151609420776367, "prob": 0.16941635310649872 }, { "rank": 3, "tokenId": 105367, "text": "Mixed", "logit": 6.700873374938965, "prob": 0.03971080482006073 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "logit": 5.1551127433776855, "prob": 0.008464344777166843 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "logit": 4.758399486541748, "prob": 0.005692499224096537 }, { "rank": 6, "tokenId": 9822, "text": "Hel", "logit": 4.5053582191467285, "prob": 0.004419860430061817 }, { "rank": 7, "tokenId": 20809, "text": "Neutral", "logit": 4.228568077087402, "prob": 0.0033511982765048742 }, { "rank": 8, "tokenId": 236847, "text": "V", "logit": 3.9418649673461914, "prob": 0.002515860367566347 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 9.998998545536857, "winnerGap": 1.6373908781779676, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 9.998998545536857 }, { "rank": 2, "tokenId": 55144, "text": "Amb", "score": 8.36160766735889 }, { "rank": 3, "tokenId": 105367, "text": "Mixed", "score": 6.81576887943848 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 5.206776263420721 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 4.798921736763127 }, { "rank": 6, "tokenId": 9822, "text": "Hel", "score": 4.53969524137715 }, { "rank": 7, "tokenId": 20809, "text": "Neutral", "score": 4.256909168019945 }, { "rank": 8, "tokenId": 236847, "text": "V", "score": 3.9647869579802233 } ] }, "f32_forward": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 9.998991966247559, "winnerGap": 1.6373834609985352, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 9.998991966247559 }, { "rank": 2, "tokenId": 55144, "text": "Amb", "score": 8.361608505249023 }, { "rank": 3, "tokenId": 105367, "text": "Mixed", "score": 6.815767765045166 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 5.206776142120361 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 4.798919200897217 }, { "rank": 6, "tokenId": 9822, "text": "Hel", "score": 4.539694309234619 }, { "rank": 7, "tokenId": 20809, "text": "Neutral", "score": 4.256916046142578 }, { "rank": 8, "tokenId": 236847, "text": "V", "score": 3.964792490005493 } ] }, "f32_reverse": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 9.998991966247559, "winnerGap": 1.637375831604004, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 9.998991966247559 }, { "rank": 2, "tokenId": 55144, "text": "Amb", "score": 8.361616134643555 }, { "rank": 3, "tokenId": 105367, "text": "Mixed", "score": 6.815770626068115 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 5.206772804260254 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 4.798923492431641 }, { "rank": 6, "tokenId": 9822, "text": "Hel", "score": 4.539694786071777 }, { "rank": 7, "tokenId": 20809, "text": "Neutral", "score": 4.256907939910889 }, { "rank": 8, "tokenId": 236847, "text": "V", "score": 3.9647862911224365 } ] }, "f32_tree64": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 9.998998641967773, "winnerGap": 1.6373910903930664, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 9.998998641967773 }, { "rank": 2, "tokenId": 55144, "text": "Amb", "score": 8.361607551574707 }, { "rank": 3, "tokenId": 105367, "text": "Mixed", "score": 6.815769195556641 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 5.2067766189575195 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 4.798922538757324 }, { "rank": 6, "tokenId": 9822, "text": "Hel", "score": 4.5396952629089355 }, { "rank": 7, "tokenId": 20809, "text": "Neutral", "score": 4.256908416748047 }, { "rank": 8, "tokenId": 236847, "text": "V", "score": 3.9647865295410156 } ] }, "f16_forward": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 10.1015625, "winnerGap": 1.6875, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 10.1015625 }, { "rank": 2, "tokenId": 55144, "text": "Amb", "score": 8.4140625 }, { "rank": 3, "tokenId": 105367, "text": "Mixed", "score": 5.8203125 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 5.16015625 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 4.84375 }, { "rank": 6, "tokenId": 9822, "text": "Hel", "score": 4.578125 }, { "rank": 7, "tokenId": 20809, "text": "Neutral", "score": 4.2734375 }, { "rank": 8, "tokenId": 3637, "text": "Context", "score": 3.962890625 } ] } }, "winnerTextsByMode": { "exact": "Depends", "f32_forward": "Depends", "f32_reverse": "Depends", "f32_tree64": "Depends", "f16_forward": "Depends" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Depends" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "econ-student-debt-oneword-d", "text": "Choose one word only.\nCanceling student debt is fair or unfair to those who already paid. It is", "stableTopK": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "logit": 8.400486946105957, "prob": 0.5367624759674072 }, { "rank": 2, "tokenId": 1018, "text": "**", "logit": 7.197940826416016, "prob": 0.16125863790512085 }, { "rank": 3, "tokenId": 2805, "text": "Un", "logit": 6.847477436065674, "prob": 0.11358439922332764 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "logit": 5.657687187194824, "prob": 0.03456203639507294 }, { "rank": 5, "tokenId": 191239, "text": "Difficult", "logit": 5.478579521179199, "prob": 0.028894413262605667 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "logit": 5.344878673553467, "prob": 0.025278326123952866 }, { "rank": 7, "tokenId": 21288, "text": "Subject", "logit": 4.927038669586182, "prob": 0.016644958406686783 }, { "rank": 8, "tokenId": 34622, "text": "complex", "logit": 4.794528007507324, "prob": 0.014579212293028831 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 48333, "winnerText": "Complex", "winnerScore": 8.630989775238312, "winnerGap": 1.289951459830835, "top": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "score": 8.630989775238312 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 7.341038315407477 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": 6.970248555594736 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 5.7262320842706345 }, { "rank": 5, "tokenId": 191239, "text": "Difficult", "score": 5.540730143667012 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "score": 5.402536498651102 }, { "rank": 7, "tokenId": 21288, "text": "Subject", "score": 4.9720681725850255 }, { "rank": 8, "tokenId": 34622, "text": "complex", "score": 4.835987297289023 } ] }, "f32_forward": { "winnerTokenId": 48333, "winnerText": "Complex", "winnerScore": 8.630987167358398, "winnerGap": 1.2899503707885742, "top": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "score": 8.630987167358398 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 7.341036796569824 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": 6.970231056213379 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 5.726222991943359 }, { "rank": 5, "tokenId": 191239, "text": "Difficult", "score": 5.540725231170654 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "score": 5.402532577514648 }, { "rank": 7, "tokenId": 21288, "text": "Subject", "score": 4.972064971923828 }, { "rank": 8, "tokenId": 34622, "text": "complex", "score": 4.835991382598877 } ] }, "f32_reverse": { "winnerTokenId": 48333, "winnerText": "Complex", "winnerScore": 8.630987167358398, "winnerGap": 1.2899513244628906, "top": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "score": 8.630987167358398 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 7.341035842895508 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": 6.970251083374023 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 5.726228713989258 }, { "rank": 5, "tokenId": 191239, "text": "Difficult", "score": 5.540729999542236 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "score": 5.402535915374756 }, { "rank": 7, "tokenId": 21288, "text": "Subject", "score": 4.97206974029541 }, { "rank": 8, "tokenId": 34622, "text": "complex", "score": 4.835989952087402 } ] }, "f32_tree64": { "winnerTokenId": 48333, "winnerText": "Complex", "winnerScore": 8.630989074707031, "winnerGap": 1.2899508476257324, "top": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "score": 8.630989074707031 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 7.341038227081299 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": 6.970248699188232 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 5.726232051849365 }, { "rank": 5, "tokenId": 191239, "text": "Difficult", "score": 5.540729522705078 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "score": 5.402536392211914 }, { "rank": 7, "tokenId": 21288, "text": "Subject", "score": 4.972068786621094 }, { "rank": 8, "tokenId": 34622, "text": "complex", "score": 4.8359880447387695 } ] }, "f16_forward": { "winnerTokenId": 1018, "winnerText": "**", "winnerScore": 7.375, "winnerGap": 0.5, "top": [ { "rank": 1, "tokenId": 1018, "text": "**", "score": 7.375 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 6.875 }, { "rank": 3, "tokenId": 191239, "text": "Difficult", "score": 5.5390625 }, { "rank": 4, "tokenId": 1584, "text": "Cont", "score": 5.37890625 }, { "rank": 5, "tokenId": 34622, "text": "complex", "score": 4.875 }, { "rank": 6, "tokenId": 48333, "text": "Complex", "score": 4.68359375 }, { "rank": 7, "tokenId": 818, "text": "The", "score": 4.2890625 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 4.203125 } ] } }, "winnerTextsByMode": { "exact": "Complex", "f32_forward": "Complex", "f32_reverse": "Complex", "f32_tree64": "Complex", "f16_forward": "**" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Complex", "**" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 48333, 106 ], "decodedText": "Complex", "steps": [ { "step": 0, "tokenId": 48333, "text": "Complex" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 1018, 48333, 1018 ], "decodedText": "**Complex**", "steps": [ { "step": 0, "tokenId": 1018, "text": "**" }, { "step": 1, "tokenId": 48333, "text": "Complex" }, { "step": 2, "tokenId": 1018, "text": "**" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 2, "differingStepCount": 3, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "econ-trickle-down-oneword-a", "text": "Answer with exactly one word only.\nTrickle-down economics works or fails in practice. It", "stableTopK": [ { "rank": 1, "tokenId": 236847, "text": "V", "logit": 11.852731704711914, "prob": 0.4316113591194153 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "logit": 11.465584754943848, "prob": 0.2930603623390198 }, { "rank": 3, "tokenId": 236811, "text": "F", "logit": 10.883563995361328, "prob": 0.16375277936458588 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "logit": 10.016096115112305, "prob": 0.06877841800451279 }, { "rank": 5, "tokenId": 113214, "text": "depends", "logit": 8.455989837646484, "prob": 0.014451290480792522 }, { "rank": 6, "tokenId": 144828, "text": "fails", "logit": 7.3747453689575195, "prob": 0.004901490174233913 }, { "rank": 7, "tokenId": 894, "text": "St", "logit": 6.589773178100586, "prob": 0.0022357283160090446 }, { "rank": 8, "tokenId": 1967, "text": "var", "logit": 6.507379055023193, "prob": 0.0020589022897183895 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 236847, "winnerText": "V", "winnerScore": 12.534555116739229, "winnerGap": 0.4560368917560442, "top": [ { "rank": 1, "tokenId": 236847, "text": "V", "score": 12.534555116739229 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 12.078518224983185 }, { "rank": 3, "tokenId": 236811, "text": "F", "score": 11.402693977549593 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 10.415323573861 }, { "rank": 5, "tokenId": 113214, "text": "depends", "score": 8.691250565399045 }, { "rank": 6, "tokenId": 144828, "text": "fails", "score": 7.528928831597034 }, { "rank": 7, "tokenId": 894, "text": "St", "score": 6.6989368442759645 }, { "rank": 8, "tokenId": 1967, "text": "var", "score": 6.612421672500053 } ] }, "f32_forward": { "winnerTokenId": 236847, "winnerText": "V", "winnerScore": 12.534554481506348, "winnerGap": 0.4560394287109375, "top": [ { "rank": 1, "tokenId": 236847, "text": "V", "score": 12.534554481506348 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 12.07851505279541 }, { "rank": 3, "tokenId": 236811, "text": "F", "score": 11.4027099609375 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 10.415334701538086 }, { "rank": 5, "tokenId": 113214, "text": "depends", "score": 8.691261291503906 }, { "rank": 6, "tokenId": 144828, "text": "fails", "score": 7.5289306640625 }, { "rank": 7, "tokenId": 894, "text": "St", "score": 6.698939323425293 }, { "rank": 8, "tokenId": 1967, "text": "var", "score": 6.612414836883545 } ] }, "f32_reverse": { "winnerTokenId": 236847, "winnerText": "V", "winnerScore": 12.534551620483398, "winnerGap": 0.45603179931640625, "top": [ { "rank": 1, "tokenId": 236847, "text": "V", "score": 12.534551620483398 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 12.078519821166992 }, { "rank": 3, "tokenId": 236811, "text": "F", "score": 11.402697563171387 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 10.415328025817871 }, { "rank": 5, "tokenId": 113214, "text": "depends", "score": 8.691258430480957 }, { "rank": 6, "tokenId": 144828, "text": "fails", "score": 7.528924942016602 }, { "rank": 7, "tokenId": 894, "text": "St", "score": 6.698932647705078 }, { "rank": 8, "tokenId": 1967, "text": "var", "score": 6.612423419952393 } ] }, "f32_tree64": { "winnerTokenId": 236847, "winnerText": "V", "winnerScore": 12.534555435180664, "winnerGap": 0.4560365676879883, "top": [ { "rank": 1, "tokenId": 236847, "text": "V", "score": 12.534555435180664 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 12.078518867492676 }, { "rank": 3, "tokenId": 236811, "text": "F", "score": 11.402693748474121 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 10.415323257446289 }, { "rank": 5, "tokenId": 113214, "text": "depends", "score": 8.69124984741211 }, { "rank": 6, "tokenId": 144828, "text": "fails", "score": 7.528928756713867 }, { "rank": 7, "tokenId": 894, "text": "St", "score": 6.698936462402344 }, { "rank": 8, "tokenId": 1967, "text": "var", "score": 6.612421989440918 } ] }, "f16_forward": { "winnerTokenId": 236847, "winnerText": "V", "winnerScore": 12.5703125, "winnerGap": 0.5078125, "top": [ { "rank": 1, "tokenId": 236847, "text": "V", "score": 12.5703125 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 12.0625 }, { "rank": 3, "tokenId": 236811, "text": "F", "score": 11.5 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 10.46875 }, { "rank": 5, "tokenId": 113214, "text": "depends", "score": 8.65625 }, { "rank": 6, "tokenId": 144828, "text": "fails", "score": 7.53515625 }, { "rank": 7, "tokenId": 894, "text": "St", "score": 6.76953125 }, { "rank": 8, "tokenId": 1967, "text": "var", "score": 6.58203125 } ] } }, "winnerTextsByMode": { "exact": "V", "f32_forward": "V", "f32_reverse": "V", "f32_tree64": "V", "f16_forward": "V" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "V" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "econ-union-power-oneword-b", "text": "Respond with one word only.\nStronger labor unions help or hurt workers. They", "stableTopK": [ { "rank": 1, "tokenId": 28368, "text": "Help", "logit": 10.88725757598877, "prob": 0.9807524681091309 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "logit": 6.396163463592529, "prob": 0.010992640629410744 }, { "rank": 3, "tokenId": 22186, "text": "Both", "logit": 5.258649826049805, "prob": 0.00352440751157701 }, { "rank": 4, "tokenId": 236847, "text": "V", "logit": 4.327152252197266, "prob": 0.0013884871732443571 }, { "rank": 5, "tokenId": 17002, "text": "help", "logit": 3.7294158935546875, "prob": 0.0007637448143213987 }, { "rank": 6, "tokenId": 9822, "text": "Hel", "logit": 3.7285220623016357, "prob": 0.0007630624459125102 }, { "rank": 7, "tokenId": 178899, "text": "Benefit", "logit": 3.0951120853424072, "prob": 0.00040501734474673867 }, { "rank": 8, "tokenId": 31049, "text": "Depend", "logit": 3.0343165397644043, "prob": 0.00038112764013931155 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 28368, "winnerText": "Help", "winnerScore": 11.406951483037993, "winnerGap": 4.9111415601333945, "top": [ { "rank": 1, "tokenId": 28368, "text": "Help", "score": 11.406951483037993 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 6.4958099229045985 }, { "rank": 3, "tokenId": 22186, "text": "Both", "score": 5.313521657020672 }, { "rank": 4, "tokenId": 236847, "text": "V", "score": 4.357537979731905 }, { "rank": 5, "tokenId": 17002, "text": "help", "score": 3.7488108371909235 }, { "rank": 6, "tokenId": 9822, "text": "Hel", "score": 3.7479015216519187 }, { "rank": 7, "tokenId": 178899, "text": "Benefit", "score": 3.106163819991446 }, { "rank": 8, "tokenId": 31049, "text": "Depend", "score": 3.044732337502566 } ] }, "f32_forward": { "winnerTokenId": 28368, "winnerText": "Help", "winnerScore": 11.406943321228027, "winnerGap": 4.911136150360107, "top": [ { "rank": 1, "tokenId": 28368, "text": "Help", "score": 11.406943321228027 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 6.49580717086792 }, { "rank": 3, "tokenId": 22186, "text": "Both", "score": 5.313518524169922 }, { "rank": 4, "tokenId": 236847, "text": "V", "score": 4.357542514801025 }, { "rank": 5, "tokenId": 17002, "text": "help", "score": 3.7488064765930176 }, { "rank": 6, "tokenId": 9822, "text": "Hel", "score": 3.7479026317596436 }, { "rank": 7, "tokenId": 178899, "text": "Benefit", "score": 3.106163501739502 }, { "rank": 8, "tokenId": 31049, "text": "Depend", "score": 3.0447287559509277 } ] }, "f32_reverse": { "winnerTokenId": 28368, "winnerText": "Help", "winnerScore": 11.406959533691406, "winnerGap": 4.911149501800537, "top": [ { "rank": 1, "tokenId": 28368, "text": "Help", "score": 11.406959533691406 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 6.495810031890869 }, { "rank": 3, "tokenId": 22186, "text": "Both", "score": 5.3135247230529785 }, { "rank": 4, "tokenId": 236847, "text": "V", "score": 4.357536315917969 }, { "rank": 5, "tokenId": 17002, "text": "help", "score": 3.7488081455230713 }, { "rank": 6, "tokenId": 9822, "text": "Hel", "score": 3.747894525527954 }, { "rank": 7, "tokenId": 178899, "text": "Benefit", "score": 3.10616397857666 }, { "rank": 8, "tokenId": 31049, "text": "Depend", "score": 3.044731378555298 } ] }, "f32_tree64": { "winnerTokenId": 28368, "winnerText": "Help", "winnerScore": 11.406951904296875, "winnerGap": 4.911142349243164, "top": [ { "rank": 1, "tokenId": 28368, "text": "Help", "score": 11.406951904296875 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 6.495809555053711 }, { "rank": 3, "tokenId": 22186, "text": "Both", "score": 5.313521385192871 }, { "rank": 4, "tokenId": 236847, "text": "V", "score": 4.357538223266602 }, { "rank": 5, "tokenId": 17002, "text": "help", "score": 3.748810291290283 }, { "rank": 6, "tokenId": 9822, "text": "Hel", "score": 3.747901439666748 }, { "rank": 7, "tokenId": 178899, "text": "Benefit", "score": 3.10616397857666 }, { "rank": 8, "tokenId": 31049, "text": "Depend", "score": 3.0447325706481934 } ] }, "f16_forward": { "winnerTokenId": 28368, "winnerText": "Help", "winnerScore": 11.40625, "winnerGap": 4.859375, "top": [ { "rank": 1, "tokenId": 28368, "text": "Help", "score": 11.40625 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 6.546875 }, { "rank": 3, "tokenId": 22186, "text": "Both", "score": 5.30859375 }, { "rank": 4, "tokenId": 236847, "text": "V", "score": 4.38671875 }, { "rank": 5, "tokenId": 17002, "text": "help", "score": 3.771484375 }, { "rank": 6, "tokenId": 9822, "text": "Hel", "score": 3.73828125 }, { "rank": 7, "tokenId": 178899, "text": "Benefit", "score": 3.123046875 }, { "rank": 8, "tokenId": 31049, "text": "Depend", "score": 3.0625 } ] } }, "winnerTextsByMode": { "exact": "Help", "f32_forward": "Help", "f32_reverse": "Help", "f32_tree64": "Help", "f16_forward": "Help" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Help" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "encryption-true-false-oneword-c", "text": "No explanation. One word only.\nEncrypting user data at rest is required here, true or false? It is", "stableTopK": [ { "rank": 1, "tokenId": 4339, "text": "True", "logit": 12.740204811096191, "prob": 0.9999827146530151 }, { "rank": 2, "tokenId": 3397, "text": "true", "logit": 1.0822808742523193, "prob": 0.000008650086783745792 }, { "rank": 3, "tokenId": 9277, "text": "False", "logit": 0.8941214084625244, "prob": 0.000007166446266637649 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "logit": -2.4531052112579346, "prob": 2.5212841592292534e-7 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "logit": -2.649242877960205, "prob": 2.0722410454254714e-7 }, { "rank": 6, "tokenId": 6288, "text": " True", "logit": -2.9295613765716553, "prob": 1.5656674179354013e-7 }, { "rank": 7, "tokenId": 1018, "text": "**", "logit": -3.1854395866394043, "prob": 1.2121965653477673e-7 }, { "rank": 8, "tokenId": 1847, "text": " true", "logit": -4.837864875793457, "prob": 2.3223828193863483e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.601401668082861, "winnerGap": 12.518650191949298, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.601401668082861 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 1.0827514761335628 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": 0.894384849575792 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": -2.4585959275560434 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": -2.6561602302258174 }, { "rank": 6, "tokenId": 6288, "text": " True", "score": -2.9389266220412082 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -3.197492885793488 }, { "rank": 8, "tokenId": 1847, "text": " true", "score": -4.880468997820142 } ] }, "f32_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.601401329040527, "winnerGap": 12.518651247024536, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.601401329040527 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 1.0827500820159912 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": 0.8943843245506287 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": -2.4585938453674316 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": -2.6561620235443115 }, { "rank": 6, "tokenId": 6288, "text": " True", "score": -2.938927412033081 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -3.197493076324463 }, { "rank": 8, "tokenId": 1847, "text": " true", "score": -4.880468845367432 } ] }, "f32_reverse": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.601391792297363, "winnerGap": 12.518643379211426, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.601391792297363 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 1.0827484130859375 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": 0.8943859338760376 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": -2.458594560623169 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": -2.6561596393585205 }, { "rank": 6, "tokenId": 6288, "text": " True", "score": -2.9389264583587646 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -3.19749116897583 }, { "rank": 8, "tokenId": 1847, "text": " true", "score": -4.880471229553223 } ] }, "f32_tree64": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.601402282714844, "winnerGap": 12.518650889396667, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.601402282714844 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 1.0827513933181763 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": 0.8943843841552734 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": -2.4585964679718018 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": -2.6561601161956787 }, { "rank": 6, "tokenId": 6288, "text": " True", "score": -2.938927173614502 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -3.1974925994873047 }, { "rank": 8, "tokenId": 1847, "text": " true", "score": -4.880469799041748 } ] }, "f16_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 12.703125, "winnerGap": 12.2900390625, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 12.703125 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 0.4130859375 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": 0.09222412109375 }, { "rank": 4, "tokenId": 6288, "text": " True", "score": -2.646484375 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": -3.107421875 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -3.193359375 }, { "rank": 7, "tokenId": 22369, "text": "TRUE", "score": -3.544921875 }, { "rank": 8, "tokenId": 90981, "text": "Often", "score": -3.912109375 } ] } }, "winnerTextsByMode": { "exact": "True", "f32_forward": "True", "f32_reverse": "True", "f32_tree64": "True", "f16_forward": "True" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "True" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "encryption-true-false-choice-oneword-d", "text": "Choose one word only.\nChoose exactly one word: true or false.\nEncrypting user data at rest is required here.", "stableTopK": [ { "rank": 1, "tokenId": 4339, "text": "True", "logit": 15.802857398986816, "prob": 0.8910547494888306 }, { "rank": 2, "tokenId": 3397, "text": "true", "logit": 13.681679725646973, "prob": 0.10682886838912964 }, { "rank": 3, "tokenId": 1018, "text": "**", "logit": 9.56598949432373, "prob": 0.0017428783467039466 }, { "rank": 4, "tokenId": 9277, "text": "False", "logit": 7.884185314178467, "prob": 0.00032424164237454534 }, { "rank": 5, "tokenId": 4530, "text": "false", "logit": 5.179098129272461, "prob": 0.000021680252757505514 }, { "rank": 6, "tokenId": 22369, "text": "TRUE", "logit": 4.108491897583008, "prob": 0.000007432004167640116 }, { "rank": 7, "tokenId": 24852, "text": "Choose", "logit": 2.559168815612793, "prob": 0.0000015784961533427122 }, { "rank": 8, "tokenId": 818, "text": "The", "logit": 2.0678882598876953, "prob": 9.657908321969444e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 17.569582355946622, "winnerGap": 2.7999762677040234, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 17.569582355946622 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 14.769606088242599 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 9.911537984722475 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": 8.073613226997207 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": 5.231491779723763 }, { "rank": 6, "tokenId": 22369, "text": "TRUE", "score": 4.134467995761138 }, { "rank": 7, "tokenId": 24852, "text": "Choose", "score": 2.5654024261281734 }, { "rank": 8, "tokenId": 818, "text": "The", "score": 2.0711747316107534 } ] }, "f32_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 17.56959342956543, "winnerGap": 2.799999237060547, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 17.56959342956543 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 14.769594192504883 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 9.911542892456055 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": 8.073609352111816 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": 5.2314982414245605 }, { "rank": 6, "tokenId": 22369, "text": "TRUE", "score": 4.134467124938965 }, { "rank": 7, "tokenId": 24852, "text": "Choose", "score": 2.5654051303863525 }, { "rank": 8, "tokenId": 818, "text": "The", "score": 2.0711684226989746 } ] }, "f32_reverse": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 17.569589614868164, "winnerGap": 2.7999963760375977, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 17.569589614868164 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 14.769593238830566 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 9.911530494689941 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": 8.073614120483398 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": 5.2314958572387695 }, { "rank": 6, "tokenId": 22369, "text": "TRUE", "score": 4.134469509124756 }, { "rank": 7, "tokenId": 24852, "text": "Choose", "score": 2.5654027462005615 }, { "rank": 8, "tokenId": 818, "text": "The", "score": 2.0711746215820312 } ] }, "f32_tree64": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 17.569581985473633, "winnerGap": 2.7999753952026367, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 17.569581985473633 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 14.769606590270996 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 9.911539077758789 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": 8.073612213134766 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": 5.231491565704346 }, { "rank": 6, "tokenId": 22369, "text": "TRUE", "score": 4.134468078613281 }, { "rank": 7, "tokenId": 24852, "text": "Choose", "score": 2.565401554107666 }, { "rank": 8, "tokenId": 818, "text": "The", "score": 2.071173906326294 } ] }, "f16_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 17.578125, "winnerGap": 6.7890625, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 17.578125 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 10.7890625 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 9.9296875 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": 5.26953125 }, { "rank": 5, "tokenId": 9277, "text": "False", "score": 4.17578125 }, { "rank": 6, "tokenId": 22369, "text": "TRUE", "score": 4.09375 }, { "rank": 7, "tokenId": 24852, "text": "Choose", "score": 2.666015625 }, { "rank": 8, "tokenId": 3292, "text": "$\\", "score": 1.89453125 } ] } }, "winnerTextsByMode": { "exact": "True", "f32_forward": "True", "f32_reverse": "True", "f32_tree64": "True", "f16_forward": "True" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "True" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "eth-ai-sentencing-oneword-a", "text": "Answer with exactly one word only.\nAn AI deciding prison sentences is efficient or inhumane. It is", "stableTopK": [ { "rank": 1, "tokenId": 902, "text": "In", "logit": 13.102149963378906, "prob": 0.25552165508270264 }, { "rank": 2, "tokenId": 174890, "text": "Efficient", "logit": 12.984868049621582, "prob": 0.22724422812461853 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "logit": 12.814410209655762, "prob": 0.19163019955158234 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "logit": 11.867415428161621, "prob": 0.07433431595563889 }, { "rank": 5, "tokenId": 181023, "text": "Ine", "logit": 11.733428955078125, "prob": 0.06501293182373047 }, { "rank": 6, "tokenId": 21009, "text": "Problem", "logit": 11.64990520477295, "prob": 0.059803396463394165 }, { "rank": 7, "tokenId": 31728, "text": "Eth", "logit": 11.311952590942383, "prob": 0.04265352338552475 }, { "rank": 8, "tokenId": 171917, "text": "Efficiency", "logit": 10.665743827819824, "prob": 0.022351671010255814 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 902, "winnerText": "In", "winnerScore": 14.045795649519375, "winnerGap": 0.14461981254115308, "top": [ { "rank": 1, "tokenId": 902, "text": "In", "score": 14.045795649519375 }, { "rank": 2, "tokenId": 174890, "text": "Efficient", "score": 13.901175836978222 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "score": 13.692051911849203 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 12.551958840555505 }, { "rank": 5, "tokenId": 181023, "text": "Ine", "score": 12.393443973773431 }, { "rank": 6, "tokenId": 21009, "text": "Problem", "score": 12.29496342795022 }, { "rank": 7, "tokenId": 31728, "text": "Eth", "score": 11.899010501562339 }, { "rank": 8, "tokenId": 171917, "text": "Efficiency", "score": 11.152612212301058 } ] }, "f32_forward": { "winnerTokenId": 902, "winnerText": "In", "winnerScore": 14.045804977416992, "winnerGap": 0.14461994171142578, "top": [ { "rank": 1, "tokenId": 902, "text": "In", "score": 14.045804977416992 }, { "rank": 2, "tokenId": 174890, "text": "Efficient", "score": 13.901185035705566 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "score": 13.692049980163574 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 12.551961898803711 }, { "rank": 5, "tokenId": 181023, "text": "Ine", "score": 12.393447875976562 }, { "rank": 6, "tokenId": 21009, "text": "Problem", "score": 12.294963836669922 }, { "rank": 7, "tokenId": 31728, "text": "Eth", "score": 11.899006843566895 }, { "rank": 8, "tokenId": 171917, "text": "Efficiency", "score": 11.152619361877441 } ] }, "f32_reverse": { "winnerTokenId": 902, "winnerText": "In", "winnerScore": 14.045791625976562, "winnerGap": 0.1446094512939453, "top": [ { "rank": 1, "tokenId": 902, "text": "In", "score": 14.045791625976562 }, { "rank": 2, "tokenId": 174890, "text": "Efficient", "score": 13.901182174682617 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "score": 13.692046165466309 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 12.551953315734863 }, { "rank": 5, "tokenId": 181023, "text": "Ine", "score": 12.393448829650879 }, { "rank": 6, "tokenId": 21009, "text": "Problem", "score": 12.294963836669922 }, { "rank": 7, "tokenId": 31728, "text": "Eth", "score": 11.899007797241211 }, { "rank": 8, "tokenId": 171917, "text": "Efficiency", "score": 11.152610778808594 } ] }, "f32_tree64": { "winnerTokenId": 902, "winnerText": "In", "winnerScore": 14.045795440673828, "winnerGap": 0.14461898803710938, "top": [ { "rank": 1, "tokenId": 902, "text": "In", "score": 14.045795440673828 }, { "rank": 2, "tokenId": 174890, "text": "Efficient", "score": 13.901176452636719 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "score": 13.692052841186523 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 12.551959037780762 }, { "rank": 5, "tokenId": 181023, "text": "Ine", "score": 12.393444061279297 }, { "rank": 6, "tokenId": 21009, "text": "Problem", "score": 12.294963836669922 }, { "rank": 7, "tokenId": 31728, "text": "Eth", "score": 11.899009704589844 }, { "rank": 8, "tokenId": 171917, "text": "Efficiency", "score": 11.152612686157227 } ] }, "f16_forward": { "winnerTokenId": 902, "winnerText": "In", "winnerScore": 14.15625, "winnerGap": 0.1640625, "top": [ { "rank": 1, "tokenId": 902, "text": "In", "score": 14.15625 }, { "rank": 2, "tokenId": 174890, "text": "Efficient", "score": 13.9921875 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "score": 13.5625 }, { "rank": 4, "tokenId": 181023, "text": "Ine", "score": 12.421875 }, { "rank": 5, "tokenId": 31728, "text": "Eth", "score": 11.984375 }, { "rank": 6, "tokenId": 171917, "text": "Efficiency", "score": 11.109375 }, { "rank": 7, "tokenId": 31900, "text": "Deb", "score": 10.6015625 }, { "rank": 8, "tokenId": 6700, "text": "Am", "score": 9.390625 } ] } }, "winnerTextsByMode": { "exact": "In", "f32_forward": "In", "f32_reverse": "In", "f32_tree64": "In", "f16_forward": "In" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "In" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "eth-animal-testing-oneword-b", "text": "Respond with one word only.\nAnimal testing for life-saving medicine is justified or unjustified. It is", "stableTopK": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "logit": 11.373291015625, "prob": 0.964098334312439 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "logit": 7.2728400230407715, "prob": 0.015970485284924507 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "logit": 6.775478363037109, "prob": 0.009712178260087967 }, { "rank": 4, "tokenId": 3637, "text": "Context", "logit": 5.825756549835205, "prob": 0.0037571429274976254 }, { "rank": 5, "tokenId": 11896, "text": "Just", "logit": 5.01802396774292, "prob": 0.0016751893563196063 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "logit": 4.862195014953613, "prob": 0.0014334688894450665 }, { "rank": 7, "tokenId": 21288, "text": "Subject", "logit": 4.5813398361206055, "prob": 0.0010824664495885372 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "logit": 4.27150821685791, "prob": 0.0007940653595142066 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 11.97058506654281, "winnerGap": 4.550024041042505, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 11.97058506654281 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 7.420561025500305 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": 6.894338294377681 }, { "rank": 4, "tokenId": 3637, "text": "Context", "score": 5.900692401914036 }, { "rank": 5, "tokenId": 11896, "text": "Just", "score": 5.065624689175554 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "score": 4.905457818060679 }, { "rank": 7, "tokenId": 21288, "text": "Subject", "score": 4.617458577734094 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 4.300729168104851 } ] }, "f32_forward": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 11.970592498779297, "winnerGap": 4.550024509429932, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 11.970592498779297 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 7.420567989349365 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": 6.894347190856934 }, { "rank": 4, "tokenId": 3637, "text": "Context", "score": 5.900696754455566 }, { "rank": 5, "tokenId": 11896, "text": "Just", "score": 5.0656352043151855 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "score": 4.905467510223389 }, { "rank": 7, "tokenId": 21288, "text": "Subject", "score": 4.617461204528809 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 4.30072546005249 } ] }, "f32_reverse": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 11.970599174499512, "winnerGap": 4.550039291381836, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 11.970599174499512 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 7.420559883117676 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": 6.894336700439453 }, { "rank": 4, "tokenId": 3637, "text": "Context", "score": 5.900691986083984 }, { "rank": 5, "tokenId": 11896, "text": "Just", "score": 5.0656232833862305 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "score": 4.905453205108643 }, { "rank": 7, "tokenId": 21288, "text": "Subject", "score": 4.6174540519714355 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 4.3007283210754395 } ] }, "f32_tree64": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 11.970584869384766, "winnerGap": 4.550023078918457, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 11.970584869384766 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 7.420561790466309 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": 6.894339561462402 }, { "rank": 4, "tokenId": 3637, "text": "Context", "score": 5.900691986083984 }, { "rank": 5, "tokenId": 11896, "text": "Just", "score": 5.065623760223389 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "score": 4.905458450317383 }, { "rank": 7, "tokenId": 21288, "text": "Subject", "score": 4.617458343505859 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 4.300728797912598 } ] }, "f16_forward": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 11.984375, "winnerGap": 4.5234375, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 11.984375 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 7.4609375 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": 7.0546875 }, { "rank": 4, "tokenId": 11896, "text": "Just", "score": 5.15234375 }, { "rank": 5, "tokenId": 1584, "text": "Cont", "score": 4.890625 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 4.59765625 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "score": 4.38671875 }, { "rank": 8, "tokenId": 236847, "text": "V", "score": 2.63671875 } ] } }, "winnerTextsByMode": { "exact": "Deb", "f32_forward": "Deb", "f32_reverse": "Deb", "f32_tree64": "Deb", "f16_forward": "Deb" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Deb" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "eth-clone-organ-oneword-c", "text": "No explanation. One word only.\nCloning a human to harvest organs for a dying child is moral or immoral. It is", "stableTopK": [ { "rank": 1, "tokenId": 1514, "text": "Im", "logit": 7.741856575012207, "prob": 0.8037530183792114 }, { "rank": 2, "tokenId": 2035, "text": "Tr", "logit": 5.090350151062012, "prob": 0.05670064687728882 }, { "rank": 3, "tokenId": 31728, "text": "Eth", "logit": 5.056433200836182, "prob": 0.05480978265404701 }, { "rank": 4, "tokenId": 6700, "text": "Am", "logit": 4.063899517059326, "prob": 0.020314501598477364 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "logit": 3.9687917232513428, "prob": 0.01847146637737751 }, { "rank": 6, "tokenId": 154745, "text": "Impossible", "logit": 3.4364147186279297, "prob": 0.010846584104001522 }, { "rank": 7, "tokenId": 27505, "text": "Mor", "logit": 3.149691343307495, "prob": 0.008142740465700626 }, { "rank": 8, "tokenId": 37701, "text": "Ter", "logit": 2.7183785438537598, "prob": 0.005289977416396141 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 1514, "winnerText": "Im", "winnerScore": 7.920929853821818, "winnerGap": 2.7808663312435513, "top": [ { "rank": 1, "tokenId": 1514, "text": "Im", "score": 7.920929853821818 }, { "rank": 2, "tokenId": 2035, "text": "Tr", "score": 5.1400635225782665 }, { "rank": 3, "tokenId": 31728, "text": "Eth", "score": 5.10514737421321 }, { "rank": 4, "tokenId": 6700, "text": "Am", "score": 4.089036165639463 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 3.9921901131140913 }, { "rank": 6, "tokenId": 154745, "text": "Impossible", "score": 3.451562306269794 }, { "rank": 7, "tokenId": 27505, "text": "Mor", "score": 3.1613401327084034 }, { "rank": 8, "tokenId": 37701, "text": "Ter", "score": 2.7258529552150037 } ] }, "f32_forward": { "winnerTokenId": 1514, "winnerText": "Im", "winnerScore": 7.92093563079834, "winnerGap": 2.7808728218078613, "top": [ { "rank": 1, "tokenId": 1514, "text": "Im", "score": 7.92093563079834 }, { "rank": 2, "tokenId": 2035, "text": "Tr", "score": 5.1400628089904785 }, { "rank": 3, "tokenId": 31728, "text": "Eth", "score": 5.1051554679870605 }, { "rank": 4, "tokenId": 6700, "text": "Am", "score": 4.089031219482422 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 3.992194175720215 }, { "rank": 6, "tokenId": 154745, "text": "Impossible", "score": 3.4515631198883057 }, { "rank": 7, "tokenId": 27505, "text": "Mor", "score": 3.1613314151763916 }, { "rank": 8, "tokenId": 37701, "text": "Ter", "score": 2.725849151611328 } ] }, "f32_reverse": { "winnerTokenId": 1514, "winnerText": "Im", "winnerScore": 7.920935153961182, "winnerGap": 2.780877113342285, "top": [ { "rank": 1, "tokenId": 1514, "text": "Im", "score": 7.920935153961182 }, { "rank": 2, "tokenId": 2035, "text": "Tr", "score": 5.1400580406188965 }, { "rank": 3, "tokenId": 31728, "text": "Eth", "score": 5.105148792266846 }, { "rank": 4, "tokenId": 6700, "text": "Am", "score": 4.089035511016846 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 3.9921860694885254 }, { "rank": 6, "tokenId": 154745, "text": "Impossible", "score": 3.45156192779541 }, { "rank": 7, "tokenId": 27505, "text": "Mor", "score": 3.161336898803711 }, { "rank": 8, "tokenId": 37701, "text": "Ter", "score": 2.72584867477417 } ] }, "f32_tree64": { "winnerTokenId": 1514, "winnerText": "Im", "winnerScore": 7.920929908752441, "winnerGap": 2.7808666229248047, "top": [ { "rank": 1, "tokenId": 1514, "text": "Im", "score": 7.920929908752441 }, { "rank": 2, "tokenId": 2035, "text": "Tr", "score": 5.140063285827637 }, { "rank": 3, "tokenId": 31728, "text": "Eth", "score": 5.105147361755371 }, { "rank": 4, "tokenId": 6700, "text": "Am", "score": 4.08903694152832 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 3.9921913146972656 }, { "rank": 6, "tokenId": 154745, "text": "Impossible", "score": 3.4515621662139893 }, { "rank": 7, "tokenId": 27505, "text": "Mor", "score": 3.16133975982666 }, { "rank": 8, "tokenId": 37701, "text": "Ter", "score": 2.72585391998291 } ] }, "f16_forward": { "winnerTokenId": 31728, "winnerText": "Eth", "winnerScore": 5.11328125, "winnerGap": 0.9140625, "top": [ { "rank": 1, "tokenId": 31728, "text": "Eth", "score": 5.11328125 }, { "rank": 2, "tokenId": 6700, "text": "Am", "score": 4.19921875 }, { "rank": 3, "tokenId": 2035, "text": "Tr", "score": 4.171875 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 4.10546875 }, { "rank": 5, "tokenId": 1514, "text": "Im", "score": 3.994140625 }, { "rank": 6, "tokenId": 154745, "text": "Impossible", "score": 3.4921875 }, { "rank": 7, "tokenId": 27505, "text": "Mor", "score": 3.203125 }, { "rank": 8, "tokenId": 2805, "text": "Un", "score": 2.203125 } ] } }, "winnerTextsByMode": { "exact": "Im", "f32_forward": "Im", "f32_reverse": "Im", "f32_tree64": "Im", "f16_forward": "Eth" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Im", "Eth" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 1514, 120854 ], "decodedText": "Immoral", "steps": [ { "step": 0, "tokenId": 1514, "text": "Im" }, { "step": 1, "tokenId": 120854, "text": "moral" } ] }, "f16_forward": { "tokenIds": [ 31728, 800 ], "decodedText": "Ethical", "steps": [ { "step": 0, "tokenId": 31728, "text": "Eth" }, { "step": 1, "tokenId": 800, "text": "ical" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "eth-deepfake-expose-oneword-d", "text": "Choose one word only.\nUsing a deepfake to expose a real crime is justified or unjustified. It is", "stableTopK": [ { "rank": 1, "tokenId": 1584, "text": "Cont", "logit": 10.883344650268555, "prob": 0.2750990390777588 }, { "rank": 2, "tokenId": 11896, "text": "Just", "logit": 10.528076171875, "prob": 0.1928403526544571 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "logit": 10.492753028869629, "prob": 0.1861475259065628 }, { "rank": 4, "tokenId": 1018, "text": "**", "logit": 10.187446594238281, "prob": 0.1371716558933258 }, { "rank": 5, "tokenId": 2805, "text": "Un", "logit": 9.690617561340332, "prob": 0.08346305787563324 }, { "rank": 6, "tokenId": 55144, "text": "Amb", "logit": 8.413098335266113, "prob": 0.023263484239578247 }, { "rank": 7, "tokenId": 31900, "text": "Deb", "logit": 8.302770614624023, "prob": 0.020833391696214676 }, { "rank": 8, "tokenId": 191239, "text": "Difficult", "logit": 7.819431304931641, "prob": 0.012848380953073502 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 1584, "winnerText": "Cont", "winnerScore": 11.402444174303588, "winnerGap": 0.4071216679668357, "top": [ { "rank": 1, "tokenId": 1584, "text": "Cont", "score": 11.402444174303588 }, { "rank": 2, "tokenId": 11896, "text": "Just", "score": 10.995322506336752 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": 10.955054651469132 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 10.608588070771319 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": 10.050480524338408 }, { "rank": 6, "tokenId": 55144, "text": "Amb", "score": 8.644667504578791 }, { "rank": 7, "tokenId": 31900, "text": "Deb", "score": 8.525062509586515 }, { "rank": 8, "tokenId": 191239, "text": "Difficult", "score": 8.004100419270955 } ] }, "f32_forward": { "winnerTokenId": 1584, "winnerText": "Cont", "winnerScore": 11.402433395385742, "winnerGap": 0.4071035385131836, "top": [ { "rank": 1, "tokenId": 1584, "text": "Cont", "score": 11.402433395385742 }, { "rank": 2, "tokenId": 11896, "text": "Just", "score": 10.995329856872559 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": 10.955058097839355 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 10.608585357666016 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": 10.050482749938965 }, { "rank": 6, "tokenId": 55144, "text": "Amb", "score": 8.644672393798828 }, { "rank": 7, "tokenId": 31900, "text": "Deb", "score": 8.525063514709473 }, { "rank": 8, "tokenId": 191239, "text": "Difficult", "score": 8.004094123840332 } ] }, "f32_reverse": { "winnerTokenId": 1584, "winnerText": "Cont", "winnerScore": 11.402438163757324, "winnerGap": 0.40711498260498047, "top": [ { "rank": 1, "tokenId": 1584, "text": "Cont", "score": 11.402438163757324 }, { "rank": 2, "tokenId": 11896, "text": "Just", "score": 10.995323181152344 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": 10.955050468444824 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 10.60859203338623 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": 10.050482749938965 }, { "rank": 6, "tokenId": 55144, "text": "Amb", "score": 8.644665718078613 }, { "rank": 7, "tokenId": 31900, "text": "Deb", "score": 8.52506160736084 }, { "rank": 8, "tokenId": 191239, "text": "Difficult", "score": 8.0040922164917 } ] }, "f32_tree64": { "winnerTokenId": 1584, "winnerText": "Cont", "winnerScore": 11.402444839477539, "winnerGap": 0.4071216583251953, "top": [ { "rank": 1, "tokenId": 1584, "text": "Cont", "score": 11.402444839477539 }, { "rank": 2, "tokenId": 11896, "text": "Just", "score": 10.995323181152344 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": 10.955053329467773 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 10.608587265014648 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": 10.050481796264648 }, { "rank": 6, "tokenId": 55144, "text": "Amb", "score": 8.644667625427246 }, { "rank": 7, "tokenId": 31900, "text": "Deb", "score": 8.52506160736084 }, { "rank": 8, "tokenId": 191239, "text": "Difficult", "score": 8.004100799560547 } ] }, "f16_forward": { "winnerTokenId": 1584, "winnerText": "Cont", "winnerScore": 11.4609375, "winnerGap": 0.5, "top": [ { "rank": 1, "tokenId": 1584, "text": "Cont", "score": 11.4609375 }, { "rank": 2, "tokenId": 11896, "text": "Just", "score": 10.9609375 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 10.703125 }, { "rank": 4, "tokenId": 2805, "text": "Un", "score": 10.078125 }, { "rank": 5, "tokenId": 55144, "text": "Amb", "score": 8.5625 }, { "rank": 6, "tokenId": 191239, "text": "Difficult", "score": 7.94921875 }, { "rank": 7, "tokenId": 21288, "text": "Subject", "score": 7.94140625 }, { "rank": 8, "tokenId": 31728, "text": "Eth", "score": 7.8046875 } ] } }, "winnerTextsByMode": { "exact": "Cont", "f32_forward": "Cont", "f32_reverse": "Cont", "f32_tree64": "Cont", "f16_forward": "Cont" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Cont" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "eth-draft-war-oneword-a", "text": "Answer with exactly one word only.\nA military draft in wartime is necessary or oppressive. It is", "stableTopK": [ { "rank": 1, "tokenId": 229224, "text": "Necessary", "logit": 20.92656135559082, "prob": 0.9978424310684204 }, { "rank": 2, "tokenId": 173959, "text": "Necess", "logit": 14.46275520324707, "prob": 0.0015554878627881408 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "logit": 12.097281455993652, "prob": 0.00014606777403969318 }, { "rank": 4, "tokenId": 22089, "text": "necessary", "logit": 11.717829704284668, "prob": 0.0000999448966467753 }, { "rank": 5, "tokenId": 55144, "text": "Amb", "logit": 11.577427864074707, "prob": 0.00008685301872901618 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "logit": 11.136994361877441, "prob": 0.000055912259995238855 }, { "rank": 7, "tokenId": 3637, "text": "Context", "logit": 10.730281829833984, "prob": 0.000037228368455544114 }, { "rank": 8, "tokenId": 236796, "text": "D", "logit": 10.442967414855957, "prob": 0.000027931546355830505 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 229224, "winnerText": "Necessary", "winnerScore": 25.875501396415928, "winnerGap": 10.10432154550723, "top": [ { "rank": 1, "tokenId": 229224, "text": "Necessary", "score": 25.875501396415928 }, { "rank": 2, "tokenId": 173959, "text": "Necess", "score": 15.771179850908698 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": 12.825459818881527 }, { "rank": 4, "tokenId": 22089, "text": "necessary", "score": 12.37503709948092 }, { "rank": 5, "tokenId": 55144, "text": "Amb", "score": 12.209707689665159 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "score": 11.695571202742293 }, { "rank": 7, "tokenId": 3637, "text": "Context", "score": 11.226551658884393 }, { "rank": 8, "tokenId": 236796, "text": "D", "score": 10.898371920340745 } ] }, "f32_forward": { "winnerTokenId": 229224, "winnerText": "Necessary", "winnerScore": 25.875516891479492, "winnerGap": 10.104339599609375, "top": [ { "rank": 1, "tokenId": 229224, "text": "Necessary", "score": 25.875516891479492 }, { "rank": 2, "tokenId": 173959, "text": "Necess", "score": 15.771177291870117 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": 12.825457572937012 }, { "rank": 4, "tokenId": 22089, "text": "necessary", "score": 12.375033378601074 }, { "rank": 5, "tokenId": 55144, "text": "Amb", "score": 12.209693908691406 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "score": 11.695564270019531 }, { "rank": 7, "tokenId": 3637, "text": "Context", "score": 11.226555824279785 }, { "rank": 8, "tokenId": 236796, "text": "D", "score": 10.898372650146484 } ] }, "f32_reverse": { "winnerTokenId": 229224, "winnerText": "Necessary", "winnerScore": 25.875507354736328, "winnerGap": 10.104330062866211, "top": [ { "rank": 1, "tokenId": 229224, "text": "Necessary", "score": 25.875507354736328 }, { "rank": 2, "tokenId": 173959, "text": "Necess", "score": 15.771177291870117 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": 12.82546615600586 }, { "rank": 4, "tokenId": 22089, "text": "necessary", "score": 12.375028610229492 }, { "rank": 5, "tokenId": 55144, "text": "Amb", "score": 12.209699630737305 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "score": 11.695577621459961 }, { "rank": 7, "tokenId": 3637, "text": "Context", "score": 11.226552963256836 }, { "rank": 8, "tokenId": 236796, "text": "D", "score": 10.898381233215332 } ] }, "f32_tree64": { "winnerTokenId": 229224, "winnerText": "Necessary", "winnerScore": 25.875499725341797, "winnerGap": 10.104320526123047, "top": [ { "rank": 1, "tokenId": 229224, "text": "Necessary", "score": 25.875499725341797 }, { "rank": 2, "tokenId": 173959, "text": "Necess", "score": 15.77117919921875 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": 12.825458526611328 }, { "rank": 4, "tokenId": 22089, "text": "necessary", "score": 12.375038146972656 }, { "rank": 5, "tokenId": 55144, "text": "Amb", "score": 12.209708213806152 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "score": 11.695570945739746 }, { "rank": 7, "tokenId": 3637, "text": "Context", "score": 11.22655200958252 }, { "rank": 8, "tokenId": 236796, "text": "D", "score": 10.898370742797852 } ] }, "f16_forward": { "winnerTokenId": 229224, "winnerText": "Necessary", "winnerScore": 25.921875, "winnerGap": 10.125, "top": [ { "rank": 1, "tokenId": 229224, "text": "Necessary", "score": 25.921875 }, { "rank": 2, "tokenId": 173959, "text": "Necess", "score": 15.796875 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": 12.8984375 }, { "rank": 4, "tokenId": 22089, "text": "necessary", "score": 12.453125 }, { "rank": 5, "tokenId": 1584, "text": "Cont", "score": 11.75 }, { "rank": 6, "tokenId": 3637, "text": "Context", "score": 10.1875 }, { "rank": 7, "tokenId": 21009, "text": "Problem", "score": 9.5 }, { "rank": 8, "tokenId": 84551, "text": "Situ", "score": 9.453125 } ] } }, "winnerTextsByMode": { "exact": "Necessary", "f32_forward": "Necessary", "f32_reverse": "Necessary", "f32_tree64": "Necessary", "f16_forward": "Necessary" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Necessary" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "eth-forgive-unforgivable-oneword-b", "text": "Respond with one word only.\nSome acts are forgivable or unforgivable. They are", "stableTopK": [ { "rank": 1, "tokenId": 21288, "text": "Subject", "logit": 9.64526653289795, "prob": 0.3928762972354889 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "logit": 8.995979309082031, "prob": 0.20524565875530243 }, { "rank": 3, "tokenId": 192034, "text": "Judgment", "logit": 8.031085014343262, "prob": 0.07820340991020203 }, { "rank": 4, "tokenId": 3637, "text": "Context", "logit": 8.019686698913574, "prob": 0.07731708884239197 }, { "rank": 5, "tokenId": 19204, "text": "Variable", "logit": 7.586677551269531, "prob": 0.05014435201883316 }, { "rank": 6, "tokenId": 31900, "text": "Deb", "logit": 7.21510648727417, "prob": 0.03458205237984657 }, { "rank": 7, "tokenId": 148823, "text": "Moral", "logit": 6.9124531745910645, "prob": 0.02555112913250923 }, { "rank": 8, "tokenId": 236847, "text": "V", "logit": 6.717187881469727, "prob": 0.021018777042627335 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 21288, "winnerText": "Subject", "winnerScore": 9.999868978296638, "winnerGap": 0.7186980535147551, "top": [ { "rank": 1, "tokenId": 21288, "text": "Subject", "score": 9.999868978296638 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 9.281170924781883 }, { "rank": 3, "tokenId": 192034, "text": "Judgment", "score": 8.231627247414298 }, { "rank": 4, "tokenId": 3637, "text": "Context", "score": 8.21935606170109 }, { "rank": 5, "tokenId": 19204, "text": "Variable", "score": 7.75491099167116 }, { "rank": 6, "tokenId": 31900, "text": "Deb", "score": 7.3592558082875 }, { "rank": 7, "tokenId": 148823, "text": "Moral", "score": 7.038841151279421 }, { "rank": 8, "tokenId": 236847, "text": "V", "score": 6.832941296189475 } ] }, "f32_forward": { "winnerTokenId": 21288, "winnerText": "Subject", "winnerScore": 9.999862670898438, "winnerGap": 0.7187023162841797, "top": [ { "rank": 1, "tokenId": 21288, "text": "Subject", "score": 9.999862670898438 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 9.281160354614258 }, { "rank": 3, "tokenId": 192034, "text": "Judgment", "score": 8.231621742248535 }, { "rank": 4, "tokenId": 3637, "text": "Context", "score": 8.219358444213867 }, { "rank": 5, "tokenId": 19204, "text": "Variable", "score": 7.754919052124023 }, { "rank": 6, "tokenId": 31900, "text": "Deb", "score": 7.359259605407715 }, { "rank": 7, "tokenId": 148823, "text": "Moral", "score": 7.038835048675537 }, { "rank": 8, "tokenId": 236847, "text": "V", "score": 6.832944869995117 } ] }, "f32_reverse": { "winnerTokenId": 21288, "winnerText": "Subject", "winnerScore": 9.999872207641602, "winnerGap": 0.7187080383300781, "top": [ { "rank": 1, "tokenId": 21288, "text": "Subject", "score": 9.999872207641602 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 9.281164169311523 }, { "rank": 3, "tokenId": 192034, "text": "Judgment", "score": 8.23162841796875 }, { "rank": 4, "tokenId": 3637, "text": "Context", "score": 8.219350814819336 }, { "rank": 5, "tokenId": 19204, "text": "Variable", "score": 7.754911422729492 }, { "rank": 6, "tokenId": 31900, "text": "Deb", "score": 7.359255313873291 }, { "rank": 7, "tokenId": 148823, "text": "Moral", "score": 7.038842678070068 }, { "rank": 8, "tokenId": 236847, "text": "V", "score": 6.832937717437744 } ] }, "f32_tree64": { "winnerTokenId": 21288, "winnerText": "Subject", "winnerScore": 9.999869346618652, "winnerGap": 0.7186975479125977, "top": [ { "rank": 1, "tokenId": 21288, "text": "Subject", "score": 9.999869346618652 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 9.281171798706055 }, { "rank": 3, "tokenId": 192034, "text": "Judgment", "score": 8.23162841796875 }, { "rank": 4, "tokenId": 3637, "text": "Context", "score": 8.219355583190918 }, { "rank": 5, "tokenId": 19204, "text": "Variable", "score": 7.754910469055176 }, { "rank": 6, "tokenId": 31900, "text": "Deb", "score": 7.359255790710449 }, { "rank": 7, "tokenId": 148823, "text": "Moral", "score": 7.03884220123291 }, { "rank": 8, "tokenId": 236847, "text": "V", "score": 6.832942008972168 } ] }, "f16_forward": { "winnerTokenId": 48333, "winnerText": "Complex", "winnerScore": 9.3046875, "winnerGap": 1.09375, "top": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "score": 9.3046875 }, { "rank": 2, "tokenId": 3637, "text": "Context", "score": 8.2109375 }, { "rank": 3, "tokenId": 19204, "text": "Variable", "score": 7.890625 }, { "rank": 4, "tokenId": 192034, "text": "Judgment", "score": 7.25 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": 6.78515625 }, { "rank": 6, "tokenId": 8724, "text": "Dec", "score": 6.72265625 }, { "rank": 7, "tokenId": 27505, "text": "Mor", "score": 6.71875 }, { "rank": 8, "tokenId": 13552, "text": "subject", "score": 6.70703125 } ] } }, "winnerTextsByMode": { "exact": "Subject", "f32_forward": "Subject", "f32_reverse": "Subject", "f32_tree64": "Subject", "f16_forward": "Complex" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Subject", "Complex" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 21288, 705 ], "decodedText": "Subjective", "steps": [ { "step": 0, "tokenId": 21288, "text": "Subject" }, { "step": 1, "tokenId": 705, "text": "ive" } ] }, "f16_forward": { "tokenIds": [ 48333, 106 ], "decodedText": "Complex", "steps": [ { "step": 0, "tokenId": 48333, "text": "Complex" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "eth-genetic-editing-oneword-c", "text": "No explanation. One word only.\nEditing a human embryo's genes to prevent disease is ethical or unethical. It is", "stableTopK": [ { "rank": 1, "tokenId": 31728, "text": "Eth", "logit": 7.683229446411133, "prob": 0.7419682145118713 }, { "rank": 2, "tokenId": 31900, "text": "Deb", "logit": 6.01113224029541, "prob": 0.13938072323799133 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "logit": 5.698544979095459, "prob": 0.10196421295404434 }, { "rank": 4, "tokenId": 236788, "text": "E", "logit": 2.588585376739502, "prob": 0.004547885153442621 }, { "rank": 5, "tokenId": 1584, "text": "Cont", "logit": 2.085376262664795, "prob": 0.00274959416128695 }, { "rank": 6, "tokenId": 201053, "text": "Ethics", "logit": 1.5745705366134644, "prob": 0.0016497892793267965 }, { "rank": 7, "tokenId": 21288, "text": "Subject", "logit": 1.540252685546875, "prob": 0.001594132510945201 }, { "rank": 8, "tokenId": 179306, "text": "Depends", "logit": 1.3247352838516235, "prob": 0.0012850688071921468 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 31728, "winnerText": "Eth", "winnerScore": 7.858150534172523, "winnerGap": 1.7645775754263617, "top": [ { "rank": 1, "tokenId": 31728, "text": "Eth", "score": 7.858150534172523 }, { "rank": 2, "tokenId": 31900, "text": "Deb", "score": 6.093572958746162 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": 5.7686075767891545 }, { "rank": 4, "tokenId": 236788, "text": "E", "score": 2.595040490459902 }, { "rank": 5, "tokenId": 1584, "text": "Cont", "score": 2.088742308710577 }, { "rank": 6, "tokenId": 201053, "text": "Ethics", "score": 1.57601662877671 }, { "rank": 7, "tokenId": 21288, "text": "Subject", "score": 1.541606774874431 }, { "rank": 8, "tokenId": 179306, "text": "Depends", "score": 1.3255979032191476 } ] }, "f32_forward": { "winnerTokenId": 31728, "winnerText": "Eth", "winnerScore": 7.858154296875, "winnerGap": 1.764592170715332, "top": [ { "rank": 1, "tokenId": 31728, "text": "Eth", "score": 7.858154296875 }, { "rank": 2, "tokenId": 31900, "text": "Deb", "score": 6.093562126159668 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": 5.768604278564453 }, { "rank": 4, "tokenId": 236788, "text": "E", "score": 2.595040798187256 }, { "rank": 5, "tokenId": 1584, "text": "Cont", "score": 2.0887441635131836 }, { "rank": 6, "tokenId": 201053, "text": "Ethics", "score": 1.5760183334350586 }, { "rank": 7, "tokenId": 21288, "text": "Subject", "score": 1.5416032075881958 }, { "rank": 8, "tokenId": 179306, "text": "Depends", "score": 1.3255984783172607 } ] }, "f32_reverse": { "winnerTokenId": 31728, "winnerText": "Eth", "winnerScore": 7.858154773712158, "winnerGap": 1.7645835876464844, "top": [ { "rank": 1, "tokenId": 31728, "text": "Eth", "score": 7.858154773712158 }, { "rank": 2, "tokenId": 31900, "text": "Deb", "score": 6.093571186065674 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": 5.76860237121582 }, { "rank": 4, "tokenId": 236788, "text": "E", "score": 2.595041036605835 }, { "rank": 5, "tokenId": 1584, "text": "Cont", "score": 2.0887441635131836 }, { "rank": 6, "tokenId": 201053, "text": "Ethics", "score": 1.5760152339935303 }, { "rank": 7, "tokenId": 21288, "text": "Subject", "score": 1.541604995727539 }, { "rank": 8, "tokenId": 179306, "text": "Depends", "score": 1.3255972862243652 } ] }, "f32_tree64": { "winnerTokenId": 31728, "winnerText": "Eth", "winnerScore": 7.858150482177734, "winnerGap": 1.7645773887634277, "top": [ { "rank": 1, "tokenId": 31728, "text": "Eth", "score": 7.858150482177734 }, { "rank": 2, "tokenId": 31900, "text": "Deb", "score": 6.093573093414307 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": 5.7686076164245605 }, { "rank": 4, "tokenId": 236788, "text": "E", "score": 2.595041275024414 }, { "rank": 5, "tokenId": 1584, "text": "Cont", "score": 2.0887434482574463 }, { "rank": 6, "tokenId": 201053, "text": "Ethics", "score": 1.576016902923584 }, { "rank": 7, "tokenId": 21288, "text": "Subject", "score": 1.54160737991333 }, { "rank": 8, "tokenId": 179306, "text": "Depends", "score": 1.3255977630615234 } ] }, "f16_forward": { "winnerTokenId": 31728, "winnerText": "Eth", "winnerScore": 7.8984375, "winnerGap": 1.76953125, "top": [ { "rank": 1, "tokenId": 31728, "text": "Eth", "score": 7.8984375 }, { "rank": 2, "tokenId": 31900, "text": "Deb", "score": 6.12890625 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": 5.82421875 }, { "rank": 4, "tokenId": 1584, "text": "Cont", "score": 2.099609375 }, { "rank": 5, "tokenId": 236788, "text": "E", "score": 1.5830078125 }, { "rank": 6, "tokenId": 201053, "text": "Ethics", "score": 1.3837890625 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": 1.3779296875 }, { "rank": 8, "tokenId": 46065, "text": "Bio", "score": 0.8720703125 } ] } }, "winnerTextsByMode": { "exact": "Eth", "f32_forward": "Eth", "f32_reverse": "Eth", "f32_tree64": "Eth", "f16_forward": "Eth" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Eth" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "eth-lie-dying-oneword-d", "text": "Choose one word only.\nLying to a dying patient about their prognosis is compassionate or dishonest. It is", "stableTopK": [ { "rank": 1, "tokenId": 197628, "text": "Dish", "logit": 16.93889617919922, "prob": 0.8589968085289001 }, { "rank": 2, "tokenId": 41776, "text": "dish", "logit": 15.0264310836792, "prob": 0.12688741087913513 }, { "rank": 3, "tokenId": 1018, "text": "**", "logit": 12.782743453979492, "prob": 0.013458521105349064 }, { "rank": 4, "tokenId": 3321, "text": "De", "logit": 9.39011001586914, "prob": 0.00045247579691931605 }, { "rank": 5, "tokenId": 4830, "text": "Dis", "logit": 7.123760223388672, "prob": 0.000046917208237573504 }, { "rank": 6, "tokenId": 168154, "text": "Compass", "logit": 6.4616241455078125, "prob": 0.00002419747761450708 }, { "rank": 7, "tokenId": 79151, "text": "Harm", "logit": 6.425700664520264, "prob": 0.000023343649445450865 }, { "rank": 8, "tokenId": 181023, "text": "Ine", "logit": 5.723880767822266, "prob": 0.000011571035429369658 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 197628, "winnerText": "Dish", "winnerScore": 19.18812054283432, "winnerGap": 2.6736725980688476, "top": [ { "rank": 1, "tokenId": 197628, "text": "Dish", "score": 19.18812054283432 }, { "rank": 2, "tokenId": 41776, "text": "dish", "score": 16.51444794476547 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 13.653338333836126 }, { "rank": 4, "tokenId": 3321, "text": "De", "score": 9.716157972925718 }, { "rank": 5, "tokenId": 4830, "text": "Dis", "score": 7.262375380727747 }, { "rank": 6, "tokenId": 168154, "text": "Compass", "score": 6.564425972282173 }, { "rank": 7, "tokenId": 79151, "text": "Harm", "score": 6.526762592297686 }, { "rank": 8, "tokenId": 181023, "text": "Ine", "score": 5.7948897362483685 } ] }, "f32_forward": { "winnerTokenId": 197628, "winnerText": "Dish", "winnerScore": 19.188106536865234, "winnerGap": 2.673656463623047, "top": [ { "rank": 1, "tokenId": 197628, "text": "Dish", "score": 19.188106536865234 }, { "rank": 2, "tokenId": 41776, "text": "dish", "score": 16.514450073242188 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 13.653336524963379 }, { "rank": 4, "tokenId": 3321, "text": "De", "score": 9.716180801391602 }, { "rank": 5, "tokenId": 4830, "text": "Dis", "score": 7.262378692626953 }, { "rank": 6, "tokenId": 168154, "text": "Compass", "score": 6.564417362213135 }, { "rank": 7, "tokenId": 79151, "text": "Harm", "score": 6.526745319366455 }, { "rank": 8, "tokenId": 181023, "text": "Ine", "score": 5.79489278793335 } ] }, "f32_reverse": { "winnerTokenId": 197628, "winnerText": "Dish", "winnerScore": 19.188119888305664, "winnerGap": 2.673666000366211, "top": [ { "rank": 1, "tokenId": 197628, "text": "Dish", "score": 19.188119888305664 }, { "rank": 2, "tokenId": 41776, "text": "dish", "score": 16.514453887939453 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 13.653339385986328 }, { "rank": 4, "tokenId": 3321, "text": "De", "score": 9.716160774230957 }, { "rank": 5, "tokenId": 4830, "text": "Dis", "score": 7.2623724937438965 }, { "rank": 6, "tokenId": 168154, "text": "Compass", "score": 6.564427852630615 }, { "rank": 7, "tokenId": 79151, "text": "Harm", "score": 6.526762962341309 }, { "rank": 8, "tokenId": 181023, "text": "Ine", "score": 5.794891834259033 } ] }, "f32_tree64": { "winnerTokenId": 197628, "winnerText": "Dish", "winnerScore": 19.18811798095703, "winnerGap": 2.6736698150634766, "top": [ { "rank": 1, "tokenId": 197628, "text": "Dish", "score": 19.18811798095703 }, { "rank": 2, "tokenId": 41776, "text": "dish", "score": 16.514448165893555 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 13.653337478637695 }, { "rank": 4, "tokenId": 3321, "text": "De", "score": 9.716157913208008 }, { "rank": 5, "tokenId": 4830, "text": "Dis", "score": 7.262375831604004 }, { "rank": 6, "tokenId": 168154, "text": "Compass", "score": 6.564425468444824 }, { "rank": 7, "tokenId": 79151, "text": "Harm", "score": 6.526764392852783 }, { "rank": 8, "tokenId": 181023, "text": "Ine", "score": 5.794890880584717 } ] }, "f16_forward": { "winnerTokenId": 197628, "winnerText": "Dish", "winnerScore": 19.5, "winnerGap": 2.984375, "top": [ { "rank": 1, "tokenId": 197628, "text": "Dish", "score": 19.5 }, { "rank": 2, "tokenId": 41776, "text": "dish", "score": 16.515625 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 13.578125 }, { "rank": 4, "tokenId": 4830, "text": "Dis", "score": 7.390625 }, { "rank": 5, "tokenId": 79151, "text": "Harm", "score": 6.51953125 }, { "rank": 6, "tokenId": 168154, "text": "Compass", "score": 6.50390625 }, { "rank": 7, "tokenId": 181023, "text": "Ine", "score": 5.8359375 }, { "rank": 8, "tokenId": 236798, "text": "L", "score": 5.62109375 } ] } }, "winnerTextsByMode": { "exact": "Dish", "f32_forward": "Dish", "f32_reverse": "Dish", "f32_tree64": "Dish", "f16_forward": "Dish" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Dish" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "eth-mercy-kill-oneword-a", "text": "Answer with exactly one word only.\nA soldier mercy-killing a mortally wounded comrade is right or wrong. It is", "stableTopK": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "logit": 10.410009384155273, "prob": 0.6664815545082092 }, { "rank": 2, "tokenId": 21288, "text": "Subject", "logit": 8.45700454711914, "prob": 0.09453854709863663 }, { "rank": 3, "tokenId": 3637, "text": "Context", "logit": 8.303954124450684, "prob": 0.08112224191427231 }, { "rank": 4, "tokenId": 31728, "text": "Eth", "logit": 8.02946662902832, "prob": 0.061649780720472336 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "logit": 7.6359124183654785, "prob": 0.0415923185646534 }, { "rank": 6, "tokenId": 55144, "text": "Amb", "logit": 6.6752095222473145, "prob": 0.01591421291232109 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "logit": 5.809935092926025, "prob": 0.006698865443468094 }, { "rank": 8, "tokenId": 62892, "text": "Grey", "logit": 5.79442024230957, "prob": 0.0065957349725067616 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 48333, "winnerText": "Complex", "winnerScore": 10.860881993435072, "winnerGap": 2.168529793754132, "top": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "score": 10.860881993435072 }, { "rank": 2, "tokenId": 21288, "text": "Subject", "score": 8.69235219968094 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 8.52634631344727 }, { "rank": 4, "tokenId": 31728, "text": "Eth", "score": 8.229887445551054 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 7.8075346281526805 }, { "rank": 6, "tokenId": 55144, "text": "Amb", "score": 6.788759288886439 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": 5.8842500393881245 }, { "rank": 8, "tokenId": 62892, "text": "Grey", "score": 5.8681300784503225 } ] }, "f32_forward": { "winnerTokenId": 48333, "winnerText": "Complex", "winnerScore": 10.860899925231934, "winnerGap": 2.168550491333008, "top": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "score": 10.860899925231934 }, { "rank": 2, "tokenId": 21288, "text": "Subject", "score": 8.692349433898926 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 8.526346206665039 }, { "rank": 4, "tokenId": 31728, "text": "Eth", "score": 8.229891777038574 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 7.80753231048584 }, { "rank": 6, "tokenId": 55144, "text": "Amb", "score": 6.788757801055908 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": 5.884251594543457 }, { "rank": 8, "tokenId": 62892, "text": "Grey", "score": 5.868128776550293 } ] }, "f32_reverse": { "winnerTokenId": 48333, "winnerText": "Complex", "winnerScore": 10.860871315002441, "winnerGap": 2.168515205383301, "top": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "score": 10.860871315002441 }, { "rank": 2, "tokenId": 21288, "text": "Subject", "score": 8.69235610961914 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 8.52634334564209 }, { "rank": 4, "tokenId": 31728, "text": "Eth", "score": 8.229884147644043 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 7.8075361251831055 }, { "rank": 6, "tokenId": 55144, "text": "Amb", "score": 6.788759708404541 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": 5.884251117706299 }, { "rank": 8, "tokenId": 62892, "text": "Grey", "score": 5.868126392364502 } ] }, "f32_tree64": { "winnerTokenId": 48333, "winnerText": "Complex", "winnerScore": 10.860881805419922, "winnerGap": 2.168529510498047, "top": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "score": 10.860881805419922 }, { "rank": 2, "tokenId": 21288, "text": "Subject", "score": 8.692352294921875 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 8.526347160339355 }, { "rank": 4, "tokenId": 31728, "text": "Eth", "score": 8.229887962341309 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 7.807535171508789 }, { "rank": 6, "tokenId": 55144, "text": "Amb", "score": 6.788758754730225 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": 5.884250640869141 }, { "rank": 8, "tokenId": 62892, "text": "Grey", "score": 5.868129730224609 } ] }, "f16_forward": { "winnerTokenId": 48333, "winnerText": "Complex", "winnerScore": 10.8203125, "winnerGap": 1.9609375, "top": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "score": 10.8203125 }, { "rank": 2, "tokenId": 21288, "text": "Subject", "score": 8.859375 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 8.5625 }, { "rank": 4, "tokenId": 31728, "text": "Eth", "score": 8.171875 }, { "rank": 5, "tokenId": 55144, "text": "Amb", "score": 6.8515625 }, { "rank": 6, "tokenId": 62892, "text": "Grey", "score": 5.83203125 }, { "rank": 7, "tokenId": 27505, "text": "Mor", "score": 5.734375 }, { "rank": 8, "tokenId": 148823, "text": "Moral", "score": 5.25 } ] } }, "winnerTextsByMode": { "exact": "Complex", "f32_forward": "Complex", "f32_reverse": "Complex", "f32_tree64": "Complex", "f16_forward": "Complex" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Complex" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "eth-parent-spy-oneword-b", "text": "Respond with one word only.\nA parent reading their teenager's private diary is protective or invasive. It is", "stableTopK": [ { "rank": 1, "tokenId": 902, "text": "In", "logit": 6.862846374511719, "prob": 0.9999884963035583 }, { "rank": 2, "tokenId": 83176, "text": "invasive", "logit": -5.179675579071045, "prob": 0.00000588835746384575 }, { "rank": 3, "tokenId": 22186, "text": "Both", "logit": -5.440051555633545, "prob": 0.0000045385204430203885 }, { "rank": 4, "tokenId": 29622, "text": "Inv", "logit": -9.091619491577148, "prob": 1.1777647301869365e-7 }, { "rank": 5, "tokenId": 3898, "text": "Int", "logit": -9.256004333496094, "prob": 9.992336913455802e-8 }, { "rank": 6, "tokenId": 27413, "text": "both", "logit": -9.331114768981934, "prob": 9.269302125858303e-8 }, { "rank": 7, "tokenId": 29335, "text": "Private", "logit": -9.35407829284668, "prob": 9.058872052492006e-8 }, { "rank": 8, "tokenId": 495, "text": "in", "logit": -9.704668045043945, "prob": 6.379915618026644e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 902, "winnerText": "In", "winnerScore": 6.9864656543968735, "winnerGap": 12.218555157770787, "top": [ { "rank": 1, "tokenId": 902, "text": "In", "score": 6.9864656543968735 }, { "rank": 2, "tokenId": 83176, "text": "invasive", "score": -5.232089503373913 }, { "rank": 3, "tokenId": 22186, "text": "Both", "score": -5.5008820333676525 }, { "rank": 4, "tokenId": 29622, "text": "Inv", "score": -9.386374974486017 }, { "rank": 5, "tokenId": 3898, "text": "Int", "score": -9.567715405619275 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": -9.650792574242189 }, { "rank": 7, "tokenId": 29335, "text": "Private", "score": -9.676224314730888 }, { "rank": 8, "tokenId": 495, "text": "in", "score": -10.066169053039971 } ] }, "f32_forward": { "winnerTokenId": 902, "winnerText": "In", "winnerScore": 6.986462593078613, "winnerGap": 12.218545913696289, "top": [ { "rank": 1, "tokenId": 902, "text": "In", "score": 6.986462593078613 }, { "rank": 2, "tokenId": 83176, "text": "invasive", "score": -5.232083320617676 }, { "rank": 3, "tokenId": 22186, "text": "Both", "score": -5.500881671905518 }, { "rank": 4, "tokenId": 29622, "text": "Inv", "score": -9.386373519897461 }, { "rank": 5, "tokenId": 3898, "text": "Int", "score": -9.567716598510742 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": -9.65079402923584 }, { "rank": 7, "tokenId": 29335, "text": "Private", "score": -9.676214218139648 }, { "rank": 8, "tokenId": 495, "text": "in", "score": -10.066161155700684 } ] }, "f32_reverse": { "winnerTokenId": 902, "winnerText": "In", "winnerScore": 6.986468315124512, "winnerGap": 12.218550682067871, "top": [ { "rank": 1, "tokenId": 902, "text": "In", "score": 6.986468315124512 }, { "rank": 2, "tokenId": 83176, "text": "invasive", "score": -5.232082366943359 }, { "rank": 3, "tokenId": 22186, "text": "Both", "score": -5.500874996185303 }, { "rank": 4, "tokenId": 29622, "text": "Inv", "score": -9.386370658874512 }, { "rank": 5, "tokenId": 3898, "text": "Int", "score": -9.56771469116211 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": -9.650797843933105 }, { "rank": 7, "tokenId": 29335, "text": "Private", "score": -9.676229476928711 }, { "rank": 8, "tokenId": 495, "text": "in", "score": -10.066168785095215 } ] }, "f32_tree64": { "winnerTokenId": 902, "winnerText": "In", "winnerScore": 6.986466407775879, "winnerGap": 12.21855354309082, "top": [ { "rank": 1, "tokenId": 902, "text": "In", "score": 6.986466407775879 }, { "rank": 2, "tokenId": 83176, "text": "invasive", "score": -5.232087135314941 }, { "rank": 3, "tokenId": 22186, "text": "Both", "score": -5.500881671905518 }, { "rank": 4, "tokenId": 29622, "text": "Inv", "score": -9.386374473571777 }, { "rank": 5, "tokenId": 3898, "text": "Int", "score": -9.56771469116211 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": -9.65079116821289 }, { "rank": 7, "tokenId": 29335, "text": "Private", "score": -9.676223754882812 }, { "rank": 8, "tokenId": 495, "text": "in", "score": -10.066167831420898 } ] }, "f16_forward": { "winnerTokenId": 902, "winnerText": "In", "winnerScore": 7.01171875, "winnerGap": 12.40625, "top": [ { "rank": 1, "tokenId": 902, "text": "In", "score": 7.01171875 }, { "rank": 2, "tokenId": 83176, "text": "invasive", "score": -5.39453125 }, { "rank": 3, "tokenId": 22186, "text": "Both", "score": -5.515625 }, { "rank": 4, "tokenId": 138, "text": " ", "score": -9.2734375 }, { "rank": 5, "tokenId": 29622, "text": "Inv", "score": -9.4296875 }, { "rank": 6, "tokenId": 33071, "text": "Pot", "score": -9.5390625 }, { "rank": 7, "tokenId": 1204, "text": "IN", "score": -9.578125 }, { "rank": 8, "tokenId": 3898, "text": "Int", "score": -9.6015625 } ] } }, "winnerTextsByMode": { "exact": "In", "f32_forward": "In", "f32_reverse": "In", "f32_tree64": "In", "f16_forward": "In" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "In" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "eth-piracy-poverty-oneword-c", "text": "No explanation. One word only.\nPirating textbooks when you cannot afford them is wrong or understandable. It is", "stableTopK": [ { "rank": 1, "tokenId": 87072, "text": "Illegal", "logit": 10.101204872131348, "prob": 0.3198990821838379 }, { "rank": 2, "tokenId": 58459, "text": "Wrong", "logit": 9.859000205993652, "prob": 0.25108736753463745 }, { "rank": 3, "tokenId": 818, "text": "The", "logit": 9.209176063537598, "prob": 0.13110214471817017 }, { "rank": 4, "tokenId": 31728, "text": "Eth", "logit": 9.035048484802246, "prob": 0.11015067994594574 }, { "rank": 5, "tokenId": 229224, "text": "Necessary", "logit": 8.458351135253906, "prob": 0.06187720596790314 }, { "rank": 6, "tokenId": 2805, "text": "Un", "logit": 7.881336212158203, "prob": 0.03474852070212364 }, { "rank": 7, "tokenId": 20647, "text": "Expl", "logit": 7.415206432342529, "prob": 0.02180211804807186 }, { "rank": 8, "tokenId": 12834, "text": "Il", "logit": 6.67388916015625, "prob": 0.01038840040564537 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 87072, "winnerText": "Illegal", "winnerScore": 10.511208064326189, "winnerGap": 0.27234761751008385, "top": [ { "rank": 1, "tokenId": 87072, "text": "Illegal", "score": 10.511208064326189 }, { "rank": 2, "tokenId": 58459, "text": "Wrong", "score": 10.238860446816105 }, { "rank": 3, "tokenId": 818, "text": "The", "score": 9.51598710365505 }, { "rank": 4, "tokenId": 31728, "text": "Eth", "score": 9.32411903878944 }, { "rank": 5, "tokenId": 229224, "text": "Necessary", "score": 8.693816502428774 }, { "rank": 6, "tokenId": 2805, "text": "Un", "score": 8.070554194152535 }, { "rank": 7, "tokenId": 20647, "text": "Expl", "score": 7.572006412575206 }, { "rank": 8, "tokenId": 12834, "text": "Il", "score": 6.787375978020787 } ] }, "f32_forward": { "winnerTokenId": 87072, "winnerText": "Illegal", "winnerScore": 10.511213302612305, "winnerGap": 0.27234745025634766, "top": [ { "rank": 1, "tokenId": 87072, "text": "Illegal", "score": 10.511213302612305 }, { "rank": 2, "tokenId": 58459, "text": "Wrong", "score": 10.238865852355957 }, { "rank": 3, "tokenId": 818, "text": "The", "score": 9.515987396240234 }, { "rank": 4, "tokenId": 31728, "text": "Eth", "score": 9.324113845825195 }, { "rank": 5, "tokenId": 229224, "text": "Necessary", "score": 8.693812370300293 }, { "rank": 6, "tokenId": 2805, "text": "Un", "score": 8.070549011230469 }, { "rank": 7, "tokenId": 20647, "text": "Expl", "score": 7.5720038414001465 }, { "rank": 8, "tokenId": 12834, "text": "Il", "score": 6.787388801574707 } ] }, "f32_reverse": { "winnerTokenId": 87072, "winnerText": "Illegal", "winnerScore": 10.511197090148926, "winnerGap": 0.2723398208618164, "top": [ { "rank": 1, "tokenId": 87072, "text": "Illegal", "score": 10.511197090148926 }, { "rank": 2, "tokenId": 58459, "text": "Wrong", "score": 10.23885726928711 }, { "rank": 3, "tokenId": 818, "text": "The", "score": 9.515984535217285 }, { "rank": 4, "tokenId": 31728, "text": "Eth", "score": 9.324115753173828 }, { "rank": 5, "tokenId": 229224, "text": "Necessary", "score": 8.693818092346191 }, { "rank": 6, "tokenId": 2805, "text": "Un", "score": 8.070564270019531 }, { "rank": 7, "tokenId": 20647, "text": "Expl", "score": 7.572010040283203 }, { "rank": 8, "tokenId": 12834, "text": "Il", "score": 6.787378787994385 } ] }, "f32_tree64": { "winnerTokenId": 87072, "winnerText": "Illegal", "winnerScore": 10.511208534240723, "winnerGap": 0.27234935760498047, "top": [ { "rank": 1, "tokenId": 87072, "text": "Illegal", "score": 10.511208534240723 }, { "rank": 2, "tokenId": 58459, "text": "Wrong", "score": 10.238859176635742 }, { "rank": 3, "tokenId": 818, "text": "The", "score": 9.515987396240234 }, { "rank": 4, "tokenId": 31728, "text": "Eth", "score": 9.324118614196777 }, { "rank": 5, "tokenId": 229224, "text": "Necessary", "score": 8.693817138671875 }, { "rank": 6, "tokenId": 2805, "text": "Un", "score": 8.070555686950684 }, { "rank": 7, "tokenId": 20647, "text": "Expl", "score": 7.572007179260254 }, { "rank": 8, "tokenId": 12834, "text": "Il", "score": 6.787375450134277 } ] }, "f16_forward": { "winnerTokenId": 818, "winnerText": "The", "winnerScore": 9.4296875, "winnerGap": 0.125, "top": [ { "rank": 1, "tokenId": 818, "text": "The", "score": 9.4296875 }, { "rank": 2, "tokenId": 31728, "text": "Eth", "score": 9.3046875 }, { "rank": 3, "tokenId": 229224, "text": "Necessary", "score": 8.7421875 }, { "rank": 4, "tokenId": 2805, "text": "Un", "score": 8.0390625 }, { "rank": 5, "tokenId": 12834, "text": "Il", "score": 6.79296875 }, { "rank": 6, "tokenId": 87072, "text": "Illegal", "score": 6.57421875 }, { "rank": 7, "tokenId": 58459, "text": "Wrong", "score": 6.3828125 }, { "rank": 8, "tokenId": 141198, "text": "Understand", "score": 5.9375 } ] } }, "winnerTextsByMode": { "exact": "Illegal", "f32_forward": "Illegal", "f32_reverse": "Illegal", "f32_tree64": "Illegal", "f16_forward": "The" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Illegal", "The" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 87072, 106 ], "decodedText": "Illegal", "steps": [ { "step": 0, "tokenId": 87072, "text": "Illegal" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 818, 765 ], "decodedText": "Theft", "steps": [ { "step": 0, "tokenId": 818, "text": "The" }, { "step": 1, "tokenId": 765, "text": "ft" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "eth-self-driving-swerve-oneword-d", "text": "Choose one word only.\nA self-driving car should protect its passenger or the pedestrian. It should protect", "stableTopK": [ { "rank": 1, "tokenId": 103570, "text": "passenger", "logit": 22.142471313476562, "prob": 0.7414742708206177 }, { "rank": 2, "tokenId": 10796, "text": "ped", "logit": 21.0401554107666, "prob": 0.246244415640831 }, { "rank": 3, "tokenId": 1018, "text": "**", "logit": 17.498065948486328, "prob": 0.0071294573135674 }, { "rank": 4, "tokenId": 107599, "text": "Passenger", "logit": 16.050857543945312, "prob": 0.001677033957093954 }, { "rank": 5, "tokenId": 4373, "text": "pass", "logit": 15.859053611755371, "prob": 0.001384338946081698 }, { "rank": 6, "tokenId": 818, "text": "The", "logit": 15.670391082763672, "prob": 0.0011463239789009094 }, { "rank": 7, "tokenId": 1437, "text": "the", "logit": 14.702980995178223, "prob": 0.00043567889952100813 }, { "rank": 8, "tokenId": 40585, "text": "Ped", "logit": 13.773170471191406, "prob": 0.00017193128587678075 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 103570, "winnerText": "passenger", "winnerScore": 28.3876057682903, "winnerGap": 2.2897160409516033, "top": [ { "rank": 1, "tokenId": 103570, "text": "passenger", "score": 28.3876057682903 }, { "rank": 2, "tokenId": 10796, "text": "ped", "score": 26.097889727338696 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 20.02208409663708 }, { "rank": 4, "tokenId": 107599, "text": "Passenger", "score": 17.914929282590826 }, { "rank": 5, "tokenId": 4373, "text": "pass", "score": 17.647468981869295 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 17.386832199287063 }, { "rank": 7, "tokenId": 1437, "text": "the", "score": 16.08573393823066 }, { "rank": 8, "tokenId": 40585, "text": "Ped", "score": 14.885329047970657 } ] }, "f32_forward": { "winnerTokenId": 103570, "winnerText": "passenger", "winnerScore": 28.387561798095703, "winnerGap": 2.289705276489258, "top": [ { "rank": 1, "tokenId": 103570, "text": "passenger", "score": 28.387561798095703 }, { "rank": 2, "tokenId": 10796, "text": "ped", "score": 26.097856521606445 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 20.022104263305664 }, { "rank": 4, "tokenId": 107599, "text": "Passenger", "score": 17.914953231811523 }, { "rank": 5, "tokenId": 4373, "text": "pass", "score": 17.647478103637695 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 17.38682746887207 }, { "rank": 7, "tokenId": 1437, "text": "the", "score": 16.08572769165039 }, { "rank": 8, "tokenId": 40585, "text": "Ped", "score": 14.88534164428711 } ] }, "f32_reverse": { "winnerTokenId": 103570, "winnerText": "passenger", "winnerScore": 28.38760757446289, "winnerGap": 2.2897300720214844, "top": [ { "rank": 1, "tokenId": 103570, "text": "passenger", "score": 28.38760757446289 }, { "rank": 2, "tokenId": 10796, "text": "ped", "score": 26.097877502441406 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 20.022092819213867 }, { "rank": 4, "tokenId": 107599, "text": "Passenger", "score": 17.914926528930664 }, { "rank": 5, "tokenId": 4373, "text": "pass", "score": 17.647470474243164 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 17.386838912963867 }, { "rank": 7, "tokenId": 1437, "text": "the", "score": 16.08572769165039 }, { "rank": 8, "tokenId": 40585, "text": "Ped", "score": 14.885333061218262 } ] }, "f32_tree64": { "winnerTokenId": 103570, "winnerText": "passenger", "winnerScore": 28.387605667114258, "winnerGap": 2.2897167205810547, "top": [ { "rank": 1, "tokenId": 103570, "text": "passenger", "score": 28.387605667114258 }, { "rank": 2, "tokenId": 10796, "text": "ped", "score": 26.097888946533203 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 20.022083282470703 }, { "rank": 4, "tokenId": 107599, "text": "Passenger", "score": 17.914926528930664 }, { "rank": 5, "tokenId": 4373, "text": "pass", "score": 17.64746856689453 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 17.38683319091797 }, { "rank": 7, "tokenId": 1437, "text": "the", "score": 16.085735321044922 }, { "rank": 8, "tokenId": 40585, "text": "Ped", "score": 14.885329246520996 } ] }, "f16_forward": { "winnerTokenId": 103570, "winnerText": "passenger", "winnerScore": 28.46875, "winnerGap": 2.171875, "top": [ { "rank": 1, "tokenId": 103570, "text": "passenger", "score": 28.46875 }, { "rank": 2, "tokenId": 10796, "text": "ped", "score": 26.296875 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 19.953125 }, { "rank": 4, "tokenId": 107599, "text": "Passenger", "score": 18.015625 }, { "rank": 5, "tokenId": 4373, "text": "pass", "score": 17.609375 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 17.328125 }, { "rank": 7, "tokenId": 1437, "text": "the", "score": 16.046875 }, { "rank": 8, "tokenId": 40585, "text": "Ped", "score": 14.65625 } ] } }, "winnerTextsByMode": { "exact": "passenger", "f32_forward": "passenger", "f32_reverse": "passenger", "f32_tree64": "passenger", "f16_forward": "passenger" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "passenger" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "eth-strike-essential-oneword-a", "text": "Answer with exactly one word only.\nNurses striking during a pandemic is responsible or irresponsible. It is", "stableTopK": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "logit": 11.849617004394531, "prob": 0.8837082386016846 }, { "rank": 2, "tokenId": 3637, "text": "Context", "logit": 9.711564064025879, "prob": 0.10417518764734268 }, { "rank": 3, "tokenId": 1558, "text": "Com", "logit": 6.252890110015869, "prob": 0.003278546268120408 }, { "rank": 4, "tokenId": 55144, "text": "Amb", "logit": 6.1260762214660645, "prob": 0.002888063434511423 }, { "rank": 5, "tokenId": 84551, "text": "Situ", "logit": 5.592145919799805, "prob": 0.0016932603903114796 }, { "rank": 6, "tokenId": 43868, "text": "Nu", "logit": 5.128983020782471, "prob": 0.0010655520018190145 }, { "rank": 7, "tokenId": 22186, "text": "Both", "logit": 4.75201416015625, "prob": 0.0007309020147658885 }, { "rank": 8, "tokenId": 148591, "text": "Responsible", "logit": 3.852060079574585, "prob": 0.0002971762150991708 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 48333, "winnerText": "Complex", "winnerScore": 12.530867759310055, "winnerGap": 2.4569971282450425, "top": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "score": 12.530867759310055 }, { "rank": 2, "tokenId": 3637, "text": "Context", "score": 10.073870631065013 }, { "rank": 3, "tokenId": 1558, "text": "Com", "score": 6.345872632579216 }, { "rank": 4, "tokenId": 55144, "text": "Amb", "score": 6.213425214112547 }, { "rank": 5, "tokenId": 84551, "text": "Situ", "score": 5.658298901376384 }, { "rank": 6, "tokenId": 43868, "text": "Nu", "score": 5.179851692047336 }, { "rank": 7, "tokenId": 22186, "text": "Both", "score": 4.792367478831821 }, { "rank": 8, "tokenId": 148591, "text": "Responsible", "score": 3.87344314514759 } ] }, "f32_forward": { "winnerTokenId": 48333, "winnerText": "Complex", "winnerScore": 12.530872344970703, "winnerGap": 2.457002639770508, "top": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "score": 12.530872344970703 }, { "rank": 2, "tokenId": 3637, "text": "Context", "score": 10.073869705200195 }, { "rank": 3, "tokenId": 1558, "text": "Com", "score": 6.345876216888428 }, { "rank": 4, "tokenId": 55144, "text": "Amb", "score": 6.213428497314453 }, { "rank": 5, "tokenId": 84551, "text": "Situ", "score": 5.658300399780273 }, { "rank": 6, "tokenId": 43868, "text": "Nu", "score": 5.179850101470947 }, { "rank": 7, "tokenId": 22186, "text": "Both", "score": 4.792364597320557 }, { "rank": 8, "tokenId": 148591, "text": "Responsible", "score": 3.8734400272369385 } ] }, "f32_reverse": { "winnerTokenId": 48333, "winnerText": "Complex", "winnerScore": 12.530872344970703, "winnerGap": 2.457003593444824, "top": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "score": 12.530872344970703 }, { "rank": 2, "tokenId": 3637, "text": "Context", "score": 10.073868751525879 }, { "rank": 3, "tokenId": 1558, "text": "Com", "score": 6.34587287902832 }, { "rank": 4, "tokenId": 55144, "text": "Amb", "score": 6.213423252105713 }, { "rank": 5, "tokenId": 84551, "text": "Situ", "score": 5.65830135345459 }, { "rank": 6, "tokenId": 43868, "text": "Nu", "score": 5.179853439331055 }, { "rank": 7, "tokenId": 22186, "text": "Both", "score": 4.792361736297607 }, { "rank": 8, "tokenId": 148591, "text": "Responsible", "score": 3.8734428882598877 } ] }, "f32_tree64": { "winnerTokenId": 48333, "winnerText": "Complex", "winnerScore": 12.530866622924805, "winnerGap": 2.4569969177246094, "top": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "score": 12.530866622924805 }, { "rank": 2, "tokenId": 3637, "text": "Context", "score": 10.073869705200195 }, { "rank": 3, "tokenId": 1558, "text": "Com", "score": 6.345871925354004 }, { "rank": 4, "tokenId": 55144, "text": "Amb", "score": 6.213425636291504 }, { "rank": 5, "tokenId": 84551, "text": "Situ", "score": 5.658299446105957 }, { "rank": 6, "tokenId": 43868, "text": "Nu", "score": 5.17985200881958 }, { "rank": 7, "tokenId": 22186, "text": "Both", "score": 4.792366981506348 }, { "rank": 8, "tokenId": 148591, "text": "Responsible", "score": 3.873443603515625 } ] }, "f16_forward": { "winnerTokenId": 48333, "winnerText": "Complex", "winnerScore": 12.6484375, "winnerGap": 6.24609375, "top": [ { "rank": 1, "tokenId": 48333, "text": "Complex", "score": 12.6484375 }, { "rank": 2, "tokenId": 1558, "text": "Com", "score": 6.40234375 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "score": 6.1875 }, { "rank": 4, "tokenId": 3637, "text": "Context", "score": 6.08984375 }, { "rank": 5, "tokenId": 84551, "text": "Situ", "score": 5.69140625 }, { "rank": 6, "tokenId": 22186, "text": "Both", "score": 4.8203125 }, { "rank": 7, "tokenId": 43868, "text": "Nu", "score": 4.2421875 }, { "rank": 8, "tokenId": 148591, "text": "Responsible", "score": 3.9140625 } ] } }, "winnerTextsByMode": { "exact": "Complex", "f32_forward": "Complex", "f32_reverse": "Complex", "f32_tree64": "Complex", "f16_forward": "Complex" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Complex" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "eth-trolley-oneword-b", "text": "Respond with one word only.\nPull the lever to save five but kill one — yes or no?", "stableTopK": [ { "rank": 1, "tokenId": 3771, "text": "No", "logit": 8.024896621704102, "prob": 0.9996246099472046 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "logit": 0.07971983402967453, "prob": 0.0003542342165019363 }, { "rank": 3, "tokenId": 154745, "text": "Impossible", "logit": -4.306285381317139, "prob": 0.000004410345809446881 }, { "rank": 4, "tokenId": 1904, "text": "no", "logit": -4.3750200271606445, "prob": 0.000004117385742574697 }, { "rank": 5, "tokenId": 191239, "text": "Difficult", "logit": -4.828357696533203, "prob": 0.0000026166130737692583 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "logit": -5.006026744842529, "prob": 0.0000021906794245296624 }, { "rank": 7, "tokenId": 58686, "text": "Cannot", "logit": -5.093679428100586, "prob": 0.00000200683530238166 }, { "rank": 8, "tokenId": 80163, "text": "Sit", "logit": -6.663608074188232, "prob": 4.175422247953975e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 8.224966101279108, "winnerGap": 8.145246988477474, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 8.224966101279108 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 0.07971911280163368 }, { "rank": 3, "tokenId": 154745, "text": "Impossible", "score": -4.336230240316752 }, { "rank": 4, "tokenId": 1904, "text": "no", "score": -4.406435851136608 }, { "rank": 5, "tokenId": 191239, "text": "Difficult", "score": -4.870709156653264 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": -5.053284483655295 }, { "rank": 7, "tokenId": 58686, "text": "Cannot", "score": -5.143493462984836 }, { "rank": 8, "tokenId": 80163, "text": "Sit", "score": -6.776557726957364 } ] }, "f32_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 8.224960327148438, "winnerGap": 8.14523784071207, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 8.224960327148438 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 0.07972248643636703 }, { "rank": 3, "tokenId": 154745, "text": "Impossible", "score": -4.3362274169921875 }, { "rank": 4, "tokenId": 1904, "text": "no", "score": -4.406436920166016 }, { "rank": 5, "tokenId": 191239, "text": "Difficult", "score": -4.8707098960876465 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": -5.053284168243408 }, { "rank": 7, "tokenId": 58686, "text": "Cannot", "score": -5.143490791320801 }, { "rank": 8, "tokenId": 80163, "text": "Sit", "score": -6.776556491851807 } ] }, "f32_reverse": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 8.224966049194336, "winnerGap": 8.145244061946869, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 8.224966049194336 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 0.07972198724746704 }, { "rank": 3, "tokenId": 154745, "text": "Impossible", "score": -4.3362274169921875 }, { "rank": 4, "tokenId": 1904, "text": "no", "score": -4.406436920166016 }, { "rank": 5, "tokenId": 191239, "text": "Difficult", "score": -4.870700359344482 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": -5.05329704284668 }, { "rank": 7, "tokenId": 58686, "text": "Cannot", "score": -5.143496513366699 }, { "rank": 8, "tokenId": 80163, "text": "Sit", "score": -6.776558876037598 } ] }, "f32_tree64": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 8.224966049194336, "winnerGap": 8.145247220993042, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 8.224966049194336 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 0.07971882820129395 }, { "rank": 3, "tokenId": 154745, "text": "Impossible", "score": -4.336230278015137 }, { "rank": 4, "tokenId": 1904, "text": "no", "score": -4.406435966491699 }, { "rank": 5, "tokenId": 191239, "text": "Difficult", "score": -4.870708465576172 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": -5.05328369140625 }, { "rank": 7, "tokenId": 58686, "text": "Cannot", "score": -5.143494606018066 }, { "rank": 8, "tokenId": 80163, "text": "Sit", "score": -6.7765583992004395 } ] }, "f16_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 4.31640625, "winnerGap": 4.24169921875, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 4.31640625 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 0.07470703125 }, { "rank": 3, "tokenId": 154745, "text": "Impossible", "score": -4.33984375 }, { "rank": 4, "tokenId": 1904, "text": "no", "score": -4.39453125 }, { "rank": 5, "tokenId": 7018, "text": "NO", "score": -4.43359375 }, { "rank": 6, "tokenId": 236776, "text": "A", "score": -4.6640625 }, { "rank": 7, "tokenId": 115863, "text": "Analyze", "score": -4.75390625 }, { "rank": 8, "tokenId": 15162, "text": "Program", "score": -4.7578125 } ] } }, "winnerTextsByMode": { "exact": "No", "f32_forward": "No", "f32_reverse": "No", "f32_tree64": "No", "f16_forward": "No" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "No" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "eth-vigilante-oneword-c", "text": "No explanation. One word only.\nVigilante justice when the legal system fails is noble or dangerous. It is", "stableTopK": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "logit": 15.338310241699219, "prob": 0.9323009252548218 }, { "rank": 2, "tokenId": 6700, "text": "Am", "logit": 11.94344425201416, "prob": 0.031274084001779556 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "logit": 11.575711250305176, "prob": 0.02165110968053341 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "logit": 9.755098342895508, "prob": 0.003505887696519494 }, { "rank": 5, "tokenId": 62892, "text": "Grey", "logit": 9.318031311035156, "prob": 0.0022645515855401754 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "logit": 9.168493270874023, "prob": 0.0019500181078910828 }, { "rank": 7, "tokenId": 31728, "text": "Eth", "logit": 8.854832649230957, "prob": 0.001425008988007903 }, { "rank": 8, "tokenId": 236536, "text": "Dangerous", "logit": 8.642890930175781, "prob": 0.001152849174104631 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 16.933710647262913, "winnerGap": 4.291509227708104, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 16.933710647262913 }, { "rank": 2, "tokenId": 6700, "text": "Am", "score": 12.642201419554809 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": 12.207700282169544 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 10.122528618841681 }, { "rank": 5, "tokenId": 62892, "text": "Grey", "score": 9.6363137349312 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 9.471089823427022 }, { "rank": 7, "tokenId": 31728, "text": "Eth", "score": 9.126321589757346 }, { "rank": 8, "tokenId": 236536, "text": "Dangerous", "score": 8.894673539586364 } ] }, "f32_forward": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 16.9337100982666, "winnerGap": 4.291513442993164, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 16.9337100982666 }, { "rank": 2, "tokenId": 6700, "text": "Am", "score": 12.642196655273438 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": 12.207708358764648 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 10.122523307800293 }, { "rank": 5, "tokenId": 62892, "text": "Grey", "score": 9.636313438415527 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 9.471096992492676 }, { "rank": 7, "tokenId": 31728, "text": "Eth", "score": 9.126310348510742 }, { "rank": 8, "tokenId": 236536, "text": "Dangerous", "score": 8.894684791564941 } ] }, "f32_reverse": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 16.933706283569336, "winnerGap": 4.291494369506836, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 16.933706283569336 }, { "rank": 2, "tokenId": 6700, "text": "Am", "score": 12.6422119140625 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": 12.207695007324219 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 10.122529983520508 }, { "rank": 5, "tokenId": 62892, "text": "Grey", "score": 9.636310577392578 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 9.471086502075195 }, { "rank": 7, "tokenId": 31728, "text": "Eth", "score": 9.126324653625488 }, { "rank": 8, "tokenId": 236536, "text": "Dangerous", "score": 8.894676208496094 } ] }, "f32_tree64": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 16.933712005615234, "winnerGap": 4.291511535644531, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 16.933712005615234 }, { "rank": 2, "tokenId": 6700, "text": "Am", "score": 12.642200469970703 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": 12.2076997756958 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 10.122528076171875 }, { "rank": 5, "tokenId": 62892, "text": "Grey", "score": 9.636313438415527 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 9.471090316772461 }, { "rank": 7, "tokenId": 31728, "text": "Eth", "score": 9.126320838928223 }, { "rank": 8, "tokenId": 236536, "text": "Dangerous", "score": 8.894674301147461 } ] }, "f16_forward": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 16.9375, "winnerGap": 4.21875, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 16.9375 }, { "rank": 2, "tokenId": 6700, "text": "Am", "score": 12.71875 }, { "rank": 3, "tokenId": 62892, "text": "Grey", "score": 9.6484375 }, { "rank": 4, "tokenId": 21288, "text": "Subject", "score": 9.390625 }, { "rank": 5, "tokenId": 31728, "text": "Eth", "score": 9.03125 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "score": 8.515625 }, { "rank": 7, "tokenId": 31900, "text": "Deb", "score": 8.1953125 }, { "rank": 8, "tokenId": 27505, "text": "Mor", "score": 8.1171875 } ] } }, "winnerTextsByMode": { "exact": "Amb", "f32_forward": "Amb", "f32_reverse": "Amb", "f32_tree64": "Amb", "f16_forward": "Amb" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Amb" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "eth-whistleblower-oneword-d", "text": "Choose one word only.\nA whistleblower who leaks classified documents is a hero or a traitor. They are", "stableTopK": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "logit": 15.714991569519043, "prob": 0.7161386609077454 }, { "rank": 2, "tokenId": 1018, "text": "**", "logit": 14.579312324523926, "prob": 0.2300264984369278 }, { "rank": 3, "tokenId": 15344, "text": "Tra", "logit": 12.396313667297363, "prob": 0.025924693793058395 }, { "rank": 4, "tokenId": 1525, "text": "amb", "logit": 12.286328315734863, "prob": 0.023224564269185066 }, { "rank": 5, "tokenId": 11341, "text": "tra", "logit": 8.764121055603027, "prob": 0.000685918319504708 }, { "rank": 6, "tokenId": 2035, "text": "Tr", "logit": 8.61772632598877, "prob": 0.0005925076548010111 }, { "rank": 7, "tokenId": 39748, "text": "Hero", "logit": 8.466873168945312, "prob": 0.0005095411906950176 }, { "rank": 8, "tokenId": 2825, "text": "Wh", "logit": 8.15517807006836, "prob": 0.0003730884927790612 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 17.44823021036823, "winnerGap": 1.524829784168027, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 17.44823021036823 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 15.923400426200203 }, { "rank": 3, "tokenId": 15344, "text": "Tra", "score": 13.184296562628319 }, { "rank": 4, "tokenId": 1525, "text": "amb", "score": 13.051912395879402 }, { "rank": 5, "tokenId": 11341, "text": "tra", "score": 9.027042464155258 }, { "rank": 6, "tokenId": 2035, "text": "Tr", "score": 8.867235731906902 }, { "rank": 7, "tokenId": 39748, "text": "Hero", "score": 8.703073075046596 }, { "rank": 8, "tokenId": 2825, "text": "Wh", "score": 8.365463071245351 } ] }, "f32_forward": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 17.448232650756836, "winnerGap": 1.5248403549194336, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 17.448232650756836 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 15.923392295837402 }, { "rank": 3, "tokenId": 15344, "text": "Tra", "score": 13.18430233001709 }, { "rank": 4, "tokenId": 1525, "text": "amb", "score": 13.051904678344727 }, { "rank": 5, "tokenId": 11341, "text": "tra", "score": 9.027045249938965 }, { "rank": 6, "tokenId": 2035, "text": "Tr", "score": 8.867231369018555 }, { "rank": 7, "tokenId": 39748, "text": "Hero", "score": 8.703073501586914 }, { "rank": 8, "tokenId": 2825, "text": "Wh", "score": 8.365449905395508 } ] }, "f32_reverse": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 17.448219299316406, "winnerGap": 1.5248165130615234, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 17.448219299316406 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 15.923402786254883 }, { "rank": 3, "tokenId": 15344, "text": "Tra", "score": 13.184298515319824 }, { "rank": 4, "tokenId": 1525, "text": "amb", "score": 13.051920890808105 }, { "rank": 5, "tokenId": 11341, "text": "tra", "score": 9.027039527893066 }, { "rank": 6, "tokenId": 2035, "text": "Tr", "score": 8.867236137390137 }, { "rank": 7, "tokenId": 39748, "text": "Hero", "score": 8.70307445526123 }, { "rank": 8, "tokenId": 2825, "text": "Wh", "score": 8.365461349487305 } ] }, "f32_tree64": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 17.448230743408203, "winnerGap": 1.524831771850586, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 17.448230743408203 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 15.923398971557617 }, { "rank": 3, "tokenId": 15344, "text": "Tra", "score": 13.184295654296875 }, { "rank": 4, "tokenId": 1525, "text": "amb", "score": 13.051912307739258 }, { "rank": 5, "tokenId": 11341, "text": "tra", "score": 9.027042388916016 }, { "rank": 6, "tokenId": 2035, "text": "Tr", "score": 8.86723518371582 }, { "rank": 7, "tokenId": 39748, "text": "Hero", "score": 8.703072547912598 }, { "rank": 8, "tokenId": 2825, "text": "Wh", "score": 8.365463256835938 } ] }, "f16_forward": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 17.625, "winnerGap": 1.65625, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 17.625 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 15.96875 }, { "rank": 3, "tokenId": 15344, "text": "Tra", "score": 13.2734375 }, { "rank": 4, "tokenId": 1525, "text": "amb", "score": 13.0703125 }, { "rank": 5, "tokenId": 11341, "text": "tra", "score": 9.0546875 }, { "rank": 6, "tokenId": 6700, "text": "Am", "score": 8.390625 }, { "rank": 7, "tokenId": 191239, "text": "Difficult", "score": 7.86328125 }, { "rank": 8, "tokenId": 48333, "text": "Complex", "score": 7.65234375 } ] } }, "winnerTextsByMode": { "exact": "Amb", "f32_forward": "Amb", "f32_reverse": "Amb", "f32_tree64": "Amb", "f16_forward": "Amb" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Amb" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "even-odd-17-oneword-a", "text": "Answer with exactly one word only.\nAnswer with exactly one word: even or odd.\nQuestion: 17 is\nAnswer:", "stableTopK": [ { "rank": 1, "tokenId": 19808, "text": "odd", "logit": 8.517521858215332, "prob": 1 }, { "rank": 2, "tokenId": 20952, "text": "even", "logit": -8.916555404663086, "prob": 2.6821091125839303e-8 }, { "rank": 3, "tokenId": 11049, "text": " odd", "logit": -12.508428573608398, "prob": 7.388321066059689e-10 }, { "rank": 4, "tokenId": 1018, "text": "**", "logit": -13.618481636047363, "prob": 2.4347582283645863e-10 }, { "rank": 5, "tokenId": 54119, "text": "Odd", "logit": -14.230732917785645, "prob": 1.3199530357610456e-10 }, { "rank": 6, "tokenId": 590, "text": "od", "logit": -14.45579719543457, "prob": 1.0539361744443809e-10 }, { "rank": 7, "tokenId": 811, "text": "one", "logit": -15.253706932067871, "prob": 4.745549439522101e-11 }, { "rank": 8, "tokenId": 47526, "text": "seven", "logit": -15.52412223815918, "prob": 3.6211509485406523e-11 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 19808, "winnerText": "odd", "winnerScore": 8.758137267375812, "winnerGap": 17.952112715897552, "top": [ { "rank": 1, "tokenId": 19808, "text": "odd", "score": 8.758137267375812 }, { "rank": 2, "tokenId": 20952, "text": "even", "score": -9.19397544852174 }, { "rank": 3, "tokenId": 11049, "text": " odd", "score": -13.31974511318736 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": -14.689905798341442 }, { "rank": 5, "tokenId": 54119, "text": "Odd", "score": -15.470356693670432 }, { "rank": 6, "tokenId": 590, "text": "od", "score": -15.762116713851391 }, { "rank": 7, "tokenId": 811, "text": "one", "score": -16.819390766584917 }, { "rank": 8, "tokenId": 47526, "text": "seven", "score": -17.186381689608062 } ] }, "f32_forward": { "winnerTokenId": 19808, "winnerText": "odd", "winnerScore": 8.758134841918945, "winnerGap": 17.952102661132812, "top": [ { "rank": 1, "tokenId": 19808, "text": "odd", "score": 8.758134841918945 }, { "rank": 2, "tokenId": 20952, "text": "even", "score": -9.193967819213867 }, { "rank": 3, "tokenId": 11049, "text": " odd", "score": -13.319748878479004 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": -14.689906120300293 }, { "rank": 5, "tokenId": 54119, "text": "Odd", "score": -15.47036361694336 }, { "rank": 6, "tokenId": 590, "text": "od", "score": -15.762113571166992 }, { "rank": 7, "tokenId": 811, "text": "one", "score": -16.819395065307617 }, { "rank": 8, "tokenId": 47526, "text": "seven", "score": -17.186372756958008 } ] }, "f32_reverse": { "winnerTokenId": 19808, "winnerText": "odd", "winnerScore": 8.758137702941895, "winnerGap": 17.95212745666504, "top": [ { "rank": 1, "tokenId": 19808, "text": "odd", "score": 8.758137702941895 }, { "rank": 2, "tokenId": 20952, "text": "even", "score": -9.193989753723145 }, { "rank": 3, "tokenId": 11049, "text": " odd", "score": -13.319747924804688 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": -14.689913749694824 }, { "rank": 5, "tokenId": 54119, "text": "Odd", "score": -15.47037124633789 }, { "rank": 6, "tokenId": 590, "text": "od", "score": -15.762118339538574 }, { "rank": 7, "tokenId": 811, "text": "one", "score": -16.819381713867188 }, { "rank": 8, "tokenId": 47526, "text": "seven", "score": -17.186372756958008 } ] }, "f32_tree64": { "winnerTokenId": 19808, "winnerText": "odd", "winnerScore": 8.758136749267578, "winnerGap": 17.952112197875977, "top": [ { "rank": 1, "tokenId": 19808, "text": "odd", "score": 8.758136749267578 }, { "rank": 2, "tokenId": 20952, "text": "even", "score": -9.193975448608398 }, { "rank": 3, "tokenId": 11049, "text": " odd", "score": -13.319747924804688 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": -14.689906120300293 }, { "rank": 5, "tokenId": 54119, "text": "Odd", "score": -15.470356941223145 }, { "rank": 6, "tokenId": 590, "text": "od", "score": -15.762115478515625 }, { "rank": 7, "tokenId": 811, "text": "one", "score": -16.81939125061035 }, { "rank": 8, "tokenId": 47526, "text": "seven", "score": -17.186382293701172 } ] }, "f16_forward": { "winnerTokenId": 19808, "winnerText": "odd", "winnerScore": 8.7421875, "winnerGap": 12.90234375, "top": [ { "rank": 1, "tokenId": 19808, "text": "odd", "score": 8.7421875 }, { "rank": 2, "tokenId": 20952, "text": "even", "score": -4.16015625 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": -10.8046875 }, { "rank": 4, "tokenId": 11049, "text": " odd", "score": -13.46875 }, { "rank": 5, "tokenId": 54119, "text": "Odd", "score": -15.515625 }, { "rank": 6, "tokenId": 590, "text": "od", "score": -15.6953125 }, { "rank": 7, "tokenId": 236772, "text": "-", "score": -16.15625 }, { "rank": 8, "tokenId": 107, "text": "\n", "score": -16.296875 } ] } }, "winnerTextsByMode": { "exact": "odd", "f32_forward": "odd", "f32_reverse": "odd", "f32_tree64": "odd", "f16_forward": "odd" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "odd" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "firewall-allow-block-oneword-b", "text": "Respond with one word only.\nOpening every inbound firewall port on a production database server is allow or block. It should be", "stableTopK": [ { "rank": 1, "tokenId": 10809, "text": "Block", "logit": 9.279829978942871, "prob": 0.9456053376197815 }, { "rank": 2, "tokenId": 27122, "text": "Allow", "logit": 6.3888258934021, "prob": 0.05250042304396629 }, { "rank": 3, "tokenId": 22472, "text": "Den", "logit": 2.5654220581054688, "prob": 0.0011473068734630942 }, { "rank": 4, "tokenId": 5088, "text": "block", "logit": 1.9962009191513062, "prob": 0.0006493367836810648 }, { "rank": 5, "tokenId": 210872, "text": "Blocked", "logit": -1.1903272867202759, "prob": 0.00002682738886505831 }, { "rank": 6, "tokenId": 156488, "text": "Restrict", "logit": -1.4344251155853271, "prob": 0.000021016869141021743 }, { "rank": 7, "tokenId": 205380, "text": "deny", "logit": -2.168626546859741, "prob": 0.00001008575509331422 }, { "rank": 8, "tokenId": 70242, "text": "Allowed", "logit": -3.2309389114379883, "prob": 0.000003486197783786338 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 10809, "winnerText": "Block", "winnerScore": 9.594054012861193, "winnerGap": 3.105926957952164, "top": [ { "rank": 1, "tokenId": 10809, "text": "Block", "score": 9.594054012861193 }, { "rank": 2, "tokenId": 27122, "text": "Allow", "score": 6.488127054909029 }, { "rank": 3, "tokenId": 22472, "text": "Den", "score": 2.5717016716205237 }, { "rank": 4, "tokenId": 5088, "text": "block", "score": 1.9991554788429955 }, { "rank": 5, "tokenId": 210872, "text": "Blocked", "score": -1.1909515543637672 }, { "rank": 6, "tokenId": 156488, "text": "Restrict", "score": -1.4355169786580433 }, { "rank": 7, "tokenId": 205380, "text": "deny", "score": -2.1724171206508913 }, { "rank": 8, "tokenId": 70242, "text": "Allowed", "score": -3.24352184174392 } ] }, "f32_forward": { "winnerTokenId": 10809, "winnerText": "Block", "winnerScore": 9.594062805175781, "winnerGap": 3.105931282043457, "top": [ { "rank": 1, "tokenId": 10809, "text": "Block", "score": 9.594062805175781 }, { "rank": 2, "tokenId": 27122, "text": "Allow", "score": 6.488131523132324 }, { "rank": 3, "tokenId": 22472, "text": "Den", "score": 2.5716965198516846 }, { "rank": 4, "tokenId": 5088, "text": "block", "score": 1.9991546869277954 }, { "rank": 5, "tokenId": 210872, "text": "Blocked", "score": -1.1909551620483398 }, { "rank": 6, "tokenId": 156488, "text": "Restrict", "score": -1.43551504611969 }, { "rank": 7, "tokenId": 205380, "text": "deny", "score": -2.172412157058716 }, { "rank": 8, "tokenId": 70242, "text": "Allowed", "score": -3.2435197830200195 } ] }, "f32_reverse": { "winnerTokenId": 10809, "winnerText": "Block", "winnerScore": 9.594048500061035, "winnerGap": 3.1059184074401855, "top": [ { "rank": 1, "tokenId": 10809, "text": "Block", "score": 9.594048500061035 }, { "rank": 2, "tokenId": 27122, "text": "Allow", "score": 6.48813009262085 }, { "rank": 3, "tokenId": 22472, "text": "Den", "score": 2.5717039108276367 }, { "rank": 4, "tokenId": 5088, "text": "block", "score": 1.9991519451141357 }, { "rank": 5, "tokenId": 210872, "text": "Blocked", "score": -1.1909537315368652 }, { "rank": 6, "tokenId": 156488, "text": "Restrict", "score": -1.4355207681655884 }, { "rank": 7, "tokenId": 205380, "text": "deny", "score": -2.1724228858947754 }, { "rank": 8, "tokenId": 70242, "text": "Allowed", "score": -3.243525266647339 } ] }, "f32_tree64": { "winnerTokenId": 10809, "winnerText": "Block", "winnerScore": 9.594054222106934, "winnerGap": 3.105926990509033, "top": [ { "rank": 1, "tokenId": 10809, "text": "Block", "score": 9.594054222106934 }, { "rank": 2, "tokenId": 27122, "text": "Allow", "score": 6.4881272315979 }, { "rank": 3, "tokenId": 22472, "text": "Den", "score": 2.571702241897583 }, { "rank": 4, "tokenId": 5088, "text": "block", "score": 1.9991545677185059 }, { "rank": 5, "tokenId": 210872, "text": "Blocked", "score": -1.1909503936767578 }, { "rank": 6, "tokenId": 156488, "text": "Restrict", "score": -1.435517430305481 }, { "rank": 7, "tokenId": 205380, "text": "deny", "score": -2.1724157333374023 }, { "rank": 8, "tokenId": 70242, "text": "Allowed", "score": -3.2435219287872314 } ] }, "f16_forward": { "winnerTokenId": 10809, "winnerText": "Block", "winnerScore": 9.5234375, "winnerGap": 3.02734375, "top": [ { "rank": 1, "tokenId": 10809, "text": "Block", "score": 9.5234375 }, { "rank": 2, "tokenId": 27122, "text": "Allow", "score": 6.49609375 }, { "rank": 3, "tokenId": 5088, "text": "block", "score": 2.037109375 }, { "rank": 4, "tokenId": 156488, "text": "Restrict", "score": -0.440185546875 }, { "rank": 5, "tokenId": 22472, "text": "Den", "score": -1.1455078125 }, { "rank": 6, "tokenId": 210872, "text": "Blocked", "score": -1.494140625 }, { "rank": 7, "tokenId": 205380, "text": "deny", "score": -2.19921875 }, { "rank": 8, "tokenId": 70242, "text": "Allowed", "score": -2.26953125 } ] } }, "winnerTextsByMode": { "exact": "Block", "f32_forward": "Block", "f32_reverse": "Block", "f32_tree64": "Block", "f16_forward": "Block" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Block" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "firewall-good-bad-oneword-c", "text": "No explanation. One word only.\nOpening every inbound firewall port on a production database server is good or bad. It is", "stableTopK": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "logit": 12.696904182434082, "prob": 0.9999956488609314 }, { "rank": 2, "tokenId": 236536, "text": "Dangerous", "logit": -0.49715355038642883, "prob": 0.0000018616226498124888 }, { "rank": 3, "tokenId": 99510, "text": "Ris", "logit": -0.9813820719718933, "prob": 0.0000011470804111013422 }, { "rank": 4, "tokenId": 1018, "text": "**", "logit": -1.7333894968032837, "prob": 5.407558205661189e-7 }, { "rank": 5, "tokenId": 26546, "text": "Very", "logit": -2.7817189693450928, "prob": 1.895472507840168e-7 }, { "rank": 6, "tokenId": 15242, "text": "bad", "logit": -3.2004482746124268, "prob": 1.2469978116769198e-7 }, { "rank": 7, "tokenId": 37701, "text": "Ter", "logit": -3.7549612522125244, "prob": 7.162154957995881e-8 }, { "rank": 8, "tokenId": 4830, "text": "Dis", "logit": -3.876091957092285, "prob": 6.345083392034212e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 13.548614363628701, "winnerGap": 14.045810868335828, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 13.548614363628701 }, { "rank": 2, "tokenId": 236536, "text": "Dangerous", "score": -0.4971965047071265 }, { "rank": 3, "tokenId": 99510, "text": "Ris", "score": -0.9817294288734502 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": -1.7353215496010037 }, { "rank": 5, "tokenId": 26546, "text": "Very", "score": -2.78973435056308 }, { "rank": 6, "tokenId": 15242, "text": "bad", "score": -3.2126706630981836 }, { "rank": 7, "tokenId": 37701, "text": "Ter", "score": -3.7747563362652414 }, { "rank": 8, "tokenId": 4830, "text": "Dis", "score": -3.8978795528479964 } ] }, "f32_forward": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 13.548623085021973, "winnerGap": 14.045818328857422, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 13.548623085021973 }, { "rank": 2, "tokenId": 236536, "text": "Dangerous", "score": -0.4971952438354492 }, { "rank": 3, "tokenId": 99510, "text": "Ris", "score": -0.9817309975624084 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": -1.7353224754333496 }, { "rank": 5, "tokenId": 26546, "text": "Very", "score": -2.7897346019744873 }, { "rank": 6, "tokenId": 15242, "text": "bad", "score": -3.2126755714416504 }, { "rank": 7, "tokenId": 37701, "text": "Ter", "score": -3.7747554779052734 }, { "rank": 8, "tokenId": 4830, "text": "Dis", "score": -3.8978822231292725 } ] }, "f32_reverse": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 13.548619270324707, "winnerGap": 14.045810014009476, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 13.548619270324707 }, { "rank": 2, "tokenId": 236536, "text": "Dangerous", "score": -0.4971907436847687 }, { "rank": 3, "tokenId": 99510, "text": "Ris", "score": -0.981731116771698 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": -1.7353204488754272 }, { "rank": 5, "tokenId": 26546, "text": "Very", "score": -2.7897374629974365 }, { "rank": 6, "tokenId": 15242, "text": "bad", "score": -3.2126691341400146 }, { "rank": 7, "tokenId": 37701, "text": "Ter", "score": -3.774754762649536 }, { "rank": 8, "tokenId": 4830, "text": "Dis", "score": -3.897878646850586 } ] }, "f32_tree64": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 13.548614501953125, "winnerGap": 14.045811414718628, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 13.548614501953125 }, { "rank": 2, "tokenId": 236536, "text": "Dangerous", "score": -0.49719691276550293 }, { "rank": 3, "tokenId": 99510, "text": "Ris", "score": -0.9817287921905518 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": -1.735323429107666 }, { "rank": 5, "tokenId": 26546, "text": "Very", "score": -2.789734125137329 }, { "rank": 6, "tokenId": 15242, "text": "bad", "score": -3.2126710414886475 }, { "rank": 7, "tokenId": 37701, "text": "Ter", "score": -3.774756908416748 }, { "rank": 8, "tokenId": 4830, "text": "Dis", "score": -3.897879123687744 } ] }, "f16_forward": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 13.578125, "winnerGap": 14.066162109375, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 13.578125 }, { "rank": 2, "tokenId": 236536, "text": "Dangerous", "score": -0.488037109375 }, { "rank": 3, "tokenId": 99510, "text": "Ris", "score": -0.9775390625 }, { "rank": 4, "tokenId": 67185, "text": "BAD", "score": -2.146484375 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": -2.744140625 }, { "rank": 6, "tokenId": 15242, "text": "bad", "score": -3.259765625 }, { "rank": 7, "tokenId": 90046, "text": "Poor", "score": -3.7421875 }, { "rank": 8, "tokenId": 37701, "text": "Ter", "score": -3.765625 } ] } }, "winnerTextsByMode": { "exact": "Bad", "f32_forward": "Bad", "f32_reverse": "Bad", "f32_tree64": "Bad", "f16_forward": "Bad" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Bad" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "flag-true-false-both-oneword-d", "text": "Choose one word only.\nTrue, false, or both: a flag is just cloth", "stableTopK": [ { "rank": 1, "tokenId": 9277, "text": "False", "logit": 17.644136428833008, "prob": 0.9894946813583374 }, { "rank": 2, "tokenId": 1018, "text": "**", "logit": 12.73383617401123, "prob": 0.007292848080396652 }, { "rank": 3, "tokenId": 4339, "text": "True", "logit": 11.234231948852539, "prob": 0.0016278985422104597 }, { "rank": 4, "tokenId": 22186, "text": "Both", "logit": 11.081271171569824, "prob": 0.0013970028376206756 }, { "rank": 5, "tokenId": 4530, "text": "false", "logit": 8.838937759399414, "prob": 0.00014837620256002992 }, { "rank": 6, "tokenId": 27413, "text": "both", "logit": 6.879933834075928, "prob": 0.0000209208665182814 }, { "rank": 7, "tokenId": 95889, "text": "Neither", "logit": 5.2529377937316895, "prob": 0.000004111348516744329 }, { "rank": 8, "tokenId": 25874, "text": "FALSE", "logit": 4.087106227874756, "prob": 0.000001281356730942207 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 20.24442927642191, "winnerGap": 6.650793247945781, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 20.24442927642191 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 13.59363602847613 }, { "rank": 3, "tokenId": 4339, "text": "True", "score": 11.808515461404443 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 11.631005844987481 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": 9.108908237169677 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": 7.00449980363058 }, { "rank": 7, "tokenId": 95889, "text": "Neither", "score": 5.307633947551153 }, { "rank": 8, "tokenId": 25874, "text": "FALSE", "score": 4.112679290749806 } ] }, "f32_forward": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 20.244434356689453, "winnerGap": 6.650791168212891, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 20.244434356689453 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 13.593643188476562 }, { "rank": 3, "tokenId": 4339, "text": "True", "score": 11.808512687683105 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 11.631022453308105 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": 9.108909606933594 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": 7.004498481750488 }, { "rank": 7, "tokenId": 95889, "text": "Neither", "score": 5.30763578414917 }, { "rank": 8, "tokenId": 25874, "text": "FALSE", "score": 4.112677097320557 } ] }, "f32_reverse": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 20.244415283203125, "winnerGap": 6.650777816772461, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 20.244415283203125 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 13.593637466430664 }, { "rank": 3, "tokenId": 4339, "text": "True", "score": 11.808517456054688 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 11.631011009216309 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": 9.108904838562012 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": 7.0044989585876465 }, { "rank": 7, "tokenId": 95889, "text": "Neither", "score": 5.307636737823486 }, { "rank": 8, "tokenId": 25874, "text": "FALSE", "score": 4.112678527832031 } ] }, "f32_tree64": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 20.244430541992188, "winnerGap": 6.650794982910156, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 20.244430541992188 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 13.593635559082031 }, { "rank": 3, "tokenId": 4339, "text": "True", "score": 11.808515548706055 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 11.631006240844727 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": 9.108907699584961 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": 7.004499912261963 }, { "rank": 7, "tokenId": 95889, "text": "Neither", "score": 5.307633876800537 }, { "rank": 8, "tokenId": 25874, "text": "FALSE", "score": 4.1126790046691895 } ] }, "f16_forward": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 20.421875, "winnerGap": 6.84375, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 20.421875 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 13.578125 }, { "rank": 3, "tokenId": 4339, "text": "True", "score": 11.734375 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 11.734375 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": 9.0859375 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": 7.07421875 }, { "rank": 7, "tokenId": 95889, "text": "Neither", "score": 5.32421875 }, { "rank": 8, "tokenId": 25874, "text": "FALSE", "score": 4.12109375 } ] } }, "winnerTextsByMode": { "exact": "False", "f32_forward": "False", "f32_reverse": "False", "f32_tree64": "False", "f16_forward": "False" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "False" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "frag-actually-oneword-a", "text": "Answer with exactly one word only.\nActually,", "stableTopK": [ { "rank": 1, "tokenId": 3771, "text": "No", "logit": 7.42824649810791, "prob": 0.8152398467063904 }, { "rank": 2, "tokenId": 68755, "text": "Actually", "logit": 4.968150615692139, "prob": 0.0696433037519455 }, { "rank": 3, "tokenId": 41387, "text": "Unknown", "logit": 4.0827131271362305, "prob": 0.028730204328894615 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "logit": 3.0467042922973633, "prob": 0.010195435956120491 }, { "rank": 5, "tokenId": 38532, "text": "Perhaps", "logit": 3.0176470279693604, "prob": 0.009903447702527046 }, { "rank": 6, "tokenId": 10784, "text": "Yes", "logit": 2.924237012863159, "prob": 0.009020257741212845 }, { "rank": 7, "tokenId": 154745, "text": "Impossible", "logit": 2.68080735206604, "prob": 0.007071292027831078 }, { "rank": 8, "tokenId": 27426, "text": "Begin", "logit": 2.415933609008789, "prob": 0.005425822455435991 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 7.58589561762923, "winnerGap": 2.5715633750719427, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 7.58589561762923 }, { "rank": 2, "tokenId": 68755, "text": "Actually", "score": 5.014332242557288 }, { "rank": 3, "tokenId": 41387, "text": "Unknown", "score": 4.108203895094565 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "score": 3.057242884997697 }, { "rank": 5, "tokenId": 38532, "text": "Perhaps", "score": 3.0278877244948426 }, { "rank": 6, "tokenId": 10784, "text": "Yes", "score": 2.933554965404287 }, { "rank": 7, "tokenId": 154745, "text": "Impossible", "score": 2.687977928747461 }, { "rank": 8, "tokenId": 27426, "text": "Begin", "score": 2.4211783826671436 } ] }, "f32_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 7.585906505584717, "winnerGap": 2.571566104888916, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 7.585906505584717 }, { "rank": 2, "tokenId": 68755, "text": "Actually", "score": 5.014340400695801 }, { "rank": 3, "tokenId": 41387, "text": "Unknown", "score": 4.108208656311035 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "score": 3.0572426319122314 }, { "rank": 5, "tokenId": 38532, "text": "Perhaps", "score": 3.0278897285461426 }, { "rank": 6, "tokenId": 10784, "text": "Yes", "score": 2.9335544109344482 }, { "rank": 7, "tokenId": 154745, "text": "Impossible", "score": 2.687976598739624 }, { "rank": 8, "tokenId": 27426, "text": "Begin", "score": 2.4211792945861816 } ] }, "f32_reverse": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 7.58589506149292, "winnerGap": 2.5715627670288086, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 7.58589506149292 }, { "rank": 2, "tokenId": 68755, "text": "Actually", "score": 5.014332294464111 }, { "rank": 3, "tokenId": 41387, "text": "Unknown", "score": 4.1082000732421875 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "score": 3.0572428703308105 }, { "rank": 5, "tokenId": 38532, "text": "Perhaps", "score": 3.0278851985931396 }, { "rank": 6, "tokenId": 10784, "text": "Yes", "score": 2.93355393409729 }, { "rank": 7, "tokenId": 154745, "text": "Impossible", "score": 2.687978982925415 }, { "rank": 8, "tokenId": 27426, "text": "Begin", "score": 2.421180009841919 } ] }, "f32_tree64": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 7.58589506149292, "winnerGap": 2.571563243865967, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 7.58589506149292 }, { "rank": 2, "tokenId": 68755, "text": "Actually", "score": 5.014331817626953 }, { "rank": 3, "tokenId": 41387, "text": "Unknown", "score": 4.108203411102295 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "score": 3.0572433471679688 }, { "rank": 5, "tokenId": 38532, "text": "Perhaps", "score": 3.027888298034668 }, { "rank": 6, "tokenId": 10784, "text": "Yes", "score": 2.9335551261901855 }, { "rank": 7, "tokenId": 154745, "text": "Impossible", "score": 2.6879777908325195 }, { "rank": 8, "tokenId": 27426, "text": "Begin", "score": 2.4211788177490234 } ] }, "f16_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 7.6171875, "winnerGap": 2.515625, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 7.6171875 }, { "rank": 2, "tokenId": 68755, "text": "Actually", "score": 5.1015625 }, { "rank": 3, "tokenId": 41387, "text": "Unknown", "score": 4.125 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "score": 3.015625 }, { "rank": 5, "tokenId": 38532, "text": "Perhaps", "score": 2.9921875 }, { "rank": 6, "tokenId": 10784, "text": "Yes", "score": 2.9296875 }, { "rank": 7, "tokenId": 154745, "text": "Impossible", "score": 2.6875 }, { "rank": 8, "tokenId": 27426, "text": "Begin", "score": 2.44921875 } ] } }, "winnerTextsByMode": { "exact": "No", "f32_forward": "No", "f32_reverse": "No", "f32_tree64": "No", "f16_forward": "No" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "No" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "frag-but-oneword-b", "text": "Respond with one word only.\nYes, but", "stableTopK": [ { "rank": 1, "tokenId": 11355, "text": "Why", "logit": -2.3745017051696777, "prob": 0.4588918089866638 }, { "rank": 2, "tokenId": 9366, "text": "Please", "logit": -3.1383180618286133, "prob": 0.21379081904888153 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "logit": -3.156726121902466, "prob": 0.20989134907722473 }, { "rank": 4, "tokenId": 34230, "text": "Continue", "logit": -5.674445152282715, "prob": 0.016926340758800507 }, { "rank": 5, "tokenId": 31402, "text": "Maybe", "logit": -5.938118934631348, "prob": 0.013003222644329071 }, { "rank": 6, "tokenId": 3910, "text": "How", "logit": -6.135143756866455, "prob": 0.010677860118448734 }, { "rank": 7, "tokenId": 14713, "text": "Under", "logit": -6.16774845123291, "prob": 0.010335325263440609 }, { "rank": 8, "tokenId": 155122, "text": "Explain", "logit": -6.292929172515869, "prob": 0.009119245223701 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 11355, "winnerText": "Why", "winnerScore": -2.379474671864937, "winnerGap": 0.7703672650235625, "top": [ { "rank": 1, "tokenId": 11355, "text": "Why", "score": -2.379474671864937 }, { "rank": 2, "tokenId": 9366, "text": "Please", "score": -3.1498419368884996 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": -3.168449728877814 }, { "rank": 4, "tokenId": 34230, "text": "Continue", "score": -5.743606257922195 }, { "rank": 5, "tokenId": 31402, "text": "Maybe", "score": -6.0175426280650015 }, { "rank": 6, "tokenId": 3910, "text": "How", "score": -6.222884844139056 }, { "rank": 7, "tokenId": 14713, "text": "Under", "score": -6.25692044310486 }, { "rank": 8, "tokenId": 155122, "text": "Explain", "score": -6.38774496185906 } ] }, "f32_forward": { "winnerTokenId": 11355, "winnerText": "Why", "winnerScore": -2.3794753551483154, "winnerGap": 0.7703640460968018, "top": [ { "rank": 1, "tokenId": 11355, "text": "Why", "score": -2.3794753551483154 }, { "rank": 2, "tokenId": 9366, "text": "Please", "score": -3.149839401245117 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": -3.1684491634368896 }, { "rank": 4, "tokenId": 34230, "text": "Continue", "score": -5.74360990524292 }, { "rank": 5, "tokenId": 31402, "text": "Maybe", "score": -6.017546653747559 }, { "rank": 6, "tokenId": 3910, "text": "How", "score": -6.222883224487305 }, { "rank": 7, "tokenId": 14713, "text": "Under", "score": -6.256921291351318 }, { "rank": 8, "tokenId": 155122, "text": "Explain", "score": -6.387750625610352 } ] }, "f32_reverse": { "winnerTokenId": 11355, "winnerText": "Why", "winnerScore": -2.3794801235198975, "winnerGap": 0.770359992980957, "top": [ { "rank": 1, "tokenId": 11355, "text": "Why", "score": -2.3794801235198975 }, { "rank": 2, "tokenId": 9366, "text": "Please", "score": -3.1498401165008545 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": -3.1684415340423584 }, { "rank": 4, "tokenId": 34230, "text": "Continue", "score": -5.743598937988281 }, { "rank": 5, "tokenId": 31402, "text": "Maybe", "score": -6.017541408538818 }, { "rank": 6, "tokenId": 3910, "text": "How", "score": -6.222888946533203 }, { "rank": 7, "tokenId": 14713, "text": "Under", "score": -6.2569122314453125 }, { "rank": 8, "tokenId": 155122, "text": "Explain", "score": -6.387754917144775 } ] }, "f32_tree64": { "winnerTokenId": 11355, "winnerText": "Why", "winnerScore": -2.37947416305542, "winnerGap": 0.7703678607940674, "top": [ { "rank": 1, "tokenId": 11355, "text": "Why", "score": -2.37947416305542 }, { "rank": 2, "tokenId": 9366, "text": "Please", "score": -3.1498420238494873 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": -3.168449640274048 }, { "rank": 4, "tokenId": 34230, "text": "Continue", "score": -5.7436065673828125 }, { "rank": 5, "tokenId": 31402, "text": "Maybe", "score": -6.017543315887451 }, { "rank": 6, "tokenId": 3910, "text": "How", "score": -6.222885608673096 }, { "rank": 7, "tokenId": 14713, "text": "Under", "score": -6.25692081451416 }, { "rank": 8, "tokenId": 155122, "text": "Explain", "score": -6.387743949890137 } ] }, "f16_forward": { "winnerTokenId": 20470, "winnerText": "Wait", "winnerScore": -2.12890625, "winnerGap": 0.263671875, "top": [ { "rank": 1, "tokenId": 20470, "text": "Wait", "score": -2.12890625 }, { "rank": 2, "tokenId": 11355, "text": "Why", "score": -2.392578125 }, { "rank": 3, "tokenId": 9366, "text": "Please", "score": -3.1328125 }, { "rank": 4, "tokenId": 47875, "text": "Reason", "score": -4.40234375 }, { "rank": 5, "tokenId": 30340, "text": "Und", "score": -4.44140625 }, { "rank": 6, "tokenId": 3910, "text": "How", "score": -5.2109375 }, { "rank": 7, "tokenId": 34230, "text": "Continue", "score": -5.734375 }, { "rank": 8, "tokenId": 27426, "text": "Begin", "score": -5.75390625 } ] } }, "winnerTextsByMode": { "exact": "Why", "f32_forward": "Why", "f32_reverse": "Why", "f32_tree64": "Why", "f16_forward": "Wait" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Why", "Wait" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 11355, 106 ], "decodedText": "Why", "steps": [ { "step": 0, "tokenId": 11355, "text": "Why" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 20470, 106 ], "decodedText": "Wait", "steps": [ { "step": 0, "tokenId": 20470, "text": "Wait" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "frag-dot-dot-oneword-c", "text": "No explanation. One word only.\n...", "stableTopK": [ { "rank": 1, "tokenId": 186741, "text": "Silence", "logit": 1.5463756322860718, "prob": 0.419822633266449 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "logit": 0.8892090916633606, "prob": 0.21760156750679016 }, { "rank": 3, "tokenId": 14713, "text": "Under", "logit": 0.7606664299964905, "prob": 0.19135361909866333 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "logit": -0.17488984763622284, "prob": 0.07508094608783722 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "logit": -0.4464964270591736, "prob": 0.05722324922680855 }, { "rank": 6, "tokenId": 136657, "text": "Silent", "logit": -2.305922746658325, "prob": 0.00891320500522852 }, { "rank": 7, "tokenId": 17348, "text": "Stop", "logit": -2.480212688446045, "prob": 0.007487567141652107 }, { "rank": 8, "tokenId": 60776, "text": "Waiting", "logit": -3.0464649200439453, "prob": 0.004250309430062771 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 186741, "winnerText": "Silence", "winnerScore": 1.5477479010676358, "winnerGap": 0.6582771337744173, "top": [ { "rank": 1, "tokenId": 186741, "text": "Silence", "score": 1.5477479010676358 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 0.8894707672932185 }, { "rank": 3, "tokenId": 14713, "text": "Under", "score": 0.7608295795135156 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": -0.17489139261573428 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "score": -0.4465284330277335 }, { "rank": 6, "tokenId": 136657, "text": "Silent", "score": -2.310486333947928 }, { "rank": 7, "tokenId": 17348, "text": "Stop", "score": -2.485888771011341 }, { "rank": 8, "tokenId": 60776, "text": "Waiting", "score": -3.057003028312675 } ] }, "f32_forward": { "winnerTokenId": 186741, "winnerText": "Silence", "winnerScore": 1.5477508306503296, "winnerGap": 0.6582797765731812, "top": [ { "rank": 1, "tokenId": 186741, "text": "Silence", "score": 1.5477508306503296 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 0.8894710540771484 }, { "rank": 3, "tokenId": 14713, "text": "Under", "score": 0.7608293890953064 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": -0.17489439249038696 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "score": -0.44653111696243286 }, { "rank": 6, "tokenId": 136657, "text": "Silent", "score": -2.310483932495117 }, { "rank": 7, "tokenId": 17348, "text": "Stop", "score": -2.4858906269073486 }, { "rank": 8, "tokenId": 60776, "text": "Waiting", "score": -3.0570056438446045 } ] }, "f32_reverse": { "winnerTokenId": 186741, "winnerText": "Silence", "winnerScore": 1.547746181488037, "winnerGap": 0.6582774519920349, "top": [ { "rank": 1, "tokenId": 186741, "text": "Silence", "score": 1.547746181488037 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 0.8894687294960022 }, { "rank": 3, "tokenId": 14713, "text": "Under", "score": 0.760829746723175 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": -0.17489586770534515 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "score": -0.44653046131134033 }, { "rank": 6, "tokenId": 136657, "text": "Silent", "score": -2.3104872703552246 }, { "rank": 7, "tokenId": 17348, "text": "Stop", "score": -2.4858899116516113 }, { "rank": 8, "tokenId": 60776, "text": "Waiting", "score": -3.057003974914551 } ] }, "f32_tree64": { "winnerTokenId": 186741, "winnerText": "Silence", "winnerScore": 1.54774808883667, "winnerGap": 0.6582773923873901, "top": [ { "rank": 1, "tokenId": 186741, "text": "Silence", "score": 1.54774808883667 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 0.8894706964492798 }, { "rank": 3, "tokenId": 14713, "text": "Under", "score": 0.7608293294906616 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": -0.17489075660705566 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "score": -0.4465283751487732 }, { "rank": 6, "tokenId": 136657, "text": "Silent", "score": -2.310486078262329 }, { "rank": 7, "tokenId": 17348, "text": "Stop", "score": -2.4858880043029785 }, { "rank": 8, "tokenId": 60776, "text": "Waiting", "score": -3.0570030212402344 } ] }, "f16_forward": { "winnerTokenId": 186741, "winnerText": "Silence", "winnerScore": 1.5712890625, "winnerGap": 0.640625, "top": [ { "rank": 1, "tokenId": 186741, "text": "Silence", "score": 1.5712890625 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 0.9306640625 }, { "rank": 3, "tokenId": 14713, "text": "Under", "score": 0.765625 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": -0.1746826171875 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "score": -0.178466796875 }, { "rank": 6, "tokenId": 136657, "text": "Silent", "score": -2.31640625 }, { "rank": 7, "tokenId": 17348, "text": "Stop", "score": -2.490234375 }, { "rank": 8, "tokenId": 60776, "text": "Waiting", "score": -3.05078125 } ] } }, "winnerTextsByMode": { "exact": "Silence", "f32_forward": "Silence", "f32_reverse": "Silence", "f32_tree64": "Silence", "f16_forward": "Silence" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Silence" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "frag-however-oneword-d", "text": "Choose one word only.\nIt was good. However,", "stableTopK": [ { "rank": 1, "tokenId": 9675, "text": "However", "logit": 18.225749969482422, "prob": 0.9973049759864807 }, { "rank": 2, "tokenId": 97485, "text": "however", "logit": 12.21107292175293, "prob": 0.002436053939163685 }, { "rank": 3, "tokenId": 5503, "text": "but", "logit": 9.075811386108398, "prob": 0.00010594004561426118 }, { "rank": 4, "tokenId": 4573, "text": "But", "logit": 8.715312957763672, "prob": 0.00007387503137579188 }, { "rank": 5, "tokenId": 121588, "text": "Namun", "logit": 7.297143936157227, "prob": 0.000017889355149236508 }, { "rank": 6, "tokenId": 1018, "text": "**", "logit": 7.288817405700684, "prob": 0.000017741018382366747 }, { "rank": 7, "tokenId": 71115, "text": "Nevertheless", "logit": 6.698945045471191, "prob": 0.000009835584933171049 }, { "rank": 8, "tokenId": 3153, "text": " However", "logit": 6.544910907745361, "prob": 0.000008431484275206458 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 9675, "winnerText": "However", "winnerScore": 21.149672845270526, "winnerGap": 8.188075919330753, "top": [ { "rank": 1, "tokenId": 9675, "text": "However", "score": 21.149672845270526 }, { "rank": 2, "tokenId": 97485, "text": "however", "score": 12.961596925939773 }, { "rank": 3, "tokenId": 5503, "text": "but", "score": 9.368966743729764 }, { "rank": 4, "tokenId": 4573, "text": "But", "score": 8.973709594769034 }, { "rank": 5, "tokenId": 121588, "text": "Namun", "score": 7.446387868619411 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 7.437540568398646 }, { "rank": 7, "tokenId": 71115, "text": "Nevertheless", "score": 6.813742210329558 }, { "rank": 8, "tokenId": 3153, "text": " However", "score": 6.6518190352450475 } ] }, "f32_forward": { "winnerTokenId": 9675, "winnerText": "However", "winnerScore": 21.149675369262695, "winnerGap": 8.188069343566895, "top": [ { "rank": 1, "tokenId": 9675, "text": "However", "score": 21.149675369262695 }, { "rank": 2, "tokenId": 97485, "text": "however", "score": 12.9616060256958 }, { "rank": 3, "tokenId": 5503, "text": "but", "score": 9.368955612182617 }, { "rank": 4, "tokenId": 4573, "text": "But", "score": 8.973712921142578 }, { "rank": 5, "tokenId": 121588, "text": "Namun", "score": 7.446403503417969 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 7.437524795532227 }, { "rank": 7, "tokenId": 71115, "text": "Nevertheless", "score": 6.81373929977417 }, { "rank": 8, "tokenId": 3153, "text": " However", "score": 6.651813507080078 } ] }, "f32_reverse": { "winnerTokenId": 9675, "winnerText": "However", "winnerScore": 21.149694442749023, "winnerGap": 8.188096046447754, "top": [ { "rank": 1, "tokenId": 9675, "text": "However", "score": 21.149694442749023 }, { "rank": 2, "tokenId": 97485, "text": "however", "score": 12.96159839630127 }, { "rank": 3, "tokenId": 5503, "text": "but", "score": 9.368962287902832 }, { "rank": 4, "tokenId": 4573, "text": "But", "score": 8.973709106445312 }, { "rank": 5, "tokenId": 121588, "text": "Namun", "score": 7.446386814117432 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 7.437541961669922 }, { "rank": 7, "tokenId": 71115, "text": "Nevertheless", "score": 6.813746929168701 }, { "rank": 8, "tokenId": 3153, "text": " However", "score": 6.651822090148926 } ] }, "f32_tree64": { "winnerTokenId": 9675, "winnerText": "However", "winnerScore": 21.149673461914062, "winnerGap": 8.188078880310059, "top": [ { "rank": 1, "tokenId": 9675, "text": "However", "score": 21.149673461914062 }, { "rank": 2, "tokenId": 97485, "text": "however", "score": 12.961594581604004 }, { "rank": 3, "tokenId": 5503, "text": "but", "score": 9.368967056274414 }, { "rank": 4, "tokenId": 4573, "text": "But", "score": 8.973709106445312 }, { "rank": 5, "tokenId": 121588, "text": "Namun", "score": 7.446388244628906 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 7.437540054321289 }, { "rank": 7, "tokenId": 71115, "text": "Nevertheless", "score": 6.81374454498291 }, { "rank": 8, "tokenId": 3153, "text": " However", "score": 6.65181827545166 } ] }, "f16_forward": { "winnerTokenId": 9675, "winnerText": "However", "winnerScore": 21.34375, "winnerGap": 11.9296875, "top": [ { "rank": 1, "tokenId": 9675, "text": "However", "score": 21.34375 }, { "rank": 2, "tokenId": 5503, "text": "but", "score": 9.4140625 }, { "rank": 3, "tokenId": 97485, "text": "however", "score": 8.96875 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 7.4375 }, { "rank": 5, "tokenId": 121588, "text": "Namun", "score": 7.41796875 }, { "rank": 6, "tokenId": 71115, "text": "Nevertheless", "score": 6.87109375 }, { "rank": 7, "tokenId": 3153, "text": " However", "score": 6.6484375 }, { "rank": 8, "tokenId": 42462, "text": "although", "score": 6.39453125 } ] } }, "winnerTextsByMode": { "exact": "However", "f32_forward": "However", "f32_reverse": "However", "f32_tree64": "However", "f16_forward": "However" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "However" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "frag-letter-next-oneword-a", "text": "Answer with exactly one word only.\nA, B, C, D,", "stableTopK": [ { "rank": 1, "tokenId": 236776, "text": "A", "logit": 8.384835243225098, "prob": 0.8130069971084595 }, { "rank": 2, "tokenId": 9336, "text": "None", "logit": 5.949026107788086, "prob": 0.0711599811911583 }, { "rank": 3, "tokenId": 236799, "text": "B", "logit": 4.933791637420654, "prob": 0.025782505050301552 }, { "rank": 4, "tokenId": 236780, "text": "C", "logit": 4.518251895904541, "prob": 0.017016038298606873 }, { "rank": 5, "tokenId": 236796, "text": "D", "logit": 4.5032958984375, "prob": 0.016763441264629364 }, { "rank": 6, "tokenId": 154745, "text": "Impossible", "logit": 4.499495506286621, "prob": 0.01669985242187977 }, { "rank": 7, "tokenId": 9366, "text": "Please", "logit": 4.03924560546875, "prob": 0.010539710521697998 }, { "rank": 8, "tokenId": 58686, "text": "Cannot", "logit": 3.368452310562134, "prob": 0.005388983525335789 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 236776, "winnerText": "A", "winnerScore": 8.614005914167983, "winnerGap": 2.585109156361831, "top": [ { "rank": 1, "tokenId": 236776, "text": "A", "score": 8.614005914167983 }, { "rank": 2, "tokenId": 9336, "text": "None", "score": 6.028896757806152 }, { "rank": 3, "tokenId": 236799, "text": "B", "score": 4.979009245775899 }, { "rank": 4, "tokenId": 236780, "text": "C", "score": 4.552886661600256 }, { "rank": 5, "tokenId": 236796, "text": "D", "score": 4.537582660690234 }, { "rank": 6, "tokenId": 154745, "text": "Impossible", "score": 4.5336973737539665 }, { "rank": 7, "tokenId": 9366, "text": "Please", "score": 4.063921930923534 }, { "rank": 8, "tokenId": 58686, "text": "Cannot", "score": 3.3827157450987535 } ] }, "f32_forward": { "winnerTokenId": 236776, "winnerText": "A", "winnerScore": 8.614009857177734, "winnerGap": 2.585111618041992, "top": [ { "rank": 1, "tokenId": 236776, "text": "A", "score": 8.614009857177734 }, { "rank": 2, "tokenId": 9336, "text": "None", "score": 6.028898239135742 }, { "rank": 3, "tokenId": 236799, "text": "B", "score": 4.979008197784424 }, { "rank": 4, "tokenId": 236780, "text": "C", "score": 4.552886009216309 }, { "rank": 5, "tokenId": 236796, "text": "D", "score": 4.537590026855469 }, { "rank": 6, "tokenId": 154745, "text": "Impossible", "score": 4.533698081970215 }, { "rank": 7, "tokenId": 9366, "text": "Please", "score": 4.063920974731445 }, { "rank": 8, "tokenId": 58686, "text": "Cannot", "score": 3.3827178478240967 } ] }, "f32_reverse": { "winnerTokenId": 236776, "winnerText": "A", "winnerScore": 8.613998413085938, "winnerGap": 2.585102081298828, "top": [ { "rank": 1, "tokenId": 236776, "text": "A", "score": 8.613998413085938 }, { "rank": 2, "tokenId": 9336, "text": "None", "score": 6.028896331787109 }, { "rank": 3, "tokenId": 236799, "text": "B", "score": 4.97900915145874 }, { "rank": 4, "tokenId": 236780, "text": "C", "score": 4.552885055541992 }, { "rank": 5, "tokenId": 236796, "text": "D", "score": 4.537588119506836 }, { "rank": 6, "tokenId": 154745, "text": "Impossible", "score": 4.533700942993164 }, { "rank": 7, "tokenId": 9366, "text": "Please", "score": 4.0639166831970215 }, { "rank": 8, "tokenId": 58686, "text": "Cannot", "score": 3.3827168941497803 } ] }, "f32_tree64": { "winnerTokenId": 236776, "winnerText": "A", "winnerScore": 8.614006042480469, "winnerGap": 2.5851097106933594, "top": [ { "rank": 1, "tokenId": 236776, "text": "A", "score": 8.614006042480469 }, { "rank": 2, "tokenId": 9336, "text": "None", "score": 6.028896331787109 }, { "rank": 3, "tokenId": 236799, "text": "B", "score": 4.97900915145874 }, { "rank": 4, "tokenId": 236780, "text": "C", "score": 4.552886009216309 }, { "rank": 5, "tokenId": 236796, "text": "D", "score": 4.5375823974609375 }, { "rank": 6, "tokenId": 154745, "text": "Impossible", "score": 4.533697128295898 }, { "rank": 7, "tokenId": 9366, "text": "Please", "score": 4.063921928405762 }, { "rank": 8, "tokenId": 58686, "text": "Cannot", "score": 3.3827157020568848 } ] }, "f16_forward": { "winnerTokenId": 236776, "winnerText": "A", "winnerScore": 8.6953125, "winnerGap": 2.84375, "top": [ { "rank": 1, "tokenId": 236776, "text": "A", "score": 8.6953125 }, { "rank": 2, "tokenId": 9336, "text": "None", "score": 5.8515625 }, { "rank": 3, "tokenId": 236799, "text": "B", "score": 5.00390625 }, { "rank": 4, "tokenId": 236780, "text": "C", "score": 4.64453125 }, { "rank": 5, "tokenId": 154745, "text": "Impossible", "score": 4.5625 }, { "rank": 6, "tokenId": 236796, "text": "D", "score": 4.4921875 }, { "rank": 7, "tokenId": 58686, "text": "Cannot", "score": 3.39453125 }, { "rank": 8, "tokenId": 41387, "text": "Unknown", "score": 3.15234375 } ] } }, "winnerTextsByMode": { "exact": "A", "f32_forward": "A", "f32_reverse": "A", "f32_tree64": "A", "f16_forward": "A" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "A" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "frag-newline-only-oneword-b", "text": "Respond with one word only.\n\n", "stableTopK": [ { "rank": 1, "tokenId": 10784, "text": "Yes", "logit": 6.913684368133545, "prob": 0.8837133049964905 }, { "rank": 2, "tokenId": 14713, "text": "Under", "logit": 4.782489776611328, "prob": 0.10489271581172943 }, { "rank": 3, "tokenId": 19058, "text": "Okay", "logit": 2.5569918155670166, "prob": 0.011329841800034046 }, { "rank": 4, "tokenId": 7676, "text": "OK", "logit": -3.1238863468170166, "prob": 0.00003864113023155369 }, { "rank": 5, "tokenId": 33778, "text": "Ready", "logit": -4.310482978820801, "prob": 0.000011795529644587077 }, { "rank": 6, "tokenId": 20470, "text": "Wait", "logit": -4.991349697113037, "prob": 0.000005970638540020445 }, { "rank": 7, "tokenId": 17348, "text": "Stop", "logit": -5.889822006225586, "prob": 0.000002431191660434706 }, { "rank": 8, "tokenId": 9259, "text": "Hello", "logit": -6.59474515914917, "prob": 0.0000012013650803055498 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 10784, "winnerText": "Yes", "winnerScore": 7.0401322086984734, "winnerGap": 2.2165011613413483, "top": [ { "rank": 1, "tokenId": 10784, "text": "Yes", "score": 7.0401322086984734 }, { "rank": 2, "tokenId": 14713, "text": "Under", "score": 4.823631047357125 }, { "rank": 3, "tokenId": 19058, "text": "Okay", "score": 2.5632092101230626 }, { "rank": 4, "tokenId": 7676, "text": "OK", "score": -3.1352501982677055 }, { "rank": 5, "tokenId": 33778, "text": "Ready", "score": -4.340517563252234 }, { "rank": 6, "tokenId": 20470, "text": "Wait", "score": -5.038189610359694 }, { "rank": 7, "tokenId": 17348, "text": "Stop", "score": -5.967294399795719 }, { "rank": 8, "tokenId": 9259, "text": "Hello", "score": -6.704159628592265 } ] }, "f32_forward": { "winnerTokenId": 10784, "winnerText": "Yes", "winnerScore": 7.040126323699951, "winnerGap": 2.216506004333496, "top": [ { "rank": 1, "tokenId": 10784, "text": "Yes", "score": 7.040126323699951 }, { "rank": 2, "tokenId": 14713, "text": "Under", "score": 4.823620319366455 }, { "rank": 3, "tokenId": 19058, "text": "Okay", "score": 2.563204765319824 }, { "rank": 4, "tokenId": 7676, "text": "OK", "score": -3.1352522373199463 }, { "rank": 5, "tokenId": 33778, "text": "Ready", "score": -4.340518474578857 }, { "rank": 6, "tokenId": 20470, "text": "Wait", "score": -5.038185119628906 }, { "rank": 7, "tokenId": 17348, "text": "Stop", "score": -5.967294692993164 }, { "rank": 8, "tokenId": 9259, "text": "Hello", "score": -6.704163074493408 } ] }, "f32_reverse": { "winnerTokenId": 10784, "winnerText": "Yes", "winnerScore": 7.040130138397217, "winnerGap": 2.2164950370788574, "top": [ { "rank": 1, "tokenId": 10784, "text": "Yes", "score": 7.040130138397217 }, { "rank": 2, "tokenId": 14713, "text": "Under", "score": 4.823635101318359 }, { "rank": 3, "tokenId": 19058, "text": "Okay", "score": 2.563206434249878 }, { "rank": 4, "tokenId": 7676, "text": "OK", "score": -3.1352498531341553 }, { "rank": 5, "tokenId": 33778, "text": "Ready", "score": -4.340512752532959 }, { "rank": 6, "tokenId": 20470, "text": "Wait", "score": -5.038183689117432 }, { "rank": 7, "tokenId": 17348, "text": "Stop", "score": -5.967290878295898 }, { "rank": 8, "tokenId": 9259, "text": "Hello", "score": -6.704157829284668 } ] }, "f32_tree64": { "winnerTokenId": 10784, "winnerText": "Yes", "winnerScore": 7.04013204574585, "winnerGap": 2.216500759124756, "top": [ { "rank": 1, "tokenId": 10784, "text": "Yes", "score": 7.04013204574585 }, { "rank": 2, "tokenId": 14713, "text": "Under", "score": 4.823631286621094 }, { "rank": 3, "tokenId": 19058, "text": "Okay", "score": 2.563209056854248 }, { "rank": 4, "tokenId": 7676, "text": "OK", "score": -3.1352500915527344 }, { "rank": 5, "tokenId": 33778, "text": "Ready", "score": -4.340516567230225 }, { "rank": 6, "tokenId": 20470, "text": "Wait", "score": -5.038189888000488 }, { "rank": 7, "tokenId": 17348, "text": "Stop", "score": -5.967294692993164 }, { "rank": 8, "tokenId": 9259, "text": "Hello", "score": -6.704159259796143 } ] }, "f16_forward": { "winnerTokenId": 14713, "winnerText": "Under", "winnerScore": 5.04296875, "winnerGap": 1.927734375, "top": [ { "rank": 1, "tokenId": 14713, "text": "Under", "score": 5.04296875 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 3.115234375 }, { "rank": 3, "tokenId": 19058, "text": "Okay", "score": 2.755859375 }, { "rank": 4, "tokenId": 7676, "text": "OK", "score": -3.0390625 }, { "rank": 5, "tokenId": 33778, "text": "Ready", "score": -4.48046875 }, { "rank": 6, "tokenId": 8574, "text": "Can", "score": -4.52734375 }, { "rank": 7, "tokenId": 17348, "text": "Stop", "score": -5.84375 }, { "rank": 8, "tokenId": 20470, "text": "Wait", "score": -5.86328125 } ] } }, "winnerTextsByMode": { "exact": "Yes", "f32_forward": "Yes", "f32_reverse": "Yes", "f32_tree64": "Yes", "f16_forward": "Under" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Yes", "Under" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 10784, 106 ], "decodedText": "Yes", "steps": [ { "step": 0, "tokenId": 10784, "text": "Yes" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 14713, 12450 ], "decodedText": "Understood", "steps": [ { "step": 0, "tokenId": 14713, "text": "Under" }, { "step": 1, "tokenId": 12450, "text": "stood" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "frag-no-oneword-c", "text": "No explanation. One word only.\nNo.", "stableTopK": [ { "rank": 1, "tokenId": 14713, "text": "Under", "logit": 8.80876636505127, "prob": 0.6494560837745667 }, { "rank": 2, "tokenId": 34318, "text": "Correct", "logit": 6.760928630828857, "prob": 0.08378864824771881 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "logit": 6.578032493591309, "prob": 0.0697837620973587 }, { "rank": 4, "tokenId": 19058, "text": "Okay", "logit": 6.279091835021973, "prob": 0.05175187438726425 }, { "rank": 5, "tokenId": 17348, "text": "Stop", "logit": 5.9008331298828125, "prob": 0.035452790558338165 }, { "rank": 6, "tokenId": 20470, "text": "Wait", "logit": 4.744269847869873, "prob": 0.011152220889925957 }, { "rank": 7, "tokenId": 4339, "text": "True", "logit": 4.701300144195557, "prob": 0.010683163069188595 }, { "rank": 8, "tokenId": 106, "text": "", "logit": 4.688281059265137, "prob": 0.010544979944825172 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 14713, "winnerText": "Under", "winnerScore": 9.075877430099297, "winnerGap": 2.1968688632861557, "top": [ { "rank": 1, "tokenId": 14713, "text": "Under", "score": 9.075877430099297 }, { "rank": 2, "tokenId": 34318, "text": "Correct", "score": 6.879008566813141 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 6.686603786584378 }, { "rank": 4, "tokenId": 19058, "text": "Okay", "score": 6.373273319049552 }, { "rank": 5, "tokenId": 17348, "text": "Stop", "score": 5.978751254954872 }, { "rank": 6, "tokenId": 20470, "text": "Wait", "score": 4.7844207034217945 }, { "rank": 7, "tokenId": 4339, "text": "True", "score": 4.740365609746675 }, { "rank": 8, "tokenId": 106, "text": "", "score": 4.72701847724025 } ] }, "f32_forward": { "winnerTokenId": 14713, "winnerText": "Under", "winnerScore": 9.075878143310547, "winnerGap": 2.196873188018799, "top": [ { "rank": 1, "tokenId": 14713, "text": "Under", "score": 9.075878143310547 }, { "rank": 2, "tokenId": 34318, "text": "Correct", "score": 6.879004955291748 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 6.68660831451416 }, { "rank": 4, "tokenId": 19058, "text": "Okay", "score": 6.373274803161621 }, { "rank": 5, "tokenId": 17348, "text": "Stop", "score": 5.978757381439209 }, { "rank": 6, "tokenId": 20470, "text": "Wait", "score": 4.784425258636475 }, { "rank": 7, "tokenId": 4339, "text": "True", "score": 4.740360260009766 }, { "rank": 8, "tokenId": 106, "text": "", "score": 4.727021217346191 } ] }, "f32_reverse": { "winnerTokenId": 14713, "winnerText": "Under", "winnerScore": 9.075879096984863, "winnerGap": 2.196876049041748, "top": [ { "rank": 1, "tokenId": 14713, "text": "Under", "score": 9.075879096984863 }, { "rank": 2, "tokenId": 34318, "text": "Correct", "score": 6.879003047943115 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 6.686602592468262 }, { "rank": 4, "tokenId": 19058, "text": "Okay", "score": 6.373276710510254 }, { "rank": 5, "tokenId": 17348, "text": "Stop", "score": 5.978748321533203 }, { "rank": 6, "tokenId": 20470, "text": "Wait", "score": 4.784419059753418 }, { "rank": 7, "tokenId": 4339, "text": "True", "score": 4.740363121032715 }, { "rank": 8, "tokenId": 106, "text": "", "score": 4.7270188331604 } ] }, "f32_tree64": { "winnerTokenId": 14713, "winnerText": "Under", "winnerScore": 9.075876235961914, "winnerGap": 2.1968674659729004, "top": [ { "rank": 1, "tokenId": 14713, "text": "Under", "score": 9.075876235961914 }, { "rank": 2, "tokenId": 34318, "text": "Correct", "score": 6.879008769989014 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 6.686604022979736 }, { "rank": 4, "tokenId": 19058, "text": "Okay", "score": 6.373272895812988 }, { "rank": 5, "tokenId": 17348, "text": "Stop", "score": 5.978751182556152 }, { "rank": 6, "tokenId": 20470, "text": "Wait", "score": 4.784420013427734 }, { "rank": 7, "tokenId": 4339, "text": "True", "score": 4.740365982055664 }, { "rank": 8, "tokenId": 106, "text": "", "score": 4.727017402648926 } ] }, "f16_forward": { "winnerTokenId": 34318, "winnerText": "Correct", "winnerScore": 6.79296875, "winnerGap": 0.078125, "top": [ { "rank": 1, "tokenId": 34318, "text": "Correct", "score": 6.79296875 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 6.71484375 }, { "rank": 3, "tokenId": 19058, "text": "Okay", "score": 6.19921875 }, { "rank": 4, "tokenId": 17348, "text": "Stop", "score": 6.03515625 }, { "rank": 5, "tokenId": 14713, "text": "Under", "score": 5.04296875 }, { "rank": 6, "tokenId": 106, "text": "", "score": 4.78515625 }, { "rank": 7, "tokenId": 20470, "text": "Wait", "score": 4.77734375 }, { "rank": 8, "tokenId": 4339, "text": "True", "score": 4.7109375 } ] } }, "winnerTextsByMode": { "exact": "Under", "f32_forward": "Under", "f32_reverse": "Under", "f32_tree64": "Under", "f16_forward": "Correct" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Under", "Correct" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 14713, 12450 ], "decodedText": "Understood", "steps": [ { "step": 0, "tokenId": 14713, "text": "Under" }, { "step": 1, "tokenId": 12450, "text": "stood" } ] }, "f16_forward": { "tokenIds": [ 34318, 106 ], "decodedText": "Correct", "steps": [ { "step": 0, "tokenId": 34318, "text": "Correct" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "frag-null-literal-oneword-d", "text": "Choose one word only.\nnull", "stableTopK": [ { "rank": 1, "tokenId": 6455, "text": "null", "logit": 23.826763153076172, "prob": 0.9978095293045044 }, { "rank": 2, "tokenId": 9647, "text": "Null", "logit": 17.226722717285156, "prob": 0.001357333268970251 }, { "rank": 3, "tokenId": 106, "text": "", "logit": 15.652082443237305, "prob": 0.00028107932303100824 }, { "rank": 4, "tokenId": 1, "text": "", "logit": 14.54583740234375, "prob": 0.00009298069926444441 }, { "rank": 5, "tokenId": 14057, "text": "NULL", "logit": 13.58655071258545, "prob": 0.000035627050237962976 }, { "rank": 6, "tokenId": 2303, "text": " null", "logit": 13.521544456481934, "prob": 0.00003338474198244512 }, { "rank": 7, "tokenId": 63823, "text": "nothing", "logit": 13.364704132080078, "prob": 0.000028538630431285128 }, { "rank": 8, "tokenId": 1018, "text": "**", "logit": 13.30019474029541, "prob": 0.000026755746148410253 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 6455, "winnerText": "null", "winnerScore": 32.48321253276729, "winnerGap": 12.869140068249543, "top": [ { "rank": 1, "tokenId": 6455, "text": "null", "score": 32.48321253276729 }, { "rank": 2, "tokenId": 9647, "text": "Null", "score": 19.614072464517747 }, { "rank": 3, "tokenId": 106, "text": "", "score": 17.36166893547834 }, { "rank": 4, "tokenId": 1, "text": "", "score": 15.87961034721512 }, { "rank": 5, "tokenId": 14057, "text": "NULL", "score": 14.649716192799119 }, { "rank": 6, "tokenId": 2303, "text": " null", "score": 14.568028601400442 }, { "rank": 7, "tokenId": 63823, "text": "nothing", "score": 14.371784442162792 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": 14.291399194123283 } ] }, "f32_forward": { "winnerTokenId": 6455, "winnerText": "null", "winnerScore": 32.483184814453125, "winnerGap": 12.86911392211914, "top": [ { "rank": 1, "tokenId": 6455, "text": "null", "score": 32.483184814453125 }, { "rank": 2, "tokenId": 9647, "text": "Null", "score": 19.614070892333984 }, { "rank": 3, "tokenId": 106, "text": "", "score": 17.361650466918945 }, { "rank": 4, "tokenId": 1, "text": "", "score": 15.879613876342773 }, { "rank": 5, "tokenId": 14057, "text": "NULL", "score": 14.64969253540039 }, { "rank": 6, "tokenId": 2303, "text": " null", "score": 14.56804084777832 }, { "rank": 7, "tokenId": 63823, "text": "nothing", "score": 14.371785163879395 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": 14.291398048400879 } ] }, "f32_reverse": { "winnerTokenId": 6455, "winnerText": "null", "winnerScore": 32.48320007324219, "winnerGap": 12.869117736816406, "top": [ { "rank": 1, "tokenId": 6455, "text": "null", "score": 32.48320007324219 }, { "rank": 2, "tokenId": 9647, "text": "Null", "score": 19.61408233642578 }, { "rank": 3, "tokenId": 106, "text": "", "score": 17.361658096313477 }, { "rank": 4, "tokenId": 1, "text": "", "score": 15.879599571228027 }, { "rank": 5, "tokenId": 14057, "text": "NULL", "score": 14.649714469909668 }, { "rank": 6, "tokenId": 2303, "text": " null", "score": 14.56803035736084 }, { "rank": 7, "tokenId": 63823, "text": "nothing", "score": 14.371780395507812 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": 14.291403770446777 } ] }, "f32_tree64": { "winnerTokenId": 6455, "winnerText": "null", "winnerScore": 32.483211517333984, "winnerGap": 12.869138717651367, "top": [ { "rank": 1, "tokenId": 6455, "text": "null", "score": 32.483211517333984 }, { "rank": 2, "tokenId": 9647, "text": "Null", "score": 19.614072799682617 }, { "rank": 3, "tokenId": 106, "text": "", "score": 17.36166763305664 }, { "rank": 4, "tokenId": 1, "text": "", "score": 15.879610061645508 }, { "rank": 5, "tokenId": 14057, "text": "NULL", "score": 14.649717330932617 }, { "rank": 6, "tokenId": 2303, "text": " null", "score": 14.568029403686523 }, { "rank": 7, "tokenId": 63823, "text": "nothing", "score": 14.371784210205078 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": 14.291400909423828 } ] }, "f16_forward": { "winnerTokenId": 6455, "winnerText": "null", "winnerScore": 28.578125, "winnerGap": 9.203125, "top": [ { "rank": 1, "tokenId": 6455, "text": "null", "score": 28.578125 }, { "rank": 2, "tokenId": 9647, "text": "Null", "score": 19.375 }, { "rank": 3, "tokenId": 106, "text": "", "score": 17.234375 }, { "rank": 4, "tokenId": 1, "text": "", "score": 15.8046875 }, { "rank": 5, "tokenId": 14057, "text": "NULL", "score": 14.625 }, { "rank": 6, "tokenId": 2303, "text": " null", "score": 14.6171875 }, { "rank": 7, "tokenId": 63823, "text": "nothing", "score": 14.53125 }, { "rank": 8, "tokenId": 3494, "text": "Error", "score": 14.3828125 } ] } }, "winnerTextsByMode": { "exact": "null", "f32_forward": "null", "f32_reverse": "null", "f32_tree64": "null", "f16_forward": "null" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "null" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "frag-number-next-oneword-a", "text": "Answer with exactly one word only.\n1, 1, 2, 3, 5, 8,", "stableTopK": [ { "rank": 1, "tokenId": 236770, "text": "1", "logit": -2.4904675483703613, "prob": 0.7213101983070374 }, { "rank": 2, "tokenId": 1214, "text": "Th", "logit": -3.529947280883789, "prob": 0.2550831437110901 }, { "rank": 3, "tokenId": 594, "text": "th", "logit": -6.317778587341309, "prob": 0.015701530501246452 }, { "rank": 4, "tokenId": 146228, "text": "Fib", "logit": -8.04967212677002, "prob": 0.00277837086468935 }, { "rank": 5, "tokenId": 80357, "text": "Eight", "logit": -8.847990989685059, "prob": 0.0012505028862506151 }, { "rank": 6, "tokenId": 54119, "text": "Odd", "logit": -9.255573272705078, "prob": 0.0008319054613821208 }, { "rank": 7, "tokenId": 236828, "text": "8", "logit": -9.514527320861816, "prob": 0.0006421132711693645 }, { "rank": 8, "tokenId": 161459, "text": "Pascal", "logit": -10.018399238586426, "prob": 0.00038795635919086635 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 236770, "winnerText": "1", "winnerScore": -2.4962130794371973, "winnerGap": 1.0501641135772246, "top": [ { "rank": 1, "tokenId": 236770, "text": "1", "score": -2.4962130794371973 }, { "rank": 2, "tokenId": 1214, "text": "Th", "score": -3.546377193014422 }, { "rank": 3, "tokenId": 594, "text": "th", "score": -6.413741463276324 }, { "rank": 4, "tokenId": 146228, "text": "Fib", "score": -8.251652196744104 }, { "rank": 5, "tokenId": 80357, "text": "Eight", "score": -9.118816761887679 }, { "rank": 6, "tokenId": 54119, "text": "Odd", "score": -9.567237457411977 }, { "rank": 7, "tokenId": 236828, "text": "8", "score": -9.854288976675775 }, { "rank": 8, "tokenId": 161459, "text": "Pascal", "score": -10.417906776107742 } ] }, "f32_forward": { "winnerTokenId": 236770, "winnerText": "1", "winnerScore": -2.496213674545288, "winnerGap": 1.0501670837402344, "top": [ { "rank": 1, "tokenId": 236770, "text": "1", "score": -2.496213674545288 }, { "rank": 2, "tokenId": 1214, "text": "Th", "score": -3.5463807582855225 }, { "rank": 3, "tokenId": 594, "text": "th", "score": -6.413736820220947 }, { "rank": 4, "tokenId": 146228, "text": "Fib", "score": -8.251652717590332 }, { "rank": 5, "tokenId": 80357, "text": "Eight", "score": -9.118826866149902 }, { "rank": 6, "tokenId": 54119, "text": "Odd", "score": -9.567235946655273 }, { "rank": 7, "tokenId": 236828, "text": "8", "score": -9.854287147521973 }, { "rank": 8, "tokenId": 161459, "text": "Pascal", "score": -10.417915344238281 } ] }, "f32_reverse": { "winnerTokenId": 236770, "winnerText": "1", "winnerScore": -2.4962124824523926, "winnerGap": 1.0501677989959717, "top": [ { "rank": 1, "tokenId": 236770, "text": "1", "score": -2.4962124824523926 }, { "rank": 2, "tokenId": 1214, "text": "Th", "score": -3.5463802814483643 }, { "rank": 3, "tokenId": 594, "text": "th", "score": -6.413745403289795 }, { "rank": 4, "tokenId": 146228, "text": "Fib", "score": -8.251633644104004 }, { "rank": 5, "tokenId": 80357, "text": "Eight", "score": -9.118823051452637 }, { "rank": 6, "tokenId": 54119, "text": "Odd", "score": -9.567237854003906 }, { "rank": 7, "tokenId": 236828, "text": "8", "score": -9.854287147521973 }, { "rank": 8, "tokenId": 161459, "text": "Pascal", "score": -10.417949676513672 } ] }, "f32_tree64": { "winnerTokenId": 236770, "winnerText": "1", "winnerScore": -2.496213912963867, "winnerGap": 1.0501627922058105, "top": [ { "rank": 1, "tokenId": 236770, "text": "1", "score": -2.496213912963867 }, { "rank": 2, "tokenId": 1214, "text": "Th", "score": -3.5463767051696777 }, { "rank": 3, "tokenId": 594, "text": "th", "score": -6.413741111755371 }, { "rank": 4, "tokenId": 146228, "text": "Fib", "score": -8.251653671264648 }, { "rank": 5, "tokenId": 80357, "text": "Eight", "score": -9.118816375732422 }, { "rank": 6, "tokenId": 54119, "text": "Odd", "score": -9.56723690032959 }, { "rank": 7, "tokenId": 236828, "text": "8", "score": -9.854289054870605 }, { "rank": 8, "tokenId": 161459, "text": "Pascal", "score": -10.417906761169434 } ] }, "f16_forward": { "winnerTokenId": 594, "winnerText": "th", "winnerScore": -2.462890625, "winnerGap": 0.01171875, "top": [ { "rank": 1, "tokenId": 594, "text": "th", "score": -2.462890625 }, { "rank": 2, "tokenId": 236770, "text": "1", "score": -2.474609375 }, { "rank": 3, "tokenId": 146228, "text": "Fib", "score": -4.51953125 }, { "rank": 4, "tokenId": 80357, "text": "Eight", "score": -5.12109375 }, { "rank": 5, "tokenId": 1214, "text": "Th", "score": -7.546875 }, { "rank": 6, "tokenId": 3810, "text": "There", "score": -8.3046875 }, { "rank": 7, "tokenId": 236828, "text": "8", "score": -8.796875 }, { "rank": 8, "tokenId": 48084, "text": "Positive", "score": -9.125 } ] } }, "winnerTextsByMode": { "exact": "1", "f32_forward": "1", "f32_reverse": "1", "f32_tree64": "1", "f16_forward": "th" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "1", "th" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 236770, 236800 ], "decodedText": "13", "steps": [ { "step": 0, "tokenId": 236770, "text": "1" }, { "step": 1, "tokenId": 236800, "text": "3" } ] }, "f16_forward": { "tokenIds": [ 594, 47909 ], "decodedText": "thirteen", "steps": [ { "step": 0, "tokenId": 594, "text": "th" }, { "step": 1, "tokenId": 47909, "text": "irteen" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "frag-pi-digits-oneword-b", "text": "Respond with one word only.\n3.14159", "stableTopK": [ { "rank": 1, "tokenId": 14762, "text": "Pi", "logit": 3.3300015926361084, "prob": 0.9999653697013855 }, { "rank": 2, "tokenId": 2160, "text": "pi", "logit": -7.391149044036865, "prob": 0.000022072341380408034 }, { "rank": 3, "tokenId": 3292, "text": "$\\", "logit": -8.684560775756836, "prob": 0.000006055176982044941 }, { "rank": 4, "tokenId": 31804, "text": "Appro", "logit": -9.836822509765625, "prob": 0.000001912960215122439 }, { "rank": 5, "tokenId": 3516, "text": "PI", "logit": -9.965800285339355, "prob": 0.0000016814794889796758 }, { "rank": 6, "tokenId": 139876, "text": "Approx", "logit": -11.064266204833984, "prob": 5.605751880466414e-7 }, { "rank": 7, "tokenId": 32913, "text": "Constant", "logit": -11.673771858215332, "prob": 3.047396432975802e-7 }, { "rank": 8, "tokenId": 33021, "text": "Circle", "logit": -12.088263511657715, "prob": 2.013341742213015e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 14762, "winnerText": "Pi", "winnerScore": 3.3437814969014745, "winnerGap": 10.890162374056235, "top": [ { "rank": 1, "tokenId": 14762, "text": "Pi", "score": 3.3437814969014745 }, { "rank": 2, "tokenId": 2160, "text": "pi", "score": -7.5463808771547605 }, { "rank": 3, "tokenId": 3292, "text": "$\\", "score": -8.940138990372276 }, { "rank": 4, "tokenId": 31804, "text": "Appro", "score": -10.21400266728574 }, { "rank": 5, "tokenId": 3516, "text": "PI", "score": -10.358745970894262 }, { "rank": 6, "tokenId": 139876, "text": "Approx", "score": -11.611313415250095 }, { "rank": 7, "tokenId": 32913, "text": "Constant", "score": -12.323076426289873 }, { "rank": 8, "tokenId": 33021, "text": "Circle", "score": -12.814687580168254 } ] }, "f32_forward": { "winnerTokenId": 14762, "winnerText": "Pi", "winnerScore": 3.343775987625122, "winnerGap": 10.890156984329224, "top": [ { "rank": 1, "tokenId": 14762, "text": "Pi", "score": 3.343775987625122 }, { "rank": 2, "tokenId": 2160, "text": "pi", "score": -7.546380996704102 }, { "rank": 3, "tokenId": 3292, "text": "$\\", "score": -8.940138816833496 }, { "rank": 4, "tokenId": 31804, "text": "Appro", "score": -10.214001655578613 }, { "rank": 5, "tokenId": 3516, "text": "PI", "score": -10.358743667602539 }, { "rank": 6, "tokenId": 139876, "text": "Approx", "score": -11.611310005187988 }, { "rank": 7, "tokenId": 32913, "text": "Constant", "score": -12.323075294494629 }, { "rank": 8, "tokenId": 33021, "text": "Circle", "score": -12.814692497253418 } ] }, "f32_reverse": { "winnerTokenId": 14762, "winnerText": "Pi", "winnerScore": 3.3437838554382324, "winnerGap": 10.890161514282227, "top": [ { "rank": 1, "tokenId": 14762, "text": "Pi", "score": 3.3437838554382324 }, { "rank": 2, "tokenId": 2160, "text": "pi", "score": -7.546377658843994 }, { "rank": 3, "tokenId": 3292, "text": "$\\", "score": -8.940131187438965 }, { "rank": 4, "tokenId": 31804, "text": "Appro", "score": -10.213994979858398 }, { "rank": 5, "tokenId": 3516, "text": "PI", "score": -10.358756065368652 }, { "rank": 6, "tokenId": 139876, "text": "Approx", "score": -11.611315727233887 }, { "rank": 7, "tokenId": 32913, "text": "Constant", "score": -12.323067665100098 }, { "rank": 8, "tokenId": 33021, "text": "Circle", "score": -12.814697265625 } ] }, "f32_tree64": { "winnerTokenId": 14762, "winnerText": "Pi", "winnerScore": 3.343780994415283, "winnerGap": 10.890162467956543, "top": [ { "rank": 1, "tokenId": 14762, "text": "Pi", "score": 3.343780994415283 }, { "rank": 2, "tokenId": 2160, "text": "pi", "score": -7.54638147354126 }, { "rank": 3, "tokenId": 3292, "text": "$\\", "score": -8.940139770507812 }, { "rank": 4, "tokenId": 31804, "text": "Appro", "score": -10.214001655578613 }, { "rank": 5, "tokenId": 3516, "text": "PI", "score": -10.358745574951172 }, { "rank": 6, "tokenId": 139876, "text": "Approx", "score": -11.611311912536621 }, { "rank": 7, "tokenId": 32913, "text": "Constant", "score": -12.323076248168945 }, { "rank": 8, "tokenId": 33021, "text": "Circle", "score": -12.814687728881836 } ] }, "f16_forward": { "winnerTokenId": 14762, "winnerText": "Pi", "winnerScore": 3.3828125, "winnerGap": 6.904296875, "top": [ { "rank": 1, "tokenId": 14762, "text": "Pi", "score": 3.3828125 }, { "rank": 2, "tokenId": 2160, "text": "pi", "score": -3.521484375 }, { "rank": 3, "tokenId": 874, "text": " $\\", "score": -8.4609375 }, { "rank": 4, "tokenId": 3292, "text": "$\\", "score": -8.9296875 }, { "rank": 5, "tokenId": 4452, "text": "Number", "score": -9.78125 }, { "rank": 6, "tokenId": 31804, "text": "Appro", "score": -10.140625 }, { "rank": 7, "tokenId": 31070, "text": "Letter", "score": -10.3984375 }, { "rank": 8, "tokenId": 3516, "text": "PI", "score": -10.6015625 } ] } }, "winnerTextsByMode": { "exact": "Pi", "f32_forward": "Pi", "f32_reverse": "Pi", "f32_tree64": "Pi", "f16_forward": "Pi" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Pi" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "frag-single-space-oneword-c", "text": "No explanation. One word only.\n ", "stableTopK": [ { "rank": 1, "tokenId": 14713, "text": "Under", "logit": 5.800669193267822, "prob": 0.8644183874130249 }, { "rank": 2, "tokenId": 186741, "text": "Silence", "logit": 3.58683705329895, "prob": 0.09446456283330917 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "logit": 1.5121759176254272, "prob": 0.01186465285718441 }, { "rank": 4, "tokenId": 136657, "text": "Silent", "logit": 1.5037771463394165, "prob": 0.01176542229950428 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "logit": 1.2994425296783447, "prob": 0.009591048583388329 }, { "rank": 6, "tokenId": 19058, "text": "Okay", "logit": 0.460488498210907, "prob": 0.004144889768213034 }, { "rank": 7, "tokenId": 154745, "text": "Impossible", "logit": -1.7234594821929932, "prob": 0.0004666985187213868 }, { "rank": 8, "tokenId": 34496, "text": "Done", "logit": -2.112560749053955, "prob": 0.0003162655630148947 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 14713, "winnerText": "Under", "winnerScore": 5.874626728695603, "winnerGap": 2.270549753490595, "top": [ { "rank": 1, "tokenId": 14713, "text": "Under", "score": 5.874626728695603 }, { "rank": 2, "tokenId": 186741, "text": "Silence", "score": 3.6040769752050075 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": 1.5134606940394573 }, { "rank": 4, "tokenId": 136657, "text": "Silent", "score": 1.505038142942631 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 1.3002543342532906 }, { "rank": 6, "tokenId": 19058, "text": "Okay", "score": 0.4605253480647278 }, { "rank": 7, "tokenId": 154745, "text": "Impossible", "score": -1.725361048652296 }, { "rank": 8, "tokenId": 34496, "text": "Done", "score": -2.116065528974101 } ] }, "f32_forward": { "winnerTokenId": 14713, "winnerText": "Under", "winnerScore": 5.874635219573975, "winnerGap": 2.270559549331665, "top": [ { "rank": 1, "tokenId": 14713, "text": "Under", "score": 5.874635219573975 }, { "rank": 2, "tokenId": 186741, "text": "Silence", "score": 3.6040756702423096 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": 1.5134608745574951 }, { "rank": 4, "tokenId": 136657, "text": "Silent", "score": 1.5050393342971802 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 1.3002574443817139 }, { "rank": 6, "tokenId": 19058, "text": "Okay", "score": 0.46052610874176025 }, { "rank": 7, "tokenId": 154745, "text": "Impossible", "score": -1.7253589630126953 }, { "rank": 8, "tokenId": 34496, "text": "Done", "score": -2.1160664558410645 } ] }, "f32_reverse": { "winnerTokenId": 14713, "winnerText": "Under", "winnerScore": 5.874629974365234, "winnerGap": 2.270555257797241, "top": [ { "rank": 1, "tokenId": 14713, "text": "Under", "score": 5.874629974365234 }, { "rank": 2, "tokenId": 186741, "text": "Silence", "score": 3.604074716567993 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": 1.5134624242782593 }, { "rank": 4, "tokenId": 136657, "text": "Silent", "score": 1.5050387382507324 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 1.3002524375915527 }, { "rank": 6, "tokenId": 19058, "text": "Okay", "score": 0.4605265259742737 }, { "rank": 7, "tokenId": 154745, "text": "Impossible", "score": -1.7253596782684326 }, { "rank": 8, "tokenId": 34496, "text": "Done", "score": -2.1160624027252197 } ] }, "f32_tree64": { "winnerTokenId": 14713, "winnerText": "Under", "winnerScore": 5.874625205993652, "winnerGap": 2.27054762840271, "top": [ { "rank": 1, "tokenId": 14713, "text": "Under", "score": 5.874625205993652 }, { "rank": 2, "tokenId": 186741, "text": "Silence", "score": 3.6040775775909424 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": 1.5134623050689697 }, { "rank": 4, "tokenId": 136657, "text": "Silent", "score": 1.5050389766693115 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 1.3002548217773438 }, { "rank": 6, "tokenId": 19058, "text": "Okay", "score": 0.4605245590209961 }, { "rank": 7, "tokenId": 154745, "text": "Impossible", "score": -1.7253613471984863 }, { "rank": 8, "tokenId": 34496, "text": "Done", "score": -2.116065263748169 } ] }, "f16_forward": { "winnerTokenId": 14713, "winnerText": "Under", "winnerScore": 5.94140625, "winnerGap": 2.330078125, "top": [ { "rank": 1, "tokenId": 14713, "text": "Under", "score": 5.94140625 }, { "rank": 2, "tokenId": 186741, "text": "Silence", "score": 3.611328125 }, { "rank": 3, "tokenId": 136657, "text": "Silent", "score": 1.5263671875 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 1.3388671875 }, { "rank": 5, "tokenId": 20470, "text": "Wait", "score": 0.5537109375 }, { "rank": 6, "tokenId": 19058, "text": "Okay", "score": 0.466796875 }, { "rank": 7, "tokenId": 154745, "text": "Impossible", "score": -1.74609375 }, { "rank": 8, "tokenId": 34496, "text": "Done", "score": -2.126953125 } ] } }, "winnerTextsByMode": { "exact": "Under", "f32_forward": "Under", "f32_reverse": "Under", "f32_tree64": "Under", "f16_forward": "Under" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Under" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "frag-tab-oneword-d", "text": "Choose one word only.\n\t", "stableTopK": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "logit": 9.337449073791504, "prob": 0.9983876943588257 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "logit": 1.9353225231170654, "prob": 0.0006089708185754716 }, { "rank": 3, "tokenId": 20505, "text": "Choice", "logit": 1.4037443399429321, "prob": 0.0003578780160751194 }, { "rank": 4, "tokenId": 4906, "text": "One", "logit": 0.9746519923210144, "prob": 0.00023301430337596685 }, { "rank": 5, "tokenId": 9366, "text": "Please", "logit": -0.14125767350196838, "prob": 0.00007633947825524956 }, { "rank": 6, "tokenId": 14979, "text": "Word", "logit": -0.42997753620147705, "prob": 0.0000571952186874114 }, { "rank": 7, "tokenId": 11323, "text": "Select", "logit": -0.6246534585952759, "prob": 0.0000470774648420047 }, { "rank": 8, "tokenId": 60776, "text": "Waiting", "logit": -0.6963194012641907, "prob": 0.0000438216702605132 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 24852, "winnerText": "Choose", "winnerScore": 9.657809244949114, "winnerGap": 7.71979574642657, "top": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "score": 9.657809244949114 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 1.938013498522544 }, { "rank": 3, "tokenId": 20505, "text": "Choice", "score": 1.4047710131151447 }, { "rank": 4, "tokenId": 4906, "text": "One", "score": 0.9749985462931612 }, { "rank": 5, "tokenId": 9366, "text": "Please", "score": -0.14125671517362548 }, { "rank": 6, "tokenId": 14979, "text": "Word", "score": -0.43000649635631305 }, { "rank": 7, "tokenId": 11323, "text": "Select", "score": -0.6247428685031622 }, { "rank": 8, "tokenId": 60776, "text": "Waiting", "score": -0.696443482226897 } ] }, "f32_forward": { "winnerTokenId": 24852, "winnerText": "Choose", "winnerScore": 9.657812118530273, "winnerGap": 7.719796061515808, "top": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "score": 9.657812118530273 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 1.9380160570144653 }, { "rank": 3, "tokenId": 20505, "text": "Choice", "score": 1.404769778251648 }, { "rank": 4, "tokenId": 4906, "text": "One", "score": 0.9749968647956848 }, { "rank": 5, "tokenId": 9366, "text": "Please", "score": -0.1412588357925415 }, { "rank": 6, "tokenId": 14979, "text": "Word", "score": -0.4300064444541931 }, { "rank": 7, "tokenId": 11323, "text": "Select", "score": -0.6247510313987732 }, { "rank": 8, "tokenId": 60776, "text": "Waiting", "score": -0.6964451670646667 } ] }, "f32_reverse": { "winnerTokenId": 24852, "winnerText": "Choose", "winnerScore": 9.657816886901855, "winnerGap": 7.719800710678101, "top": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "score": 9.657816886901855 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 1.9380161762237549 }, { "rank": 3, "tokenId": 20505, "text": "Choice", "score": 1.4047702550888062 }, { "rank": 4, "tokenId": 4906, "text": "One", "score": 0.9749988317489624 }, { "rank": 5, "tokenId": 9366, "text": "Please", "score": -0.1412581205368042 }, { "rank": 6, "tokenId": 14979, "text": "Word", "score": -0.4300057888031006 }, { "rank": 7, "tokenId": 11323, "text": "Select", "score": -0.6247458457946777 }, { "rank": 8, "tokenId": 60776, "text": "Waiting", "score": -0.6964434385299683 } ] }, "f32_tree64": { "winnerTokenId": 24852, "winnerText": "Choose", "winnerScore": 9.657809257507324, "winnerGap": 7.7197957038879395, "top": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "score": 9.657809257507324 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 1.9380135536193848 }, { "rank": 3, "tokenId": 20505, "text": "Choice", "score": 1.404771327972412 }, { "rank": 4, "tokenId": 4906, "text": "One", "score": 0.9749981164932251 }, { "rank": 5, "tokenId": 9366, "text": "Please", "score": -0.14125710725784302 }, { "rank": 6, "tokenId": 14979, "text": "Word", "score": -0.4300069808959961 }, { "rank": 7, "tokenId": 11323, "text": "Select", "score": -0.6247432231903076 }, { "rank": 8, "tokenId": 60776, "text": "Waiting", "score": -0.6964439153671265 } ] }, "f16_forward": { "winnerTokenId": 24852, "winnerText": "Choose", "winnerScore": 9.6640625, "winnerGap": 8.2568359375, "top": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "score": 9.6640625 }, { "rank": 2, "tokenId": 20505, "text": "Choice", "score": 1.4072265625 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": 1.2646484375 }, { "rank": 4, "tokenId": 4906, "text": "One", "score": 0.9736328125 }, { "rank": 5, "tokenId": 60776, "text": "Waiting", "score": 0.310546875 }, { "rank": 6, "tokenId": 9366, "text": "Please", "score": -0.1263427734375 }, { "rank": 7, "tokenId": 11323, "text": "Select", "score": -0.62646484375 }, { "rank": 8, "tokenId": 35724, "text": "choose", "score": -1.1591796875 } ] } }, "winnerTextsByMode": { "exact": "Choose", "f32_forward": "Choose", "f32_reverse": "Choose", "f32_tree64": "Choose", "f16_forward": "Choose" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Choose" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "frag-therefore-oneword-a", "text": "Answer with exactly one word only.\nTherefore,", "stableTopK": [ { "rank": 1, "tokenId": 21204, "text": "Thus", "logit": 5.56248664855957, "prob": 0.6712928414344788 }, { "rank": 2, "tokenId": 107673, "text": "Consequently", "logit": 4.425498962402344, "prob": 0.21533991396427155 }, { "rank": 3, "tokenId": 20416, "text": "Therefore", "logit": 3.1503753662109375, "prob": 0.06016520410776138 }, { "rank": 4, "tokenId": 30792, "text": "Conclusion", "logit": 2.8164560794830322, "prob": 0.043084997683763504 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "logit": 0.9210742712020874, "prob": 0.0064739929512143135 }, { "rank": 6, "tokenId": 35367, "text": "Hence", "logit": -0.19572028517723083, "prob": 0.0021191148553043604 }, { "rank": 7, "tokenId": 17574, "text": "Because", "logit": -1.9553126096725464, "prob": 0.0003647314733825624 }, { "rank": 8, "tokenId": 4324, "text": "So", "logit": -2.095623254776001, "prob": 0.0003169838455505669 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 21204, "winnerText": "Thus", "winnerScore": 5.627575744061502, "winnerGap": 1.1695506697911, "top": [ { "rank": 1, "tokenId": 21204, "text": "Thus", "score": 5.627575744061502 }, { "rank": 2, "tokenId": 107673, "text": "Consequently", "score": 4.458025074270402 }, { "rank": 3, "tokenId": 20416, "text": "Therefore", "score": 3.162032701644719 }, { "rank": 4, "tokenId": 30792, "text": "Conclusion", "score": 2.8247763094902183 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 0.9213665595602976 }, { "rank": 6, "tokenId": 35367, "text": "Hence", "score": -0.1957264071790732 }, { "rank": 7, "tokenId": 17574, "text": "Because", "score": -1.9580877735308189 }, { "rank": 8, "tokenId": 4324, "text": "So", "score": -2.099045703805352 } ] }, "f32_forward": { "winnerTokenId": 21204, "winnerText": "Thus", "winnerScore": 5.6275858879089355, "winnerGap": 1.1695566177368164, "top": [ { "rank": 1, "tokenId": 21204, "text": "Thus", "score": 5.6275858879089355 }, { "rank": 2, "tokenId": 107673, "text": "Consequently", "score": 4.458029270172119 }, { "rank": 3, "tokenId": 20416, "text": "Therefore", "score": 3.162036418914795 }, { "rank": 4, "tokenId": 30792, "text": "Conclusion", "score": 2.8247742652893066 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 0.9213621616363525 }, { "rank": 6, "tokenId": 35367, "text": "Hence", "score": -0.19573229551315308 }, { "rank": 7, "tokenId": 17574, "text": "Because", "score": -1.9580885171890259 }, { "rank": 8, "tokenId": 4324, "text": "So", "score": -2.0990402698516846 } ] }, "f32_reverse": { "winnerTokenId": 21204, "winnerText": "Thus", "winnerScore": 5.627575397491455, "winnerGap": 1.1695499420166016, "top": [ { "rank": 1, "tokenId": 21204, "text": "Thus", "score": 5.627575397491455 }, { "rank": 2, "tokenId": 107673, "text": "Consequently", "score": 4.4580254554748535 }, { "rank": 3, "tokenId": 20416, "text": "Therefore", "score": 3.1620359420776367 }, { "rank": 4, "tokenId": 30792, "text": "Conclusion", "score": 2.8247768878936768 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 0.9213643074035645 }, { "rank": 6, "tokenId": 35367, "text": "Hence", "score": -0.19573256373405457 }, { "rank": 7, "tokenId": 17574, "text": "Because", "score": -1.95809006690979 }, { "rank": 8, "tokenId": 4324, "text": "So", "score": -2.0990512371063232 } ] }, "f32_tree64": { "winnerTokenId": 21204, "winnerText": "Thus", "winnerScore": 5.627575874328613, "winnerGap": 1.1695528030395508, "top": [ { "rank": 1, "tokenId": 21204, "text": "Thus", "score": 5.627575874328613 }, { "rank": 2, "tokenId": 107673, "text": "Consequently", "score": 4.4580230712890625 }, { "rank": 3, "tokenId": 20416, "text": "Therefore", "score": 3.162032127380371 }, { "rank": 4, "tokenId": 30792, "text": "Conclusion", "score": 2.8247756958007812 }, { "rank": 5, "tokenId": 10784, "text": "Yes", "score": 0.92136549949646 }, { "rank": 6, "tokenId": 35367, "text": "Hence", "score": -0.19572663307189941 }, { "rank": 7, "tokenId": 17574, "text": "Because", "score": -1.9580879211425781 }, { "rank": 8, "tokenId": 4324, "text": "So", "score": -2.099045515060425 } ] }, "f16_forward": { "winnerTokenId": 107673, "winnerText": "Consequently", "winnerScore": 4.5078125, "winnerGap": 1.3046875, "top": [ { "rank": 1, "tokenId": 107673, "text": "Consequently", "score": 4.5078125 }, { "rank": 2, "tokenId": 20416, "text": "Therefore", "score": 3.203125 }, { "rank": 3, "tokenId": 30792, "text": "Conclusion", "score": 2.83984375 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 0.912109375 }, { "rank": 5, "tokenId": 21204, "text": "Thus", "score": 0.37255859375 }, { "rank": 6, "tokenId": 35367, "text": "Hence", "score": -0.173583984375 }, { "rank": 7, "tokenId": 4324, "text": "So", "score": -1.0888671875 }, { "rank": 8, "tokenId": 1505, "text": "Con", "score": -1.7802734375 } ] } }, "winnerTextsByMode": { "exact": "Thus", "f32_forward": "Thus", "f32_reverse": "Thus", "f32_tree64": "Thus", "f16_forward": "Consequently" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Thus", "Consequently" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 21204, 106 ], "decodedText": "Thus", "steps": [ { "step": 0, "tokenId": 21204, "text": "Thus" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 107673, 106 ], "decodedText": "Consequently", "steps": [ { "step": 0, "tokenId": 107673, "text": "Consequently" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "frag-true-literal-oneword-b", "text": "Respond with one word only.\ntrue", "stableTopK": [ { "rank": 1, "tokenId": 4339, "text": "True", "logit": 6.807511806488037, "prob": 0.9656517505645752 }, { "rank": 2, "tokenId": 3397, "text": "true", "logit": 3.1780285835266113, "prob": 0.02561863511800766 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "logit": 1.648895263671875, "prob": 0.005552157759666443 }, { "rank": 4, "tokenId": 9277, "text": "False", "logit": -0.4660717844963074, "prob": 0.000669797183945775 }, { "rank": 5, "tokenId": 14713, "text": "Under", "logit": -0.6179805994033813, "prob": 0.0005754003650508821 }, { "rank": 6, "tokenId": 106, "text": "", "logit": -0.8625571727752686, "prob": 0.00045055922237224877 }, { "rank": 7, "tokenId": 4443, "text": "yes", "logit": -1.2799938917160034, "prob": 0.0002967983018606901 }, { "rank": 8, "tokenId": 1136, "text": "und", "logit": -1.6930073499679565, "prob": 0.00019637761579360813 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 6.92810154908771, "winnerGap": 3.7381048427261483, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 6.92810154908771 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 3.189996706361562 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 1.6505581243066914 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": -0.4661085145538948 }, { "rank": 5, "tokenId": 14713, "text": "Under", "score": -0.6180653347662997 }, { "rank": 6, "tokenId": 106, "text": "", "score": -0.862796131430569 }, { "rank": 7, "tokenId": 4443, "text": "yes", "score": -1.2807721201734008 }, { "rank": 8, "tokenId": 1136, "text": "und", "score": -1.6948066791952616 } ] }, "f32_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 6.928094387054443, "winnerGap": 3.7380988597869873, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 6.928094387054443 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 3.189995527267456 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 1.6505682468414307 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": -0.46611031889915466 }, { "rank": 5, "tokenId": 14713, "text": "Under", "score": -0.6180605888366699 }, { "rank": 6, "tokenId": 106, "text": "", "score": -0.8627951145172119 }, { "rank": 7, "tokenId": 4443, "text": "yes", "score": -1.2807698249816895 }, { "rank": 8, "tokenId": 1136, "text": "und", "score": -1.6948095560073853 } ] }, "f32_reverse": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 6.928098201751709, "winnerGap": 3.7380990982055664, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 6.928098201751709 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 3.1899991035461426 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 1.650561809539795 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": -0.4661109447479248 }, { "rank": 5, "tokenId": 14713, "text": "Under", "score": -0.6180609464645386 }, { "rank": 6, "tokenId": 106, "text": "", "score": -0.8627980947494507 }, { "rank": 7, "tokenId": 4443, "text": "yes", "score": -1.2807738780975342 }, { "rank": 8, "tokenId": 1136, "text": "und", "score": -1.6948070526123047 } ] }, "f32_tree64": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 6.928101539611816, "winnerGap": 3.7381038665771484, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 6.928101539611816 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 3.189997673034668 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 1.6505582332611084 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": -0.4661080837249756 }, { "rank": 5, "tokenId": 14713, "text": "Under", "score": -0.618065357208252 }, { "rank": 6, "tokenId": 106, "text": "", "score": -0.8627974987030029 }, { "rank": 7, "tokenId": 4443, "text": "yes", "score": -1.280772089958191 }, { "rank": 8, "tokenId": 1136, "text": "und", "score": -1.6948065757751465 } ] }, "f16_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 6.93359375, "winnerGap": 3.69921875, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 6.93359375 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 3.234375 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 1.6572265625 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": -0.40625 }, { "rank": 5, "tokenId": 14713, "text": "Under", "score": -0.57080078125 }, { "rank": 6, "tokenId": 106, "text": "", "score": -0.61865234375 }, { "rank": 7, "tokenId": 4443, "text": "yes", "score": -1.2412109375 }, { "rank": 8, "tokenId": 1847, "text": " true", "score": -1.6630859375 } ] } }, "winnerTextsByMode": { "exact": "True", "f32_forward": "True", "f32_reverse": "True", "f32_tree64": "True", "f16_forward": "True" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "True" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "frag-wait-oneword-c", "text": "No explanation. One word only.\nWait —", "stableTopK": [ { "rank": 1, "tokenId": 34230, "text": "Continue", "logit": 6.212794303894043, "prob": 0.3324045240879059 }, { "rank": 2, "tokenId": 66639, "text": "Hold", "logit": 4.84063720703125, "prob": 0.08428429067134857 }, { "rank": 3, "tokenId": 47191, "text": "Pause", "logit": 4.780167102813721, "prob": 0.07933865487575531 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "logit": 4.724221706390381, "prob": 0.0750219002366066 }, { "rank": 5, "tokenId": 52200, "text": "Processing", "logit": 4.628656387329102, "prob": 0.06818433105945587 }, { "rank": 6, "tokenId": 27426, "text": "Begin", "logit": 4.081452369689941, "prob": 0.03944908082485199 }, { "rank": 7, "tokenId": 120474, "text": "Thinking", "logit": 4.057793617248535, "prob": 0.03852671757340431 }, { "rank": 8, "tokenId": 17348, "text": "Stop", "logit": 3.9496679306030273, "prob": 0.03457829728722572 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 34230, "winnerText": "Continue", "winnerScore": 6.3039682556127055, "winnerGap": 1.420654091371249, "top": [ { "rank": 1, "tokenId": 34230, "text": "Continue", "score": 6.3039682556127055 }, { "rank": 2, "tokenId": 66639, "text": "Hold", "score": 4.883314164241456 }, { "rank": 3, "tokenId": 47191, "text": "Pause", "score": 4.821251000666933 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 4.763864356251773 }, { "rank": 5, "tokenId": 52200, "text": "Processing", "score": 4.6659197359228415 }, { "rank": 6, "tokenId": 27426, "text": "Begin", "score": 4.106917251727168 }, { "rank": 7, "tokenId": 120474, "text": "Thinking", "score": 4.082812932261273 }, { "rank": 8, "tokenId": 17348, "text": "Stop", "score": 3.9727284547984745 } ] }, "f32_forward": { "winnerTokenId": 34230, "winnerText": "Continue", "winnerScore": 6.303968906402588, "winnerGap": 1.4206533432006836, "top": [ { "rank": 1, "tokenId": 34230, "text": "Continue", "score": 6.303968906402588 }, { "rank": 2, "tokenId": 66639, "text": "Hold", "score": 4.883315563201904 }, { "rank": 3, "tokenId": 47191, "text": "Pause", "score": 4.821248531341553 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 4.763871192932129 }, { "rank": 5, "tokenId": 52200, "text": "Processing", "score": 4.665923118591309 }, { "rank": 6, "tokenId": 27426, "text": "Begin", "score": 4.1069207191467285 }, { "rank": 7, "tokenId": 120474, "text": "Thinking", "score": 4.082812309265137 }, { "rank": 8, "tokenId": 17348, "text": "Stop", "score": 3.972727060317993 } ] }, "f32_reverse": { "winnerTokenId": 34230, "winnerText": "Continue", "winnerScore": 6.303969383239746, "winnerGap": 1.4206509590148926, "top": [ { "rank": 1, "tokenId": 34230, "text": "Continue", "score": 6.303969383239746 }, { "rank": 2, "tokenId": 66639, "text": "Hold", "score": 4.8833184242248535 }, { "rank": 3, "tokenId": 47191, "text": "Pause", "score": 4.8212504386901855 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 4.763864994049072 }, { "rank": 5, "tokenId": 52200, "text": "Processing", "score": 4.665917873382568 }, { "rank": 6, "tokenId": 27426, "text": "Begin", "score": 4.1069183349609375 }, { "rank": 7, "tokenId": 120474, "text": "Thinking", "score": 4.0828142166137695 }, { "rank": 8, "tokenId": 17348, "text": "Stop", "score": 3.972728729248047 } ] }, "f32_tree64": { "winnerTokenId": 34230, "winnerText": "Continue", "winnerScore": 6.30396842956543, "winnerGap": 1.4206547737121582, "top": [ { "rank": 1, "tokenId": 34230, "text": "Continue", "score": 6.30396842956543 }, { "rank": 2, "tokenId": 66639, "text": "Hold", "score": 4.8833136558532715 }, { "rank": 3, "tokenId": 47191, "text": "Pause", "score": 4.821250915527344 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 4.7638654708862305 }, { "rank": 5, "tokenId": 52200, "text": "Processing", "score": 4.665920257568359 }, { "rank": 6, "tokenId": 27426, "text": "Begin", "score": 4.106917381286621 }, { "rank": 7, "tokenId": 120474, "text": "Thinking", "score": 4.082812786102295 }, { "rank": 8, "tokenId": 17348, "text": "Stop", "score": 3.9727282524108887 } ] }, "f16_forward": { "winnerTokenId": 34230, "winnerText": "Continue", "winnerScore": 6.296875, "winnerGap": 1.390625, "top": [ { "rank": 1, "tokenId": 34230, "text": "Continue", "score": 6.296875 }, { "rank": 2, "tokenId": 66639, "text": "Hold", "score": 4.90625 }, { "rank": 3, "tokenId": 47191, "text": "Pause", "score": 4.86328125 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 4.80078125 }, { "rank": 5, "tokenId": 52200, "text": "Processing", "score": 4.6875 }, { "rank": 6, "tokenId": 27426, "text": "Begin", "score": 4.35546875 }, { "rank": 7, "tokenId": 120474, "text": "Thinking", "score": 4.07421875 }, { "rank": 8, "tokenId": 17348, "text": "Stop", "score": 4.0390625 } ] } }, "winnerTextsByMode": { "exact": "Continue", "f32_forward": "Continue", "f32_reverse": "Continue", "f32_tree64": "Continue", "f16_forward": "Continue" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Continue" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "frag-zero-oneword-d", "text": "Choose one word only.\n0", "stableTopK": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "logit": 12.407197952270508, "prob": 0.7232204079627991 }, { "rank": 2, "tokenId": 4906, "text": "One", "logit": 11.25479507446289, "prob": 0.22844856977462769 }, { "rank": 3, "tokenId": 29337, "text": "Zero", "logit": 8.69655704498291, "prob": 0.01769130304455757 }, { "rank": 4, "tokenId": 811, "text": "one", "logit": 8.651328086853027, "prob": 0.01690896786749363 }, { "rank": 5, "tokenId": 20505, "text": "Choice", "logit": 7.4682207107543945, "prob": 0.005179646424949169 }, { "rank": 6, "tokenId": 35724, "text": "choose", "logit": 6.633573532104492, "prob": 0.002248109318315983 }, { "rank": 7, "tokenId": 13321, "text": "zero", "logit": 6.143334865570068, "prob": 0.0013769224751740694 }, { "rank": 8, "tokenId": 17005, "text": "choice", "logit": 5.8750505447387695, "prob": 0.0010529193095862865 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 24852, "winnerText": "Choose", "winnerScore": 13.19742645422706, "winnerGap": 1.3649859030802673, "top": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "score": 13.19742645422706 }, { "rank": 2, "tokenId": 4906, "text": "One", "score": 11.832440551146792 }, { "rank": 3, "tokenId": 29337, "text": "Zero", "score": 8.953231402385036 }, { "rank": 4, "tokenId": 811, "text": "one", "score": 8.903877950222798 }, { "rank": 5, "tokenId": 20505, "text": "Choice", "score": 7.628495223124428 }, { "rank": 6, "tokenId": 35724, "text": "choose", "score": 6.7449683405976 }, { "rank": 7, "tokenId": 13321, "text": "zero", "score": 6.231434495646493 }, { "rank": 8, "tokenId": 17005, "text": "choice", "score": 5.951930543134832 } ] }, "f32_forward": { "winnerTokenId": 24852, "winnerText": "Choose", "winnerScore": 13.197427749633789, "winnerGap": 1.3649969100952148, "top": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "score": 13.197427749633789 }, { "rank": 2, "tokenId": 4906, "text": "One", "score": 11.832430839538574 }, { "rank": 3, "tokenId": 29337, "text": "Zero", "score": 8.953228950500488 }, { "rank": 4, "tokenId": 811, "text": "one", "score": 8.903883934020996 }, { "rank": 5, "tokenId": 20505, "text": "Choice", "score": 7.628496170043945 }, { "rank": 6, "tokenId": 35724, "text": "choose", "score": 6.744969844818115 }, { "rank": 7, "tokenId": 13321, "text": "zero", "score": 6.2314324378967285 }, { "rank": 8, "tokenId": 17005, "text": "choice", "score": 5.951937198638916 } ] }, "f32_reverse": { "winnerTokenId": 24852, "winnerText": "Choose", "winnerScore": 13.197431564331055, "winnerGap": 1.3649835586547852, "top": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "score": 13.197431564331055 }, { "rank": 2, "tokenId": 4906, "text": "One", "score": 11.83244800567627 }, { "rank": 3, "tokenId": 29337, "text": "Zero", "score": 8.953227996826172 }, { "rank": 4, "tokenId": 811, "text": "one", "score": 8.903881072998047 }, { "rank": 5, "tokenId": 20505, "text": "Choice", "score": 7.6284942626953125 }, { "rank": 6, "tokenId": 35724, "text": "choose", "score": 6.744968891143799 }, { "rank": 7, "tokenId": 13321, "text": "zero", "score": 6.231435775756836 }, { "rank": 8, "tokenId": 17005, "text": "choice", "score": 5.95193338394165 } ] }, "f32_tree64": { "winnerTokenId": 24852, "winnerText": "Choose", "winnerScore": 13.197425842285156, "winnerGap": 1.3649845123291016, "top": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "score": 13.197425842285156 }, { "rank": 2, "tokenId": 4906, "text": "One", "score": 11.832441329956055 }, { "rank": 3, "tokenId": 29337, "text": "Zero", "score": 8.953230857849121 }, { "rank": 4, "tokenId": 811, "text": "one", "score": 8.903877258300781 }, { "rank": 5, "tokenId": 20505, "text": "Choice", "score": 7.6284942626953125 }, { "rank": 6, "tokenId": 35724, "text": "choose", "score": 6.744968414306641 }, { "rank": 7, "tokenId": 13321, "text": "zero", "score": 6.2314348220825195 }, { "rank": 8, "tokenId": 17005, "text": "choice", "score": 5.951930046081543 } ] }, "f16_forward": { "winnerTokenId": 24852, "winnerText": "Choose", "winnerScore": 13.0625, "winnerGap": 1.21875, "top": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "score": 13.0625 }, { "rank": 2, "tokenId": 4906, "text": "One", "score": 11.84375 }, { "rank": 3, "tokenId": 811, "text": "one", "score": 8.9296875 }, { "rank": 4, "tokenId": 20505, "text": "Choice", "score": 7.54296875 }, { "rank": 5, "tokenId": 35724, "text": "choose", "score": 6.6640625 }, { "rank": 6, "tokenId": 13321, "text": "zero", "score": 6.23828125 }, { "rank": 7, "tokenId": 17005, "text": "choice", "score": 5.9453125 }, { "rank": 8, "tokenId": 29337, "text": "Zero", "score": 4.9921875 } ] } }, "winnerTextsByMode": { "exact": "Choose", "f32_forward": "Choose", "f32_reverse": "Choose", "f32_tree64": "Choose", "f16_forward": "Choose" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Choose" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "france-paris-london-oneword-a", "text": "Answer with exactly one word only.\nThe capital of France is Paris or London. It is", "stableTopK": [ { "rank": 1, "tokenId": 50429, "text": "Paris", "logit": 12.389444351196289, "prob": 0.9999997019767761 }, { "rank": 2, "tokenId": 158659, "text": "paris", "logit": -3.2231717109680176, "prob": 1.657779762354039e-7 }, { "rank": 3, "tokenId": 142253, "text": "巴黎", "logit": -4.413666725158691, "prob": 5.04082322549948e-8 }, { "rank": 4, "tokenId": 163618, "text": "パリ", "logit": -4.826140403747559, "prob": 3.337078169352026e-8 }, { "rank": 5, "tokenId": 9079, "text": " Paris", "logit": -5.9135613441467285, "prob": 1.1248781461858925e-8 }, { "rank": 6, "tokenId": 31756, "text": "France", "logit": -6.814407825469971, "prob": 4.56954385441577e-9 }, { "rank": 7, "tokenId": 3128, "text": "Par", "logit": -7.023715019226074, "prob": 3.7065672753300305e-9 }, { "rank": 8, "tokenId": 1018, "text": "**", "logit": -7.655755996704102, "prob": 1.97006233548791e-9 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 50429, "winnerText": "Paris", "winnerScore": 13.176013788562216, "winnerGap": 16.411675150817018, "top": [ { "rank": 1, "tokenId": 50429, "text": "Paris", "score": 13.176013788562216 }, { "rank": 2, "tokenId": 158659, "text": "paris", "score": -3.2356613622548025 }, { "rank": 3, "tokenId": 142253, "text": "巴黎", "score": -4.445931304682944 }, { "rank": 4, "tokenId": 163618, "text": "パリ", "score": -4.868430220742596 }, { "rank": 5, "tokenId": 9079, "text": " Paris", "score": -5.991994147989846 }, { "rank": 6, "tokenId": 31756, "text": "France", "score": -6.935376675731831 }, { "rank": 7, "tokenId": 3128, "text": "Par", "score": -7.1564420236809525 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": -7.82875675775734 } ] }, "f32_forward": { "winnerTokenId": 50429, "winnerText": "Paris", "winnerScore": 13.175987243652344, "winnerGap": 16.411654472351074, "top": [ { "rank": 1, "tokenId": 50429, "text": "Paris", "score": 13.175987243652344 }, { "rank": 2, "tokenId": 158659, "text": "paris", "score": -3.2356672286987305 }, { "rank": 3, "tokenId": 142253, "text": "巴黎", "score": -4.445930004119873 }, { "rank": 4, "tokenId": 163618, "text": "パリ", "score": -4.868432521820068 }, { "rank": 5, "tokenId": 9079, "text": " Paris", "score": -5.991993427276611 }, { "rank": 6, "tokenId": 31756, "text": "France", "score": -6.935375690460205 }, { "rank": 7, "tokenId": 3128, "text": "Par", "score": -7.156455993652344 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": -7.828754901885986 } ] }, "f32_reverse": { "winnerTokenId": 50429, "winnerText": "Paris", "winnerScore": 13.176002502441406, "winnerGap": 16.41166400909424, "top": [ { "rank": 1, "tokenId": 50429, "text": "Paris", "score": 13.176002502441406 }, { "rank": 2, "tokenId": 158659, "text": "paris", "score": -3.235661506652832 }, { "rank": 3, "tokenId": 142253, "text": "巴黎", "score": -4.445937633514404 }, { "rank": 4, "tokenId": 163618, "text": "パリ", "score": -4.8684234619140625 }, { "rank": 5, "tokenId": 9079, "text": " Paris", "score": -5.991987705230713 }, { "rank": 6, "tokenId": 31756, "text": "France", "score": -6.935400009155273 }, { "rank": 7, "tokenId": 3128, "text": "Par", "score": -7.156431674957275 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": -7.828754425048828 } ] }, "f32_tree64": { "winnerTokenId": 50429, "winnerText": "Paris", "winnerScore": 13.176013946533203, "winnerGap": 16.411675930023193, "top": [ { "rank": 1, "tokenId": 50429, "text": "Paris", "score": 13.176013946533203 }, { "rank": 2, "tokenId": 158659, "text": "paris", "score": -3.2356619834899902 }, { "rank": 3, "tokenId": 142253, "text": "巴黎", "score": -4.445931911468506 }, { "rank": 4, "tokenId": 163618, "text": "パリ", "score": -4.868429660797119 }, { "rank": 5, "tokenId": 9079, "text": " Paris", "score": -5.991997718811035 }, { "rank": 6, "tokenId": 31756, "text": "France", "score": -6.935377597808838 }, { "rank": 7, "tokenId": 3128, "text": "Par", "score": -7.156442165374756 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": -7.828757286071777 } ] }, "f16_forward": { "winnerTokenId": 50429, "winnerText": "Paris", "winnerScore": 13.140625, "winnerGap": 16.32421875, "top": [ { "rank": 1, "tokenId": 50429, "text": "Paris", "score": 13.140625 }, { "rank": 2, "tokenId": 158659, "text": "paris", "score": -3.18359375 }, { "rank": 3, "tokenId": 142253, "text": "巴黎", "score": -4.41015625 }, { "rank": 4, "tokenId": 236775, "text": "\"", "score": -4.49609375 }, { "rank": 5, "tokenId": 163618, "text": "パリ", "score": -4.90625 }, { "rank": 6, "tokenId": 1437, "text": "the", "score": -5.59375 }, { "rank": 7, "tokenId": 9079, "text": " Paris", "score": -6.01171875 }, { "rank": 8, "tokenId": 106, "text": "", "score": -6.03515625 } ] } }, "winnerTextsByMode": { "exact": "Paris", "f32_forward": "Paris", "f32_reverse": "Paris", "f32_tree64": "Paris", "f16_forward": "Paris" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Paris" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "freedom-loss-explicit-oneword-b", "text": "Respond with one word only.\nAnswer with exactly one word: freedom or loss. Question: Leaving a strict religion can feel like Answer:", "stableTopK": [ { "rank": 1, "tokenId": 112605, "text": "freedom", "logit": 18.503747940063477, "prob": 0.9991663098335266 }, { "rank": 2, "tokenId": 106729, "text": "Freedom", "logit": 11.239017486572266, "prob": 0.0006992064299993217 }, { "rank": 3, "tokenId": 10558, "text": "loss", "logit": 8.845888137817383, "prob": 0.0000638678902760148 }, { "rank": 4, "tokenId": 5285, "text": "lib", "logit": 8.160565376281738, "prob": 0.00003218478741473518 }, { "rank": 5, "tokenId": 24639, "text": "transition", "logit": 7.021409511566162, "prob": 0.000010302000191586558 }, { "rank": 6, "tokenId": 27413, "text": "both", "logit": 5.831861972808838, "prob": 0.0000031355057217297144 }, { "rank": 7, "tokenId": 51775, "text": "gain", "logit": 5.694176197052002, "prob": 0.0000027321934794599656 }, { "rank": 8, "tokenId": 43279, "text": "Loss", "logit": 5.602294921875, "prob": 0.000002492343583071488 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 112605, "winnerText": "freedom", "winnerScore": 21.594298425290308, "winnerGap": 9.780215359709118, "top": [ { "rank": 1, "tokenId": 112605, "text": "freedom", "score": 21.594298425290308 }, { "rank": 2, "tokenId": 106729, "text": "Freedom", "score": 11.81408306558119 }, { "rank": 3, "tokenId": 10558, "text": "loss", "score": 9.11652120292807 }, { "rank": 4, "tokenId": 5285, "text": "lib", "score": 8.371272517794694 }, { "rank": 5, "tokenId": 24639, "text": "transition", "score": 7.154001663797523 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": 5.907035154745557 }, { "rank": 7, "tokenId": 51775, "text": "gain", "score": 5.764072480634901 }, { "rank": 8, "tokenId": 43279, "text": "Loss", "score": 5.668811710605591 } ] }, "f32_forward": { "winnerTokenId": 112605, "winnerText": "freedom", "winnerScore": 21.594303131103516, "winnerGap": 9.780229568481445, "top": [ { "rank": 1, "tokenId": 112605, "text": "freedom", "score": 21.594303131103516 }, { "rank": 2, "tokenId": 106729, "text": "Freedom", "score": 11.81407356262207 }, { "rank": 3, "tokenId": 10558, "text": "loss", "score": 9.116525650024414 }, { "rank": 4, "tokenId": 5285, "text": "lib", "score": 8.371277809143066 }, { "rank": 5, "tokenId": 24639, "text": "transition", "score": 7.153992652893066 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": 5.907033920288086 }, { "rank": 7, "tokenId": 51775, "text": "gain", "score": 5.764053821563721 }, { "rank": 8, "tokenId": 43279, "text": "Loss", "score": 5.6688127517700195 } ] }, "f32_reverse": { "winnerTokenId": 112605, "winnerText": "freedom", "winnerScore": 21.594280242919922, "winnerGap": 9.780191421508789, "top": [ { "rank": 1, "tokenId": 112605, "text": "freedom", "score": 21.594280242919922 }, { "rank": 2, "tokenId": 106729, "text": "Freedom", "score": 11.814088821411133 }, { "rank": 3, "tokenId": 10558, "text": "loss", "score": 9.1165189743042 }, { "rank": 4, "tokenId": 5285, "text": "lib", "score": 8.371273040771484 }, { "rank": 5, "tokenId": 24639, "text": "transition", "score": 7.153998851776123 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": 5.907034397125244 }, { "rank": 7, "tokenId": 51775, "text": "gain", "score": 5.764074325561523 }, { "rank": 8, "tokenId": 43279, "text": "Loss", "score": 5.668814659118652 } ] }, "f32_tree64": { "winnerTokenId": 112605, "winnerText": "freedom", "winnerScore": 21.594297409057617, "winnerGap": 9.780214309692383, "top": [ { "rank": 1, "tokenId": 112605, "text": "freedom", "score": 21.594297409057617 }, { "rank": 2, "tokenId": 106729, "text": "Freedom", "score": 11.814083099365234 }, { "rank": 3, "tokenId": 10558, "text": "loss", "score": 9.116521835327148 }, { "rank": 4, "tokenId": 5285, "text": "lib", "score": 8.371273040771484 }, { "rank": 5, "tokenId": 24639, "text": "transition", "score": 7.154002666473389 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": 5.907035827636719 }, { "rank": 7, "tokenId": 51775, "text": "gain", "score": 5.764072418212891 }, { "rank": 8, "tokenId": 43279, "text": "Loss", "score": 5.6688103675842285 } ] }, "f16_forward": { "winnerTokenId": 112605, "winnerText": "freedom", "winnerScore": 21.75, "winnerGap": 9.9765625, "top": [ { "rank": 1, "tokenId": 112605, "text": "freedom", "score": 21.75 }, { "rank": 2, "tokenId": 106729, "text": "Freedom", "score": 11.7734375 }, { "rank": 3, "tokenId": 10558, "text": "loss", "score": 9.0625 }, { "rank": 4, "tokenId": 24639, "text": "transition", "score": 7.2421875 }, { "rank": 5, "tokenId": 27413, "text": "both", "score": 5.9453125 }, { "rank": 6, "tokenId": 51775, "text": "gain", "score": 5.83203125 }, { "rank": 7, "tokenId": 43279, "text": "Loss", "score": 5.82421875 }, { "rank": 8, "tokenId": 204081, "text": "uncertain", "score": 5.44921875 } ] } }, "winnerTextsByMode": { "exact": "freedom", "f32_forward": "freedom", "f32_reverse": "freedom", "f32_tree64": "freedom", "f16_forward": "freedom" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "freedom" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "friend-stranger-explicit-oneword-c", "text": "No explanation. One word only.\nAnswer with exactly one word: friend or stranger. Question: Someone you know only online is a Answer:", "stableTopK": [ { "rank": 1, "tokenId": 1714, "text": "str", "logit": 14.776833534240723, "prob": 0.9889427423477173 }, { "rank": 2, "tokenId": 9933, "text": "friend", "logit": 9.832131385803223, "prob": 0.007042296230792999 }, { "rank": 3, "tokenId": 211008, "text": "Stranger", "logit": 9.246501922607422, "prob": 0.00392083590850234 }, { "rank": 4, "tokenId": 37662, "text": "Friend", "logit": 4.883725166320801, "prob": 0.0000499630332342349 }, { "rank": 5, "tokenId": 11267, "text": "Str", "logit": 4.630822658538818, "prob": 0.000038798472814960405 }, { "rank": 6, "tokenId": 42030, "text": " stranger", "logit": 2.424205780029297, "prob": 0.000004270641511538997 }, { "rank": 7, "tokenId": 27413, "text": "both", "logit": -0.4842827320098877, "prob": 2.3299820384181658e-7 }, { "rank": 8, "tokenId": 184095, "text": "strange", "logit": -0.8237512707710266, "prob": 1.6592936447068496e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 1714, "winnerText": "str", "winnerScore": 16.183088357105635, "winnerGap": 5.974348159603199, "top": [ { "rank": 1, "tokenId": 1714, "text": "str", "score": 16.183088357105635 }, { "rank": 2, "tokenId": 9933, "text": "friend", "score": 10.208740197502436 }, { "rank": 3, "tokenId": 211008, "text": "Stranger", "score": 9.557196038616446 }, { "rank": 4, "tokenId": 37662, "text": "Friend", "score": 4.927569458118834 }, { "rank": 5, "tokenId": 11267, "text": "Str", "score": 4.668137214854287 }, { "rank": 6, "tokenId": 42030, "text": " stranger", "score": 2.429503457781969 }, { "rank": 7, "tokenId": 27413, "text": "both", "score": -0.48432798355686657 }, { "rank": 8, "tokenId": 184095, "text": "strange", "score": -0.8239555624343495 } ] }, "f32_forward": { "winnerTokenId": 1714, "winnerText": "str", "winnerScore": 16.183095932006836, "winnerGap": 5.974350929260254, "top": [ { "rank": 1, "tokenId": 1714, "text": "str", "score": 16.183095932006836 }, { "rank": 2, "tokenId": 9933, "text": "friend", "score": 10.208745002746582 }, { "rank": 3, "tokenId": 211008, "text": "Stranger", "score": 9.557190895080566 }, { "rank": 4, "tokenId": 37662, "text": "Friend", "score": 4.927568435668945 }, { "rank": 5, "tokenId": 11267, "text": "Str", "score": 4.6681294441223145 }, { "rank": 6, "tokenId": 42030, "text": " stranger", "score": 2.4295032024383545 }, { "rank": 7, "tokenId": 27413, "text": "both", "score": -0.48432761430740356 }, { "rank": 8, "tokenId": 184095, "text": "strange", "score": -0.8239533305168152 } ] }, "f32_reverse": { "winnerTokenId": 1714, "winnerText": "str", "winnerScore": 16.18308448791504, "winnerGap": 5.974343299865723, "top": [ { "rank": 1, "tokenId": 1714, "text": "str", "score": 16.18308448791504 }, { "rank": 2, "tokenId": 9933, "text": "friend", "score": 10.208741188049316 }, { "rank": 3, "tokenId": 211008, "text": "Stranger", "score": 9.557202339172363 }, { "rank": 4, "tokenId": 37662, "text": "Friend", "score": 4.927567481994629 }, { "rank": 5, "tokenId": 11267, "text": "Str", "score": 4.668132781982422 }, { "rank": 6, "tokenId": 42030, "text": " stranger", "score": 2.429511547088623 }, { "rank": 7, "tokenId": 27413, "text": "both", "score": -0.4843301773071289 }, { "rank": 8, "tokenId": 184095, "text": "strange", "score": -0.823948860168457 } ] }, "f32_tree64": { "winnerTokenId": 1714, "winnerText": "str", "winnerScore": 16.183088302612305, "winnerGap": 5.974348068237305, "top": [ { "rank": 1, "tokenId": 1714, "text": "str", "score": 16.183088302612305 }, { "rank": 2, "tokenId": 9933, "text": "friend", "score": 10.208740234375 }, { "rank": 3, "tokenId": 211008, "text": "Stranger", "score": 9.557194709777832 }, { "rank": 4, "tokenId": 37662, "text": "Friend", "score": 4.927569389343262 }, { "rank": 5, "tokenId": 11267, "text": "Str", "score": 4.668137073516846 }, { "rank": 6, "tokenId": 42030, "text": " stranger", "score": 2.4295034408569336 }, { "rank": 7, "tokenId": 27413, "text": "both", "score": -0.484328031539917 }, { "rank": 8, "tokenId": 184095, "text": "strange", "score": -0.8239563703536987 } ] }, "f16_forward": { "winnerTokenId": 1714, "winnerText": "str", "winnerScore": 12.2265625, "winnerGap": 1.9453125, "top": [ { "rank": 1, "tokenId": 1714, "text": "str", "score": 12.2265625 }, { "rank": 2, "tokenId": 9933, "text": "friend", "score": 10.28125 }, { "rank": 3, "tokenId": 211008, "text": "Stranger", "score": 5.453125 }, { "rank": 4, "tokenId": 37662, "text": "Friend", "score": 4.94140625 }, { "rank": 5, "tokenId": 11267, "text": "Str", "score": 4.74609375 }, { "rank": 6, "tokenId": 42030, "text": " stranger", "score": 1.435546875 }, { "rank": 7, "tokenId": 27413, "text": "both", "score": -0.3828125 }, { "rank": 8, "tokenId": 184095, "text": "strange", "score": -0.82763671875 } ] } }, "winnerTextsByMode": { "exact": "str", "f32_forward": "str", "f32_reverse": "str", "f32_tree64": "str", "f16_forward": "str" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "str" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "grief-relief-both-oneword-d", "text": "Choose one word only.\nGrief, relief, or both: moving a parent into hospice can bring", "stableTopK": [ { "rank": 1, "tokenId": 236823, "text": "G", "logit": 19.249052047729492, "prob": 0.9046476483345032 }, { "rank": 2, "tokenId": 1018, "text": "**", "logit": 15.910175323486328, "prob": 0.032093990594148636 }, { "rank": 3, "tokenId": 103056, "text": "relief", "logit": 15.899162292480469, "prob": 0.03174247965216637 }, { "rank": 4, "tokenId": 236759, "text": "g", "logit": 15.563302993774414, "prob": 0.022687099874019623 }, { "rank": 5, "tokenId": 11146, "text": "Rel", "logit": 14.380851745605469, "prob": 0.006954195909202099 }, { "rank": 6, "tokenId": 41831, "text": "moving", "logit": 11.823176383972168, "prob": 0.0005388434510678053 }, { "rank": 7, "tokenId": 89237, "text": "suff", "logit": 11.456177711486816, "prob": 0.0003733164630830288 }, { "rank": 8, "tokenId": 38793, "text": "Moving", "logit": 11.406942367553711, "prob": 0.0003553812566678971 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 236823, "winnerText": "G", "winnerScore": 22.82844165781527, "winnerGap": 5.109935201707081, "top": [ { "rank": 1, "tokenId": 236823, "text": "G", "score": 22.82844165781527 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 17.718506456108187 }, { "rank": 3, "tokenId": 103056, "text": "relief", "score": 17.703190459387095 }, { "rank": 4, "tokenId": 236759, "text": "g", "score": 17.239936921578334 }, { "rank": 5, "tokenId": 11146, "text": "Rel", "score": 15.664654806675983 }, { "rank": 6, "tokenId": 41831, "text": "moving", "score": 12.49953749043625 }, { "rank": 7, "tokenId": 89237, "text": "suff", "score": 12.067496790150699 }, { "rank": 8, "tokenId": 38793, "text": "Moving", "score": 12.00990648693957 } ] }, "f32_forward": { "winnerTokenId": 236823, "winnerText": "G", "winnerScore": 22.828433990478516, "winnerGap": 5.109949111938477, "top": [ { "rank": 1, "tokenId": 236823, "text": "G", "score": 22.828433990478516 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 17.71848487854004 }, { "rank": 3, "tokenId": 103056, "text": "relief", "score": 17.703155517578125 }, { "rank": 4, "tokenId": 236759, "text": "g", "score": 17.23993682861328 }, { "rank": 5, "tokenId": 11146, "text": "Rel", "score": 15.66464900970459 }, { "rank": 6, "tokenId": 41831, "text": "moving", "score": 12.499531745910645 }, { "rank": 7, "tokenId": 89237, "text": "suff", "score": 12.067508697509766 }, { "rank": 8, "tokenId": 38793, "text": "Moving", "score": 12.009894371032715 } ] }, "f32_reverse": { "winnerTokenId": 236823, "winnerText": "G", "winnerScore": 22.82843589782715, "winnerGap": 5.109918594360352, "top": [ { "rank": 1, "tokenId": 236823, "text": "G", "score": 22.82843589782715 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 17.718517303466797 }, { "rank": 3, "tokenId": 103056, "text": "relief", "score": 17.70320701599121 }, { "rank": 4, "tokenId": 236759, "text": "g", "score": 17.239940643310547 }, { "rank": 5, "tokenId": 11146, "text": "Rel", "score": 15.664657592773438 }, { "rank": 6, "tokenId": 41831, "text": "moving", "score": 12.499527931213379 }, { "rank": 7, "tokenId": 89237, "text": "suff", "score": 12.06750202178955 }, { "rank": 8, "tokenId": 38793, "text": "Moving", "score": 12.009897232055664 } ] }, "f32_tree64": { "winnerTokenId": 236823, "winnerText": "G", "winnerScore": 22.828441619873047, "winnerGap": 5.109933853149414, "top": [ { "rank": 1, "tokenId": 236823, "text": "G", "score": 22.828441619873047 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 17.718507766723633 }, { "rank": 3, "tokenId": 103056, "text": "relief", "score": 17.703189849853516 }, { "rank": 4, "tokenId": 236759, "text": "g", "score": 17.23993682861328 }, { "rank": 5, "tokenId": 11146, "text": "Rel", "score": 15.664653778076172 }, { "rank": 6, "tokenId": 41831, "text": "moving", "score": 12.49953556060791 }, { "rank": 7, "tokenId": 89237, "text": "suff", "score": 12.067498207092285 }, { "rank": 8, "tokenId": 38793, "text": "Moving", "score": 12.009906768798828 } ] }, "f16_forward": { "winnerTokenId": 236823, "winnerText": "G", "winnerScore": 23.015625, "winnerGap": 5.3125, "top": [ { "rank": 1, "tokenId": 236823, "text": "G", "score": 23.015625 }, { "rank": 2, "tokenId": 103056, "text": "relief", "score": 17.703125 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 17.6875 }, { "rank": 4, "tokenId": 236759, "text": "g", "score": 17.28125 }, { "rank": 5, "tokenId": 11146, "text": "Rel", "score": 15.6484375 }, { "rank": 6, "tokenId": 89237, "text": "suff", "score": 12.234375 }, { "rank": 7, "tokenId": 38793, "text": "Moving", "score": 12.046875 }, { "rank": 8, "tokenId": 22186, "text": "Both", "score": 10.109375 } ] } }, "winnerTextsByMode": { "exact": "G", "f32_forward": "G", "f32_reverse": "G", "f32_tree64": "G", "f16_forward": "G" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "G" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "grief-relief-explicit-oneword-a", "text": "Answer with exactly one word only.\nAnswer with exactly one word: grief or relief. Question: Moving a parent into hospice can bring Answer:", "stableTopK": [ { "rank": 1, "tokenId": 236759, "text": "g", "logit": 13.608177185058594, "prob": 0.8266564011573792 }, { "rank": 2, "tokenId": 103056, "text": "relief", "logit": 12.032734870910645, "prob": 0.17104844748973846 }, { "rank": 3, "tokenId": 236823, "text": "G", "logit": 7.173924446105957, "prob": 0.0013272862415760756 }, { "rank": 4, "tokenId": 11146, "text": "Rel", "logit": 6.7641143798828125, "prob": 0.0008810211438685656 }, { "rank": 5, "tokenId": 37767, "text": "brief", "logit": 2.6194446086883545, "prob": 0.00001396301195200067 }, { "rank": 6, "tokenId": 2921, "text": "rel", "logit": 2.4572501182556152, "prob": 0.000011872411960212048 }, { "rank": 7, "tokenId": 89237, "text": "suff", "logit": 2.442798376083374, "prob": 0.00001170206905953819 }, { "rank": 8, "tokenId": 27413, "text": "both", "logit": 2.164295196533203, "prob": 0.000008857481589075178 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 236759, "winnerText": "g", "winnerScore": 14.676934920992359, "winnerGap": 1.9284805336343211, "top": [ { "rank": 1, "tokenId": 236759, "text": "g", "score": 14.676934920992359 }, { "rank": 2, "tokenId": 103056, "text": "relief", "score": 12.748454387358038 }, { "rank": 3, "tokenId": 236823, "text": "G", "score": 7.315560593202047 }, { "rank": 4, "tokenId": 11146, "text": "Rel", "score": 6.882367684431706 }, { "rank": 5, "tokenId": 37767, "text": "brief", "score": 2.626134676600243 }, { "rank": 6, "tokenId": 2921, "text": "rel", "score": 2.4627688846162954 }, { "rank": 7, "tokenId": 89237, "text": "suff", "score": 2.448218809882732 }, { "rank": 8, "tokenId": 27413, "text": "both", "score": 2.168064052711756 } ] }, "f32_forward": { "winnerTokenId": 236759, "winnerText": "g", "winnerScore": 14.676933288574219, "winnerGap": 1.9284839630126953, "top": [ { "rank": 1, "tokenId": 236759, "text": "g", "score": 14.676933288574219 }, { "rank": 2, "tokenId": 103056, "text": "relief", "score": 12.748449325561523 }, { "rank": 3, "tokenId": 236823, "text": "G", "score": 7.315557956695557 }, { "rank": 4, "tokenId": 11146, "text": "Rel", "score": 6.882366180419922 }, { "rank": 5, "tokenId": 37767, "text": "brief", "score": 2.6261281967163086 }, { "rank": 6, "tokenId": 2921, "text": "rel", "score": 2.462759256362915 }, { "rank": 7, "tokenId": 89237, "text": "suff", "score": 2.4482176303863525 }, { "rank": 8, "tokenId": 27413, "text": "both", "score": 2.168062925338745 } ] }, "f32_reverse": { "winnerTokenId": 236759, "winnerText": "g", "winnerScore": 14.676939010620117, "winnerGap": 1.928471565246582, "top": [ { "rank": 1, "tokenId": 236759, "text": "g", "score": 14.676939010620117 }, { "rank": 2, "tokenId": 103056, "text": "relief", "score": 12.748467445373535 }, { "rank": 3, "tokenId": 236823, "text": "G", "score": 7.315563678741455 }, { "rank": 4, "tokenId": 11146, "text": "Rel", "score": 6.8823652267456055 }, { "rank": 5, "tokenId": 37767, "text": "brief", "score": 2.6261377334594727 }, { "rank": 6, "tokenId": 2921, "text": "rel", "score": 2.4627723693847656 }, { "rank": 7, "tokenId": 89237, "text": "suff", "score": 2.4482200145721436 }, { "rank": 8, "tokenId": 27413, "text": "both", "score": 2.168062448501587 } ] }, "f32_tree64": { "winnerTokenId": 236759, "winnerText": "g", "winnerScore": 14.676935195922852, "winnerGap": 1.9284801483154297, "top": [ { "rank": 1, "tokenId": 236759, "text": "g", "score": 14.676935195922852 }, { "rank": 2, "tokenId": 103056, "text": "relief", "score": 12.748455047607422 }, { "rank": 3, "tokenId": 236823, "text": "G", "score": 7.315558433532715 }, { "rank": 4, "tokenId": 11146, "text": "Rel", "score": 6.882367134094238 }, { "rank": 5, "tokenId": 37767, "text": "brief", "score": 2.6261343955993652 }, { "rank": 6, "tokenId": 2921, "text": "rel", "score": 2.4627685546875 }, { "rank": 7, "tokenId": 89237, "text": "suff", "score": 2.4482195377349854 }, { "rank": 8, "tokenId": 27413, "text": "both", "score": 2.1680641174316406 } ] }, "f16_forward": { "winnerTokenId": 103056, "winnerText": "relief", "winnerScore": 12.625, "winnerGap": 5.24609375, "top": [ { "rank": 1, "tokenId": 103056, "text": "relief", "score": 12.625 }, { "rank": 2, "tokenId": 236823, "text": "G", "score": 7.37890625 }, { "rank": 3, "tokenId": 11146, "text": "Rel", "score": 6.9375 }, { "rank": 4, "tokenId": 236759, "text": "g", "score": 6.671875 }, { "rank": 5, "tokenId": 37767, "text": "brief", "score": 2.595703125 }, { "rank": 6, "tokenId": 89237, "text": "suff", "score": 2.4140625 }, { "rank": 7, "tokenId": 2921, "text": "rel", "score": 2.34375 }, { "rank": 8, "tokenId": 27413, "text": "both", "score": 2.125 } ] } }, "winnerTextsByMode": { "exact": "g", "f32_forward": "g", "f32_reverse": "g", "f32_tree64": "g", "f16_forward": "relief" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "g", "relief" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 236759, 6287 ], "decodedText": "grief", "steps": [ { "step": 0, "tokenId": 236759, "text": "g" }, { "step": 1, "tokenId": 6287, "text": "rief" } ] }, "f16_forward": { "tokenIds": [ 103056, 106 ], "decodedText": "relief", "steps": [ { "step": 0, "tokenId": 103056, "text": "relief" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "healing-harmful-both-oneword-b", "text": "Respond with one word only.\nHealing, harmful, or both: reopening trauma in therapy can be", "stableTopK": [ { "rank": 1, "tokenId": 99510, "text": "Ris", "logit": 10.75629711151123, "prob": 0.5850602984428406 }, { "rank": 2, "tokenId": 4499, "text": "ris", "logit": 9.54753303527832, "prob": 0.17467913031578064 }, { "rank": 3, "tokenId": 33071, "text": "Pot", "logit": 8.112290382385254, "prob": 0.04158369451761246 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "logit": 7.847367763519287, "prob": 0.031905725598335266 }, { "rank": 5, "tokenId": 1659, "text": "Ch", "logit": 7.6603779792785645, "prob": 0.026464276015758514 }, { "rank": 6, "tokenId": 191239, "text": "Difficult", "logit": 7.524017333984375, "prob": 0.023090820759534836 }, { "rank": 7, "tokenId": 2035, "text": "Tr", "logit": 7.065372467041016, "prob": 0.014596623368561268 }, { "rank": 8, "tokenId": 71398, "text": "Risk", "logit": 7.0204901695251465, "prob": 0.013955977745354176 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 99510, "winnerText": "Ris", "winnerScore": 11.256397465066769, "winnerGap": 1.3654024780434568, "top": [ { "rank": 1, "tokenId": 99510, "text": "Ris", "score": 11.256397465066769 }, { "rank": 2, "tokenId": 4499, "text": "ris", "score": 9.890994987023312 }, { "rank": 3, "tokenId": 33071, "text": "Pot", "score": 8.319175332555687 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 8.03407315632532 }, { "rank": 5, "tokenId": 1659, "text": "Ch", "score": 7.8337003467369755 }, { "rank": 6, "tokenId": 191239, "text": "Difficult", "score": 7.688008036579554 }, { "rank": 7, "tokenId": 2035, "text": "Tr", "score": 7.200528433098638 }, { "rank": 8, "tokenId": 71398, "text": "Risk", "score": 7.153028211055528 } ] }, "f32_forward": { "winnerTokenId": 99510, "winnerText": "Ris", "winnerScore": 11.256403923034668, "winnerGap": 1.365407943725586, "top": [ { "rank": 1, "tokenId": 99510, "text": "Ris", "score": 11.256403923034668 }, { "rank": 2, "tokenId": 4499, "text": "ris", "score": 9.890995979309082 }, { "rank": 3, "tokenId": 33071, "text": "Pot", "score": 8.319180488586426 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 8.03407096862793 }, { "rank": 5, "tokenId": 1659, "text": "Ch", "score": 7.833699703216553 }, { "rank": 6, "tokenId": 191239, "text": "Difficult", "score": 7.68800687789917 }, { "rank": 7, "tokenId": 2035, "text": "Tr", "score": 7.200530052185059 }, { "rank": 8, "tokenId": 71398, "text": "Risk", "score": 7.153027057647705 } ] }, "f32_reverse": { "winnerTokenId": 99510, "winnerText": "Ris", "winnerScore": 11.256401062011719, "winnerGap": 1.3654136657714844, "top": [ { "rank": 1, "tokenId": 99510, "text": "Ris", "score": 11.256401062011719 }, { "rank": 2, "tokenId": 4499, "text": "ris", "score": 9.890987396240234 }, { "rank": 3, "tokenId": 33071, "text": "Pot", "score": 8.319177627563477 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 8.034077644348145 }, { "rank": 5, "tokenId": 1659, "text": "Ch", "score": 7.833702087402344 }, { "rank": 6, "tokenId": 191239, "text": "Difficult", "score": 7.6880059242248535 }, { "rank": 7, "tokenId": 2035, "text": "Tr", "score": 7.200526237487793 }, { "rank": 8, "tokenId": 71398, "text": "Risk", "score": 7.153028964996338 } ] }, "f32_tree64": { "winnerTokenId": 99510, "winnerText": "Ris", "winnerScore": 11.25639820098877, "winnerGap": 1.3654050827026367, "top": [ { "rank": 1, "tokenId": 99510, "text": "Ris", "score": 11.25639820098877 }, { "rank": 2, "tokenId": 4499, "text": "ris", "score": 9.890993118286133 }, { "rank": 3, "tokenId": 33071, "text": "Pot", "score": 8.31917667388916 }, { "rank": 4, "tokenId": 48333, "text": "Complex", "score": 8.034072875976562 }, { "rank": 5, "tokenId": 1659, "text": "Ch", "score": 7.833699703216553 }, { "rank": 6, "tokenId": 191239, "text": "Difficult", "score": 7.6880083084106445 }, { "rank": 7, "tokenId": 2035, "text": "Tr", "score": 7.200528144836426 }, { "rank": 8, "tokenId": 71398, "text": "Risk", "score": 7.15302848815918 } ] }, "f16_forward": { "winnerTokenId": 99510, "winnerText": "Ris", "winnerScore": 11.375, "winnerGap": 2.9921875, "top": [ { "rank": 1, "tokenId": 99510, "text": "Ris", "score": 11.375 }, { "rank": 2, "tokenId": 33071, "text": "Pot", "score": 8.3828125 }, { "rank": 3, "tokenId": 1659, "text": "Ch", "score": 7.88671875 }, { "rank": 4, "tokenId": 2035, "text": "Tr", "score": 7.21875 }, { "rank": 5, "tokenId": 43510, "text": "problem", "score": 6.5625 }, { "rank": 6, "tokenId": 34622, "text": "complex", "score": 6.55859375 }, { "rank": 7, "tokenId": 139002, "text": "challeng", "score": 6.4375 }, { "rank": 8, "tokenId": 229224, "text": "Necessary", "score": 6.26953125 } ] } }, "winnerTextsByMode": { "exact": "Ris", "f32_forward": "Ris", "f32_reverse": "Ris", "f32_tree64": "Ris", "f16_forward": "Ris" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Ris" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "hello-oneword-c", "text": "No explanation. One word only.\nHello", "stableTopK": [ { "rank": 1, "tokenId": 10979, "text": "Hi", "logit": -0.021134641021490097, "prob": 0.9599158763885498 }, { "rank": 2, "tokenId": 9259, "text": "Hello", "logit": -4.017408847808838, "prob": 0.017647098749876022 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "logit": -4.873048305511475, "prob": 0.007500216830521822 }, { "rank": 4, "tokenId": 21529, "text": "Hola", "logit": -5.075793743133545, "prob": 0.006123822648078203 }, { "rank": 5, "tokenId": 236814, "text": "H", "logit": -6.1572747230529785, "prob": 0.0020765450317412615 }, { "rank": 6, "tokenId": 3810, "text": "There", "logit": -6.532878398895264, "prob": 0.0014263258781284094 }, { "rank": 7, "tokenId": 136408, "text": "Greeting", "logit": -6.566631317138672, "prob": 0.0013789867516607046 }, { "rank": 8, "tokenId": 125508, "text": "Greetings", "logit": -6.676671504974365, "prob": 0.0012352936901152134 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 10979, "winnerText": "Hi", "winnerScore": -0.02113629159885022, "winnerGap": 4.020544814009952, "top": [ { "rank": 1, "tokenId": 10979, "text": "Hi", "score": -0.02113629159885022 }, { "rank": 2, "tokenId": 9259, "text": "Hello", "score": -4.041681105608802 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": -4.916598432506106 }, { "rank": 4, "tokenId": 21529, "text": "Hola", "score": -5.12508053246691 }, { "rank": 5, "tokenId": 236814, "text": "H", "score": -6.245987318772206 }, { "rank": 6, "tokenId": 3810, "text": "There", "score": -6.639182663788009 }, { "rank": 7, "tokenId": 136408, "text": "Greeting", "score": -6.67462527390947 }, { "rank": 8, "tokenId": 125508, "text": "Greetings", "score": -6.790297277951456 } ] }, "f32_forward": { "winnerTokenId": 10979, "winnerText": "Hi", "winnerScore": -0.021137330681085587, "winnerGap": 4.02054300531745, "top": [ { "rank": 1, "tokenId": 10979, "text": "Hi", "score": -0.021137330681085587 }, { "rank": 2, "tokenId": 9259, "text": "Hello", "score": -4.041680335998535 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": -4.916599750518799 }, { "rank": 4, "tokenId": 21529, "text": "Hola", "score": -5.1250810623168945 }, { "rank": 5, "tokenId": 236814, "text": "H", "score": -6.245985507965088 }, { "rank": 6, "tokenId": 3810, "text": "There", "score": -6.639186382293701 }, { "rank": 7, "tokenId": 136408, "text": "Greeting", "score": -6.674621105194092 }, { "rank": 8, "tokenId": 125508, "text": "Greetings", "score": -6.790299415588379 } ] }, "f32_reverse": { "winnerTokenId": 10979, "winnerText": "Hi", "winnerScore": -0.021139439195394516, "winnerGap": 4.02054375782609, "top": [ { "rank": 1, "tokenId": 10979, "text": "Hi", "score": -0.021139439195394516 }, { "rank": 2, "tokenId": 9259, "text": "Hello", "score": -4.041683197021484 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": -4.916594982147217 }, { "rank": 4, "tokenId": 21529, "text": "Hola", "score": -5.125077247619629 }, { "rank": 5, "tokenId": 236814, "text": "H", "score": -6.245983600616455 }, { "rank": 6, "tokenId": 3810, "text": "There", "score": -6.63918399810791 }, { "rank": 7, "tokenId": 136408, "text": "Greeting", "score": -6.674618244171143 }, { "rank": 8, "tokenId": 125508, "text": "Greetings", "score": -6.7902960777282715 } ] }, "f32_tree64": { "winnerTokenId": 10979, "winnerText": "Hi", "winnerScore": -0.021136760711669922, "winnerGap": 4.020544528961182, "top": [ { "rank": 1, "tokenId": 10979, "text": "Hi", "score": -0.021136760711669922 }, { "rank": 2, "tokenId": 9259, "text": "Hello", "score": -4.041681289672852 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": -4.916598796844482 }, { "rank": 4, "tokenId": 21529, "text": "Hola", "score": -5.125080585479736 }, { "rank": 5, "tokenId": 236814, "text": "H", "score": -6.245987892150879 }, { "rank": 6, "tokenId": 3810, "text": "There", "score": -6.639183521270752 }, { "rank": 7, "tokenId": 136408, "text": "Greeting", "score": -6.674624443054199 }, { "rank": 8, "tokenId": 125508, "text": "Greetings", "score": -6.7902960777282715 } ] }, "f16_forward": { "winnerTokenId": 10979, "winnerText": "Hi", "winnerScore": 0.0035247802734375, "winnerGap": 3.4332122802734375, "top": [ { "rank": 1, "tokenId": 10979, "text": "Hi", "score": 0.0035247802734375 }, { "rank": 2, "tokenId": 16651, "text": "Welcome", "score": -3.4296875 }, { "rank": 3, "tokenId": 9259, "text": "Hello", "score": -4.109375 }, { "rank": 4, "tokenId": 53406, "text": "Bonjour", "score": -4.36328125 }, { "rank": 5, "tokenId": 99963, "text": "Gracias", "score": -4.84375 }, { "rank": 6, "tokenId": 10784, "text": "Yes", "score": -4.91796875 }, { "rank": 7, "tokenId": 21529, "text": "Hola", "score": -5.12109375 }, { "rank": 8, "tokenId": 85141, "text": "こんにちは", "score": -5.63671875 } ] } }, "winnerTextsByMode": { "exact": "Hi", "f32_forward": "Hi", "f32_reverse": "Hi", "f32_tree64": "Hi", "f16_forward": "Hi" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Hi" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "hotfix-approve-deny-choice-oneword-d", "text": "Choose one word only.\nChoose exactly one word: approve or deny.\nAn unsigned production hotfix should be", "stableTopK": [ { "rank": 1, "tokenId": 205380, "text": "deny", "logit": 21.692901611328125, "prob": 0.8058179616928101 }, { "rank": 2, "tokenId": 22472, "text": "Den", "logit": 20.19867515563965, "prob": 0.18084339797496796 }, { "rank": 3, "tokenId": 1018, "text": "**", "logit": 17.531648635864258, "prob": 0.012561102397739887 }, { "rank": 4, "tokenId": 2235, "text": "den", "logit": 14.151747703552246, "prob": 0.00042771591688506305 }, { "rank": 5, "tokenId": 105961, "text": "approve", "logit": 13.644671440124512, "prob": 0.0002575935795903206 }, { "rank": 6, "tokenId": 2046, "text": "App", "logit": 12.377199172973633, "prob": 0.00007252350769704208 }, { "rank": 7, "tokenId": 30590, "text": " deny", "logit": 10.0826416015625, "prob": 0.000007310803084692452 }, { "rank": 8, "tokenId": 3292, "text": "$\\", "logit": 8.650113105773926, "prob": 0.000001745122403917776 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 205380, "winnerText": "deny", "winnerScore": 27.423156190865868, "winnerGap": 2.9210855475964053, "top": [ { "rank": 1, "tokenId": 205380, "text": "deny", "score": 27.423156190865868 }, { "rank": 2, "tokenId": 22472, "text": "Den", "score": 24.502070643269462 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 20.073033467720656 }, { "rank": 4, "tokenId": 2235, "text": "den", "score": 15.368598172015922 }, { "rank": 5, "tokenId": 105961, "text": "approve", "score": 14.72290897225895 }, { "rank": 6, "tokenId": 2046, "text": "App", "score": 13.161253306279214 }, { "rank": 7, "tokenId": 30590, "text": " deny", "score": 10.490266100204941 }, { "rank": 8, "tokenId": 3292, "text": "$\\", "score": 8.902554653097894 } ] }, "f32_forward": { "winnerTokenId": 205380, "winnerText": "deny", "winnerScore": 27.42316246032715, "winnerGap": 2.921102523803711, "top": [ { "rank": 1, "tokenId": 205380, "text": "deny", "score": 27.42316246032715 }, { "rank": 2, "tokenId": 22472, "text": "Den", "score": 24.502059936523438 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 20.07303237915039 }, { "rank": 4, "tokenId": 2235, "text": "den", "score": 15.36860179901123 }, { "rank": 5, "tokenId": 105961, "text": "approve", "score": 14.722931861877441 }, { "rank": 6, "tokenId": 2046, "text": "App", "score": 13.161249160766602 }, { "rank": 7, "tokenId": 30590, "text": " deny", "score": 10.490265846252441 }, { "rank": 8, "tokenId": 3292, "text": "$\\", "score": 8.902538299560547 } ] }, "f32_reverse": { "winnerTokenId": 205380, "winnerText": "deny", "winnerScore": 27.42314338684082, "winnerGap": 2.921060562133789, "top": [ { "rank": 1, "tokenId": 205380, "text": "deny", "score": 27.42314338684082 }, { "rank": 2, "tokenId": 22472, "text": "Den", "score": 24.50208282470703 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 20.073020935058594 }, { "rank": 4, "tokenId": 2235, "text": "den", "score": 15.368608474731445 }, { "rank": 5, "tokenId": 105961, "text": "approve", "score": 14.722907066345215 }, { "rank": 6, "tokenId": 2046, "text": "App", "score": 13.161255836486816 }, { "rank": 7, "tokenId": 30590, "text": " deny", "score": 10.490262031555176 }, { "rank": 8, "tokenId": 3292, "text": "$\\", "score": 8.902552604675293 } ] }, "f32_tree64": { "winnerTokenId": 205380, "winnerText": "deny", "winnerScore": 27.42315673828125, "winnerGap": 2.9210872650146484, "top": [ { "rank": 1, "tokenId": 205380, "text": "deny", "score": 27.42315673828125 }, { "rank": 2, "tokenId": 22472, "text": "Den", "score": 24.5020694732666 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 20.07303237915039 }, { "rank": 4, "tokenId": 2235, "text": "den", "score": 15.368599891662598 }, { "rank": 5, "tokenId": 105961, "text": "approve", "score": 14.722908020019531 }, { "rank": 6, "tokenId": 2046, "text": "App", "score": 13.161252975463867 }, { "rank": 7, "tokenId": 30590, "text": " deny", "score": 10.490264892578125 }, { "rank": 8, "tokenId": 3292, "text": "$\\", "score": 8.902554512023926 } ] }, "f16_forward": { "winnerTokenId": 205380, "winnerText": "deny", "winnerScore": 27.53125, "winnerGap": 2.921875, "top": [ { "rank": 1, "tokenId": 205380, "text": "deny", "score": 27.53125 }, { "rank": 2, "tokenId": 22472, "text": "Den", "score": 24.609375 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 20.234375 }, { "rank": 4, "tokenId": 105961, "text": "approve", "score": 14.8828125 }, { "rank": 5, "tokenId": 2235, "text": "den", "score": 13.390625 }, { "rank": 6, "tokenId": 2046, "text": "App", "score": 13.1171875 }, { "rank": 7, "tokenId": 30590, "text": " deny", "score": 10.4375 }, { "rank": 8, "tokenId": 3292, "text": "$\\", "score": 9.0078125 } ] } }, "winnerTextsByMode": { "exact": "deny", "f32_forward": "deny", "f32_reverse": "deny", "f32_tree64": "deny", "f16_forward": "deny" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "deny" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "ice-temperature-oneword-a", "text": "Answer with exactly one word only.\nIce is usually", "stableTopK": [ { "rank": 1, "tokenId": 58632, "text": "Solid", "logit": 6.431631088256836, "prob": 0.995908796787262 }, { "rank": 2, "tokenId": 26252, "text": "solid", "logit": 0.231729194521904, "prob": 0.002021325984969735 }, { "rank": 3, "tokenId": 133657, "text": "Frozen", "logit": 0.21279454231262207, "prob": 0.0019834130071103573 }, { "rank": 4, "tokenId": 66020, "text": "Cold", "logit": -3.4750583171844482, "prob": 0.0000496362590638455 }, { "rank": 5, "tokenId": 17390, "text": "Water", "logit": -4.525218486785889, "prob": 0.0000173668195202481 }, { "rank": 6, "tokenId": 82028, "text": "frozen", "logit": -4.539444446563721, "prob": 0.000017121508790296502 }, { "rank": 7, "tokenId": 33231, "text": "Hard", "logit": -7.2193989753723145, "prob": 0.0000011739580259018112 }, { "rank": 8, "tokenId": 66471, "text": "cold", "logit": -8.044353485107422, "prob": 5.144928536537918e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 58632, "winnerText": "Solid", "winnerScore": 6.532982357237401, "winnerGap": 6.30124838499831, "top": [ { "rank": 1, "tokenId": 58632, "text": "Solid", "score": 6.532982357237401 }, { "rank": 2, "tokenId": 26252, "text": "solid", "score": 0.2317339722390912 }, { "rank": 3, "tokenId": 133657, "text": "Frozen", "score": 0.21279723245472582 }, { "rank": 4, "tokenId": 66020, "text": "Cold", "score": -3.4907228094698723 }, { "rank": 5, "tokenId": 17390, "text": "Water", "score": -4.560019739451853 }, { "rank": 6, "tokenId": 82028, "text": "frozen", "score": -4.574570139394282 }, { "rank": 7, "tokenId": 33231, "text": "Hard", "score": -7.363811931815128 }, { "rank": 8, "tokenId": 66471, "text": "cold", "score": -8.245912152448046 } ] }, "f32_forward": { "winnerTokenId": 58632, "winnerText": "Solid", "winnerScore": 6.532979488372803, "winnerGap": 6.301244854927063, "top": [ { "rank": 1, "tokenId": 58632, "text": "Solid", "score": 6.532979488372803 }, { "rank": 2, "tokenId": 26252, "text": "solid", "score": 0.23173463344573975 }, { "rank": 3, "tokenId": 133657, "text": "Frozen", "score": 0.21280470490455627 }, { "rank": 4, "tokenId": 66020, "text": "Cold", "score": -3.4907236099243164 }, { "rank": 5, "tokenId": 17390, "text": "Water", "score": -4.560023307800293 }, { "rank": 6, "tokenId": 82028, "text": "frozen", "score": -4.574565887451172 }, { "rank": 7, "tokenId": 33231, "text": "Hard", "score": -7.363812446594238 }, { "rank": 8, "tokenId": 66471, "text": "cold", "score": -8.245903015136719 } ] }, "f32_reverse": { "winnerTokenId": 58632, "winnerText": "Solid", "winnerScore": 6.532985687255859, "winnerGap": 6.3012515604496, "top": [ { "rank": 1, "tokenId": 58632, "text": "Solid", "score": 6.532985687255859 }, { "rank": 2, "tokenId": 26252, "text": "solid", "score": 0.23173412680625916 }, { "rank": 3, "tokenId": 133657, "text": "Frozen", "score": 0.21279700100421906 }, { "rank": 4, "tokenId": 66020, "text": "Cold", "score": -3.490720748901367 }, { "rank": 5, "tokenId": 17390, "text": "Water", "score": -4.56001091003418 }, { "rank": 6, "tokenId": 82028, "text": "frozen", "score": -4.574560165405273 }, { "rank": 7, "tokenId": 33231, "text": "Hard", "score": -7.3638153076171875 }, { "rank": 8, "tokenId": 66471, "text": "cold", "score": -8.245911598205566 } ] }, "f32_tree64": { "winnerTokenId": 58632, "winnerText": "Solid", "winnerScore": 6.532982349395752, "winnerGap": 6.301246881484985, "top": [ { "rank": 1, "tokenId": 58632, "text": "Solid", "score": 6.532982349395752 }, { "rank": 2, "tokenId": 26252, "text": "solid", "score": 0.2317354679107666 }, { "rank": 3, "tokenId": 133657, "text": "Frozen", "score": 0.21279597282409668 }, { "rank": 4, "tokenId": 66020, "text": "Cold", "score": -3.49072265625 }, { "rank": 5, "tokenId": 17390, "text": "Water", "score": -4.560019016265869 }, { "rank": 6, "tokenId": 82028, "text": "frozen", "score": -4.57457160949707 }, { "rank": 7, "tokenId": 33231, "text": "Hard", "score": -7.363813400268555 }, { "rank": 8, "tokenId": 66471, "text": "cold", "score": -8.245912551879883 } ] }, "f16_forward": { "winnerTokenId": 58632, "winnerText": "Solid", "winnerScore": 6.6171875, "winnerGap": 6.08251953125, "top": [ { "rank": 1, "tokenId": 58632, "text": "Solid", "score": 6.6171875 }, { "rank": 2, "tokenId": 133657, "text": "Frozen", "score": 0.53466796875 }, { "rank": 3, "tokenId": 26252, "text": "solid", "score": 0.1558837890625 }, { "rank": 4, "tokenId": 66020, "text": "Cold", "score": -3.40625 }, { "rank": 5, "tokenId": 66471, "text": "cold", "score": -4.2109375 }, { "rank": 6, "tokenId": 17390, "text": "Water", "score": -4.51953125 }, { "rank": 7, "tokenId": 82028, "text": "frozen", "score": -4.57421875 }, { "rank": 8, "tokenId": 236780, "text": "C", "score": -4.90625 } ] } }, "winnerTextsByMode": { "exact": "Solid", "f32_forward": "Solid", "f32_reverse": "Solid", "f32_tree64": "Solid", "f16_forward": "Solid" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Solid" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "json-array-open-oneword-b", "text": "Respond with one word only.\n[{\"type\":\"", "stableTopK": [ { "rank": 1, "tokenId": 3494, "text": "Error", "logit": 6.473020553588867, "prob": 0.7012916207313538 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "logit": 3.664665699005127, "prob": 0.04229077696800232 }, { "rank": 3, "tokenId": 3771, "text": "No", "logit": 3.6055660247802734, "prob": 0.03986382856965065 }, { "rank": 4, "tokenId": 1899, "text": "error", "logit": 3.4957900047302246, "prob": 0.0357193760573864 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "logit": 3.1471614837646484, "prob": 0.025205565616488457 }, { "rank": 6, "tokenId": 6302, "text": "Start", "logit": 2.924889326095581, "prob": 0.020182030275464058 }, { "rank": 7, "tokenId": 6455, "text": "null", "logit": 2.590571165084839, "prob": 0.014446822926402092 }, { "rank": 8, "tokenId": 4661, "text": "Input", "logit": 2.0627949237823486, "prob": 0.008522402495145798 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3494, "winnerText": "Error", "winnerScore": 6.576373488353159, "winnerGap": 2.893315288981885, "top": [ { "rank": 1, "tokenId": 3494, "text": "Error", "score": 6.576373488353159 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 3.683058199371274 }, { "rank": 3, "tokenId": 3771, "text": "No", "score": 3.623075463715132 }, { "rank": 4, "tokenId": 1899, "text": "error", "score": 3.5117402616966515 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "score": 3.1587838816968876 }, { "rank": 6, "tokenId": 6302, "text": "Start", "score": 2.934210519962786 }, { "rank": 7, "tokenId": 6455, "text": "null", "score": 2.59703852815926 }, { "rank": 8, "tokenId": 4661, "text": "Input", "score": 2.066055253251008 } ] }, "f32_forward": { "winnerTokenId": 3494, "winnerText": "Error", "winnerScore": 6.576372146606445, "winnerGap": 2.8933138847351074, "top": [ { "rank": 1, "tokenId": 3494, "text": "Error", "score": 6.576372146606445 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 3.683058261871338 }, { "rank": 3, "tokenId": 3771, "text": "No", "score": 3.623079776763916 }, { "rank": 4, "tokenId": 1899, "text": "error", "score": 3.5117404460906982 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "score": 3.158782720565796 }, { "rank": 6, "tokenId": 6302, "text": "Start", "score": 2.934208869934082 }, { "rank": 7, "tokenId": 6455, "text": "null", "score": 2.5970442295074463 }, { "rank": 8, "tokenId": 4661, "text": "Input", "score": 2.066058874130249 } ] }, "f32_reverse": { "winnerTokenId": 3494, "winnerText": "Error", "winnerScore": 6.576375961303711, "winnerGap": 2.8933186531066895, "top": [ { "rank": 1, "tokenId": 3494, "text": "Error", "score": 6.576375961303711 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 3.6830573081970215 }, { "rank": 3, "tokenId": 3771, "text": "No", "score": 3.6230733394622803 }, { "rank": 4, "tokenId": 1899, "text": "error", "score": 3.5117385387420654 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "score": 3.158784866333008 }, { "rank": 6, "tokenId": 6302, "text": "Start", "score": 2.9342122077941895 }, { "rank": 7, "tokenId": 6455, "text": "null", "score": 2.5970401763916016 }, { "rank": 8, "tokenId": 4661, "text": "Input", "score": 2.0660550594329834 } ] }, "f32_tree64": { "winnerTokenId": 3494, "winnerText": "Error", "winnerScore": 6.576374053955078, "winnerGap": 2.8933157920837402, "top": [ { "rank": 1, "tokenId": 3494, "text": "Error", "score": 6.576374053955078 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 3.683058261871338 }, { "rank": 3, "tokenId": 3771, "text": "No", "score": 3.6230757236480713 }, { "rank": 4, "tokenId": 1899, "text": "error", "score": 3.5117404460906982 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "score": 3.1587839126586914 }, { "rank": 6, "tokenId": 6302, "text": "Start", "score": 2.934210777282715 }, { "rank": 7, "tokenId": 6455, "text": "null", "score": 2.597038745880127 }, { "rank": 8, "tokenId": 4661, "text": "Input", "score": 2.0660557746887207 } ] }, "f16_forward": { "winnerTokenId": 3494, "winnerText": "Error", "winnerScore": 6.59375, "winnerGap": 2.876953125, "top": [ { "rank": 1, "tokenId": 3494, "text": "Error", "score": 6.59375 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 3.716796875 }, { "rank": 3, "tokenId": 3771, "text": "No", "score": 3.662109375 }, { "rank": 4, "tokenId": 1899, "text": "error", "score": 3.552734375 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "score": 3.21875 }, { "rank": 6, "tokenId": 6302, "text": "Start", "score": 2.998046875 }, { "rank": 7, "tokenId": 4661, "text": "Input", "score": 2.080078125 }, { "rank": 8, "tokenId": 2084, "text": "type", "score": 2.015625 } ] } }, "winnerTextsByMode": { "exact": "Error", "f32_forward": "Error", "f32_reverse": "Error", "f32_tree64": "Error", "f16_forward": "Error" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Error" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "json-nested-oneword-c", "text": "No explanation. One word only.\n{\"data\":{\"items\":[{\"id\":", "stableTopK": [ { "rank": 1, "tokenId": 3494, "text": "Error", "logit": 12.449732780456543, "prob": 0.7101415991783142 }, { "rank": 2, "tokenId": 1899, "text": "error", "logit": 10.000492095947266, "prob": 0.061327215284109116 }, { "rank": 3, "tokenId": 3771, "text": "No", "logit": 9.919804573059082, "prob": 0.05657324567437172 }, { "rank": 4, "tokenId": 6455, "text": "null", "logit": 9.64535903930664, "prob": 0.04299529641866684 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "logit": 8.576943397521973, "prob": 0.014771136455237865 }, { "rank": 6, "tokenId": 236797, "text": "N", "logit": 8.102123260498047, "prob": 0.009187601506710052 }, { "rank": 7, "tokenId": 20470, "text": "Wait", "logit": 8.035477638244629, "prob": 0.008595245890319347 }, { "rank": 8, "tokenId": 9336, "text": "None", "logit": 7.747837066650391, "prob": 0.006446701940149069 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3494, "winnerText": "Error", "winnerScore": 13.248769969460284, "winnerGap": 2.851007772689343, "top": [ { "rank": 1, "tokenId": 3494, "text": "Error", "score": 13.248769969460284 }, { "rank": 2, "tokenId": 1899, "text": "error", "score": 10.397762196770941 }, { "rank": 3, "tokenId": 3771, "text": "No", "score": 10.30707665181977 }, { "rank": 4, "tokenId": 6455, "text": "null", "score": 9.999975452545295 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "score": 8.8228021138931 }, { "rank": 6, "tokenId": 236797, "text": "N", "score": 8.308205487440539 }, { "rank": 7, "tokenId": 20470, "text": "Wait", "score": 8.236363566926357 }, { "rank": 8, "tokenId": 9336, "text": "None", "score": 7.927331970936052 } ] }, "f32_forward": { "winnerTokenId": 3494, "winnerText": "Error", "winnerScore": 13.248767852783203, "winnerGap": 2.8510141372680664, "top": [ { "rank": 1, "tokenId": 3494, "text": "Error", "score": 13.248767852783203 }, { "rank": 2, "tokenId": 1899, "text": "error", "score": 10.397753715515137 }, { "rank": 3, "tokenId": 3771, "text": "No", "score": 10.307076454162598 }, { "rank": 4, "tokenId": 6455, "text": "null", "score": 9.999975204467773 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "score": 8.822807312011719 }, { "rank": 6, "tokenId": 236797, "text": "N", "score": 8.308209419250488 }, { "rank": 7, "tokenId": 20470, "text": "Wait", "score": 8.236367225646973 }, { "rank": 8, "tokenId": 9336, "text": "None", "score": 7.927325248718262 } ] }, "f32_reverse": { "winnerTokenId": 3494, "winnerText": "Error", "winnerScore": 13.248770713806152, "winnerGap": 2.8510141372680664, "top": [ { "rank": 1, "tokenId": 3494, "text": "Error", "score": 13.248770713806152 }, { "rank": 2, "tokenId": 1899, "text": "error", "score": 10.397756576538086 }, { "rank": 3, "tokenId": 3771, "text": "No", "score": 10.307083129882812 }, { "rank": 4, "tokenId": 6455, "text": "null", "score": 9.999982833862305 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "score": 8.822802543640137 }, { "rank": 6, "tokenId": 236797, "text": "N", "score": 8.308209419250488 }, { "rank": 7, "tokenId": 20470, "text": "Wait", "score": 8.236359596252441 }, { "rank": 8, "tokenId": 9336, "text": "None", "score": 7.92733097076416 } ] }, "f32_tree64": { "winnerTokenId": 3494, "winnerText": "Error", "winnerScore": 13.248770713806152, "winnerGap": 2.851008415222168, "top": [ { "rank": 1, "tokenId": 3494, "text": "Error", "score": 13.248770713806152 }, { "rank": 2, "tokenId": 1899, "text": "error", "score": 10.397762298583984 }, { "rank": 3, "tokenId": 3771, "text": "No", "score": 10.307076454162598 }, { "rank": 4, "tokenId": 6455, "text": "null", "score": 9.999975204467773 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "score": 8.82280158996582 }, { "rank": 6, "tokenId": 236797, "text": "N", "score": 8.308205604553223 }, { "rank": 7, "tokenId": 20470, "text": "Wait", "score": 8.236363410949707 }, { "rank": 8, "tokenId": 9336, "text": "None", "score": 7.927331924438477 } ] }, "f16_forward": { "winnerTokenId": 3494, "winnerText": "Error", "winnerScore": 13.203125, "winnerGap": 2.671875, "top": [ { "rank": 1, "tokenId": 3494, "text": "Error", "score": 13.203125 }, { "rank": 2, "tokenId": 1899, "text": "error", "score": 10.53125 }, { "rank": 3, "tokenId": 3771, "text": "No", "score": 10.3203125 }, { "rank": 4, "tokenId": 6455, "text": "null", "score": 10.109375 }, { "rank": 5, "tokenId": 236797, "text": "N", "score": 8.296875 }, { "rank": 6, "tokenId": 6302, "text": "Start", "score": 7.66796875 }, { "rank": 7, "tokenId": 7676, "text": "OK", "score": 7.6015625 }, { "rank": 8, "tokenId": 7633, "text": "items", "score": 7.5625 } ] } }, "winnerTextsByMode": { "exact": "Error", "f32_forward": "Error", "f32_reverse": "Error", "f32_tree64": "Error", "f16_forward": "Error" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Error" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "json-open-oneword-d", "text": "Choose one word only.\n{\"answer\":", "stableTopK": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "logit": 6.921656608581543, "prob": 0.8325037956237793 }, { "rank": 2, "tokenId": 35724, "text": "choose", "logit": 4.403509140014648, "prob": 0.06710713356733322 }, { "rank": 3, "tokenId": 17005, "text": "choice", "logit": 4.201314926147461, "prob": 0.054822247475385666 }, { "rank": 4, "tokenId": 115863, "text": "Analyze", "logit": 2.821826457977295, "prob": 0.013799159787595272 }, { "rank": 5, "tokenId": 811, "text": "one", "logit": 1.9379960298538208, "prob": 0.00570177286863327 }, { "rank": 6, "tokenId": 20505, "text": "Choice", "logit": 1.591891884803772, "prob": 0.004033655859529972 }, { "rank": 7, "tokenId": 14979, "text": "Word", "logit": 1.236749291419983, "prob": 0.002827888820320368 }, { "rank": 8, "tokenId": 18999, "text": "single", "logit": 1.0976524353027344, "prob": 0.00246066995896399 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 24852, "winnerText": "Choose", "winnerScore": 7.048552936666805, "winnerGap": 2.6130008734038768, "top": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "score": 7.048552936666805 }, { "rank": 2, "tokenId": 35724, "text": "choose", "score": 4.435552063262929 }, { "rank": 3, "tokenId": 17005, "text": "choice", "score": 4.229104474981174 }, { "rank": 4, "tokenId": 115863, "text": "Analyze", "score": 2.830191813517583 }, { "rank": 5, "tokenId": 811, "text": "one", "score": 1.9406987249923677 }, { "rank": 6, "tokenId": 20505, "text": "Choice", "score": 1.5933870385827298 }, { "rank": 7, "tokenId": 14979, "text": "Word", "score": 1.2374499781014894 }, { "rank": 8, "tokenId": 18999, "text": "single", "score": 1.098141408117475 } ] }, "f32_forward": { "winnerTokenId": 24852, "winnerText": "Choose", "winnerScore": 7.04855489730835, "winnerGap": 2.613004207611084, "top": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "score": 7.04855489730835 }, { "rank": 2, "tokenId": 35724, "text": "choose", "score": 4.435550689697266 }, { "rank": 3, "tokenId": 17005, "text": "choice", "score": 4.229104042053223 }, { "rank": 4, "tokenId": 115863, "text": "Analyze", "score": 2.8301870822906494 }, { "rank": 5, "tokenId": 811, "text": "one", "score": 1.9407026767730713 }, { "rank": 6, "tokenId": 20505, "text": "Choice", "score": 1.5933887958526611 }, { "rank": 7, "tokenId": 14979, "text": "Word", "score": 1.2374470233917236 }, { "rank": 8, "tokenId": 18999, "text": "single", "score": 1.0981435775756836 } ] }, "f32_reverse": { "winnerTokenId": 24852, "winnerText": "Choose", "winnerScore": 7.048551559448242, "winnerGap": 2.613001823425293, "top": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "score": 7.048551559448242 }, { "rank": 2, "tokenId": 35724, "text": "choose", "score": 4.435549736022949 }, { "rank": 3, "tokenId": 17005, "text": "choice", "score": 4.229109287261963 }, { "rank": 4, "tokenId": 115863, "text": "Analyze", "score": 2.830195903778076 }, { "rank": 5, "tokenId": 811, "text": "one", "score": 1.9406965970993042 }, { "rank": 6, "tokenId": 20505, "text": "Choice", "score": 1.5933891534805298 }, { "rank": 7, "tokenId": 14979, "text": "Word", "score": 1.2374552488327026 }, { "rank": 8, "tokenId": 18999, "text": "single", "score": 1.0981396436691284 } ] }, "f32_tree64": { "winnerTokenId": 24852, "winnerText": "Choose", "winnerScore": 7.048553466796875, "winnerGap": 2.6130032539367676, "top": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "score": 7.048553466796875 }, { "rank": 2, "tokenId": 35724, "text": "choose", "score": 4.435550212860107 }, { "rank": 3, "tokenId": 17005, "text": "choice", "score": 4.229104518890381 }, { "rank": 4, "tokenId": 115863, "text": "Analyze", "score": 2.8301920890808105 }, { "rank": 5, "tokenId": 811, "text": "one", "score": 1.9406981468200684 }, { "rank": 6, "tokenId": 20505, "text": "Choice", "score": 1.5933871269226074 }, { "rank": 7, "tokenId": 14979, "text": "Word", "score": 1.2374491691589355 }, { "rank": 8, "tokenId": 18999, "text": "single", "score": 1.0981404781341553 } ] }, "f16_forward": { "winnerTokenId": 24852, "winnerText": "Choose", "winnerScore": 8.0234375, "winnerGap": 3.55859375, "top": [ { "rank": 1, "tokenId": 24852, "text": "Choose", "score": 8.0234375 }, { "rank": 2, "tokenId": 35724, "text": "choose", "score": 4.46484375 }, { "rank": 3, "tokenId": 17005, "text": "choice", "score": 3.255859375 }, { "rank": 4, "tokenId": 115863, "text": "Analyze", "score": 2.8828125 }, { "rank": 5, "tokenId": 811, "text": "one", "score": 1.984375 }, { "rank": 6, "tokenId": 20505, "text": "Choice", "score": 1.634765625 }, { "rank": 7, "tokenId": 14979, "text": "Word", "score": 1.2314453125 }, { "rank": 8, "tokenId": 18999, "text": "single", "score": 1.11328125 } ] } }, "winnerTextsByMode": { "exact": "Choose", "f32_forward": "Choose", "f32_reverse": "Choose", "f32_tree64": "Choose", "f16_forward": "Choose" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Choose" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "justice-revenge-both-oneword-a", "text": "Answer with exactly one word only.\nJustice, revenge, or both: wanting a murderer imprisoned for life is", "stableTopK": [ { "rank": 1, "tokenId": 173794, "text": "Pun", "logit": 3.4482979774475098, "prob": 0.39388370513916016 }, { "rank": 2, "tokenId": 7949, "text": "Ret", "logit": 2.4166922569274902, "prob": 0.14039361476898193 }, { "rank": 3, "tokenId": 236791, "text": "P", "logit": 1.9308561086654663, "prob": 0.08636770397424698 }, { "rank": 4, "tokenId": 72199, "text": "Justice", "logit": 1.6245020627975464, "prob": 0.06357751041650772 }, { "rank": 5, "tokenId": 15126, "text": "pun", "logit": 1.3312562704086304, "prob": 0.04741857200860977 }, { "rank": 6, "tokenId": 4243, "text": "ret", "logit": 1.1043816804885864, "prob": 0.037793584167957306 }, { "rank": 7, "tokenId": 71869, "text": "Cru", "logit": 0.7448846697807312, "prob": 0.026380956172943115 }, { "rank": 8, "tokenId": 64753, "text": "Capital", "logit": 0.7209802865982056, "prob": 0.025757813826203346 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 173794, "winnerText": "Pun", "winnerScore": 3.463602456163498, "winnerGap": 1.0416602673553061, "top": [ { "rank": 1, "tokenId": 173794, "text": "Pun", "score": 3.463602456163498 }, { "rank": 2, "tokenId": 7949, "text": "Ret", "score": 2.421942188808192 }, { "rank": 3, "tokenId": 236791, "text": "P", "score": 1.9335290455397072 }, { "rank": 4, "tokenId": 72199, "text": "Justice", "score": 1.6260918634968142 }, { "rank": 5, "tokenId": 15126, "text": "pun", "score": 1.332126397181037 }, { "rank": 6, "tokenId": 4243, "text": "ret", "score": 1.1048820280812208 }, { "rank": 7, "tokenId": 71869, "text": "Cru", "score": 0.7450391151898046 }, { "rank": 8, "tokenId": 64753, "text": "Capital", "score": 0.7211167129482696 } ] }, "f32_forward": { "winnerTokenId": 173794, "winnerText": "Pun", "winnerScore": 3.4636056423187256, "winnerGap": 1.0416648387908936, "top": [ { "rank": 1, "tokenId": 173794, "text": "Pun", "score": 3.4636056423187256 }, { "rank": 2, "tokenId": 7949, "text": "Ret", "score": 2.421940803527832 }, { "rank": 3, "tokenId": 236791, "text": "P", "score": 1.9335289001464844 }, { "rank": 4, "tokenId": 72199, "text": "Justice", "score": 1.6261006593704224 }, { "rank": 5, "tokenId": 15126, "text": "pun", "score": 1.3321255445480347 }, { "rank": 6, "tokenId": 4243, "text": "ret", "score": 1.104884386062622 }, { "rank": 7, "tokenId": 71869, "text": "Cru", "score": 0.7450464367866516 }, { "rank": 8, "tokenId": 64753, "text": "Capital", "score": 0.7211138010025024 } ] }, "f32_reverse": { "winnerTokenId": 173794, "winnerText": "Pun", "winnerScore": 3.4636037349700928, "winnerGap": 1.0416641235351562, "top": [ { "rank": 1, "tokenId": 173794, "text": "Pun", "score": 3.4636037349700928 }, { "rank": 2, "tokenId": 7949, "text": "Ret", "score": 2.4219396114349365 }, { "rank": 3, "tokenId": 236791, "text": "P", "score": 1.933525562286377 }, { "rank": 4, "tokenId": 72199, "text": "Justice", "score": 1.626092791557312 }, { "rank": 5, "tokenId": 15126, "text": "pun", "score": 1.3321218490600586 }, { "rank": 6, "tokenId": 4243, "text": "ret", "score": 1.1048812866210938 }, { "rank": 7, "tokenId": 71869, "text": "Cru", "score": 0.7450438141822815 }, { "rank": 8, "tokenId": 64753, "text": "Capital", "score": 0.7211152911186218 } ] }, "f32_tree64": { "winnerTokenId": 173794, "winnerText": "Pun", "winnerScore": 3.4636037349700928, "winnerGap": 1.0416607856750488, "top": [ { "rank": 1, "tokenId": 173794, "text": "Pun", "score": 3.4636037349700928 }, { "rank": 2, "tokenId": 7949, "text": "Ret", "score": 2.421942949295044 }, { "rank": 3, "tokenId": 236791, "text": "P", "score": 1.9335298538208008 }, { "rank": 4, "tokenId": 72199, "text": "Justice", "score": 1.6260912418365479 }, { "rank": 5, "tokenId": 15126, "text": "pun", "score": 1.3321263790130615 }, { "rank": 6, "tokenId": 4243, "text": "ret", "score": 1.1048824787139893 }, { "rank": 7, "tokenId": 71869, "text": "Cru", "score": 0.7450383305549622 }, { "rank": 8, "tokenId": 64753, "text": "Capital", "score": 0.7211155891418457 } ] }, "f16_forward": { "winnerTokenId": 173794, "winnerText": "Pun", "winnerScore": 3.544921875, "winnerGap": 1.03515625, "top": [ { "rank": 1, "tokenId": 173794, "text": "Pun", "score": 3.544921875 }, { "rank": 2, "tokenId": 7949, "text": "Ret", "score": 2.509765625 }, { "rank": 3, "tokenId": 236791, "text": "P", "score": 1.923828125 }, { "rank": 4, "tokenId": 15126, "text": "pun", "score": 1.3330078125 }, { "rank": 5, "tokenId": 4243, "text": "ret", "score": 1.10546875 }, { "rank": 6, "tokenId": 64753, "text": "Capital", "score": 0.74755859375 }, { "rank": 7, "tokenId": 105637, "text": "justice", "score": 0.2333984375 }, { "rank": 8, "tokenId": 236847, "text": "V", "score": 0.1474609375 } ] } }, "winnerTextsByMode": { "exact": "Pun", "f32_forward": "Pun", "f32_reverse": "Pun", "f32_tree64": "Pun", "f16_forward": "Pun" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Pun" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "justice-revenge-explicit-oneword-b", "text": "Respond with one word only.\nAnswer with exactly one word: justice or revenge. Question: Wanting a murderer imprisoned for life is Answer:", "stableTopK": [ { "rank": 1, "tokenId": 105637, "text": "justice", "logit": 9.183021545410156, "prob": 0.9999111294746399 }, { "rank": 2, "tokenId": 72199, "text": "Justice", "logit": -0.16923153400421143, "prob": 0.00008676198922330514 }, { "rank": 3, "tokenId": 4017, "text": "just", "logit": -5.186889171600342, "prob": 5.743656856793677e-7 }, { "rank": 4, "tokenId": 89548, "text": "reven", "logit": -5.760668754577637, "prob": 3.2359304213969153e-7 }, { "rank": 5, "tokenId": 11933, "text": " justice", "logit": -6.699991703033447, "prob": 1.2649006464471313e-7 }, { "rank": 6, "tokenId": 92768, "text": "JUST", "logit": -7.0772929191589355, "prob": 8.673543305803832e-8 }, { "rank": 7, "tokenId": 1479, "text": "Re", "logit": -7.774434566497803, "prob": 4.319483437598137e-8 }, { "rank": 8, "tokenId": 1042, "text": "mer", "logit": -7.818354606628418, "prob": 4.133877240519723e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 105637, "winnerText": "justice", "winnerScore": 9.487113434070366, "winnerGap": 9.656349891397308, "top": [ { "rank": 1, "tokenId": 105637, "text": "justice", "score": 9.487113434070366 }, { "rank": 2, "tokenId": 72199, "text": "Justice", "score": -0.16923645732694226 }, { "rank": 3, "tokenId": 4017, "text": "just", "score": -5.239517019396647 }, { "rank": 4, "tokenId": 89548, "text": "reven", "score": -5.833085764040193 }, { "rank": 5, "tokenId": 11933, "text": " justice", "score": -6.814839215584875 }, { "rank": 6, "tokenId": 92768, "text": "JUST", "score": -7.213152156617099 }, { "rank": 7, "tokenId": 1479, "text": "Re", "score": -7.955843442372254 }, { "rank": 8, "tokenId": 1042, "text": "mer", "score": -8.002941424534965 } ] }, "f32_forward": { "winnerTokenId": 105637, "winnerText": "justice", "winnerScore": 9.487117767333984, "winnerGap": 9.656354114413261, "top": [ { "rank": 1, "tokenId": 105637, "text": "justice", "score": 9.487117767333984 }, { "rank": 2, "tokenId": 72199, "text": "Justice", "score": -0.16923634707927704 }, { "rank": 3, "tokenId": 4017, "text": "just", "score": -5.239518642425537 }, { "rank": 4, "tokenId": 89548, "text": "reven", "score": -5.833076000213623 }, { "rank": 5, "tokenId": 11933, "text": " justice", "score": -6.814842700958252 }, { "rank": 6, "tokenId": 92768, "text": "JUST", "score": -7.213161945343018 }, { "rank": 7, "tokenId": 1479, "text": "Re", "score": -7.9558424949646 }, { "rank": 8, "tokenId": 1042, "text": "mer", "score": -8.00294017791748 } ] }, "f32_reverse": { "winnerTokenId": 105637, "winnerText": "justice", "winnerScore": 9.487115859985352, "winnerGap": 9.65635161101818, "top": [ { "rank": 1, "tokenId": 105637, "text": "justice", "score": 9.487115859985352 }, { "rank": 2, "tokenId": 72199, "text": "Justice", "score": -0.16923575103282928 }, { "rank": 3, "tokenId": 4017, "text": "just", "score": -5.239512920379639 }, { "rank": 4, "tokenId": 89548, "text": "reven", "score": -5.833086013793945 }, { "rank": 5, "tokenId": 11933, "text": " justice", "score": -6.81483268737793 }, { "rank": 6, "tokenId": 92768, "text": "JUST", "score": -7.2131428718566895 }, { "rank": 7, "tokenId": 1479, "text": "Re", "score": -7.955845355987549 }, { "rank": 8, "tokenId": 1042, "text": "mer", "score": -8.002941131591797 } ] }, "f32_tree64": { "winnerTokenId": 105637, "winnerText": "justice", "winnerScore": 9.487112998962402, "winnerGap": 9.65635061264038, "top": [ { "rank": 1, "tokenId": 105637, "text": "justice", "score": 9.487112998962402 }, { "rank": 2, "tokenId": 72199, "text": "Justice", "score": -0.16923761367797852 }, { "rank": 3, "tokenId": 4017, "text": "just", "score": -5.23951530456543 }, { "rank": 4, "tokenId": 89548, "text": "reven", "score": -5.833086013793945 }, { "rank": 5, "tokenId": 11933, "text": " justice", "score": -6.814841270446777 }, { "rank": 6, "tokenId": 92768, "text": "JUST", "score": -7.2131500244140625 }, { "rank": 7, "tokenId": 1479, "text": "Re", "score": -7.955843925476074 }, { "rank": 8, "tokenId": 1042, "text": "mer", "score": -8.002942085266113 } ] }, "f16_forward": { "winnerTokenId": 105637, "winnerText": "justice", "winnerScore": 9.40625, "winnerGap": 9.6177978515625, "top": [ { "rank": 1, "tokenId": 105637, "text": "justice", "score": 9.40625 }, { "rank": 2, "tokenId": 72199, "text": "Justice", "score": -0.2115478515625 }, { "rank": 3, "tokenId": 236847, "text": "V", "score": -4.41796875 }, { "rank": 4, "tokenId": 4017, "text": "just", "score": -5.296875 }, { "rank": 5, "tokenId": 14389, "text": "wr", "score": -5.44921875 }, { "rank": 6, "tokenId": 30558, "text": "positive", "score": -5.76171875 }, { "rank": 7, "tokenId": 62315, "text": "peace", "score": -5.765625 }, { "rank": 8, "tokenId": 4506, "text": "rest", "score": -5.76953125 } ] } }, "winnerTextsByMode": { "exact": "justice", "f32_forward": "justice", "f32_reverse": "justice", "f32_tree64": "justice", "f16_forward": "justice" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "justice" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "kitten-animal-oneword-c", "text": "No explanation. One word only.\nA kitten grows into a", "stableTopK": [ { "rank": 1, "tokenId": 25429, "text": "Cat", "logit": 0.1520630270242691, "prob": 0.9977813959121704 }, { "rank": 2, "tokenId": 84136, "text": "Adult", "logit": -6.10745906829834, "prob": 0.001907917088828981 }, { "rank": 3, "tokenId": 9307, "text": "cat", "logit": -8.165096282958984, "prob": 0.0002437468501739204 }, { "rank": 4, "tokenId": 82625, "text": "adult", "logit": -10.000879287719727, "prob": 0.00003887483762810007 }, { "rank": 5, "tokenId": 37515, "text": "Dog", "logit": -10.639568328857422, "prob": 0.000020525298168649897 }, { "rank": 6, "tokenId": 87607, "text": "Lion", "logit": -11.836073875427246, "prob": 0.000006203741577337496 }, { "rank": 7, "tokenId": 216597, "text": "Elephant", "logit": -14.924378395080566, "prob": 2.8276139119043364e-7 }, { "rank": 8, "tokenId": 36196, "text": "CAT", "logit": -15.250201225280762, "prob": 2.0413479262515466e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 25429, "winnerText": "Cat", "winnerScore": 0.15205878552998797, "winnerGap": 6.346054414599467, "top": [ { "rank": 1, "tokenId": 25429, "text": "Cat", "score": 0.15205878552998797 }, { "rank": 2, "tokenId": 84136, "text": "Adult", "score": -6.193995629069479 }, { "rank": 3, "tokenId": 9307, "text": "cat", "score": -8.376175566832003 }, { "rank": 4, "tokenId": 82625, "text": "adult", "score": -10.398200380591526 }, { "rank": 5, "tokenId": 37515, "text": "Dog", "score": -11.122663506538991 }, { "rank": 6, "tokenId": 87607, "text": "Lion", "score": -12.514821571084752 }, { "rank": 7, "tokenId": 216597, "text": "Elephant", "score": -16.378526578364898 }, { "rank": 8, "tokenId": 36196, "text": "CAT", "score": -16.81467143152399 } ] }, "f32_forward": { "winnerTokenId": 25429, "winnerText": "Cat", "winnerScore": 0.15207518637180328, "winnerGap": 6.346081152558327, "top": [ { "rank": 1, "tokenId": 25429, "text": "Cat", "score": 0.15207518637180328 }, { "rank": 2, "tokenId": 84136, "text": "Adult", "score": -6.194005966186523 }, { "rank": 3, "tokenId": 9307, "text": "cat", "score": -8.376169204711914 }, { "rank": 4, "tokenId": 82625, "text": "adult", "score": -10.39819622039795 }, { "rank": 5, "tokenId": 37515, "text": "Dog", "score": -11.122665405273438 }, { "rank": 6, "tokenId": 87607, "text": "Lion", "score": -12.514801979064941 }, { "rank": 7, "tokenId": 216597, "text": "Elephant", "score": -16.37852668762207 }, { "rank": 8, "tokenId": 36196, "text": "CAT", "score": -16.814664840698242 } ] }, "f32_reverse": { "winnerTokenId": 25429, "winnerText": "Cat", "winnerScore": 0.15207122266292572, "winnerGap": 6.3460733741521835, "top": [ { "rank": 1, "tokenId": 25429, "text": "Cat", "score": 0.15207122266292572 }, { "rank": 2, "tokenId": 84136, "text": "Adult", "score": -6.194002151489258 }, { "rank": 3, "tokenId": 9307, "text": "cat", "score": -8.376173973083496 }, { "rank": 4, "tokenId": 82625, "text": "adult", "score": -10.39820671081543 }, { "rank": 5, "tokenId": 37515, "text": "Dog", "score": -11.122674942016602 }, { "rank": 6, "tokenId": 87607, "text": "Lion", "score": -12.514822006225586 }, { "rank": 7, "tokenId": 216597, "text": "Elephant", "score": -16.37851905822754 }, { "rank": 8, "tokenId": 36196, "text": "CAT", "score": -16.814666748046875 } ] }, "f32_tree64": { "winnerTokenId": 25429, "winnerText": "Cat", "winnerScore": 0.15205955505371094, "winnerGap": 6.346055030822754, "top": [ { "rank": 1, "tokenId": 25429, "text": "Cat", "score": 0.15205955505371094 }, { "rank": 2, "tokenId": 84136, "text": "Adult", "score": -6.193995475769043 }, { "rank": 3, "tokenId": 9307, "text": "cat", "score": -8.376174926757812 }, { "rank": 4, "tokenId": 82625, "text": "adult", "score": -10.398200988769531 }, { "rank": 5, "tokenId": 37515, "text": "Dog", "score": -11.122665405273438 }, { "rank": 6, "tokenId": 87607, "text": "Lion", "score": -12.514822006225586 }, { "rank": 7, "tokenId": 216597, "text": "Elephant", "score": -16.378528594970703 }, { "rank": 8, "tokenId": 36196, "text": "CAT", "score": -16.814672470092773 } ] }, "f16_forward": { "winnerTokenId": 25429, "winnerText": "Cat", "winnerScore": -0.10693359375, "winnerGap": 4.26416015625, "top": [ { "rank": 1, "tokenId": 25429, "text": "Cat", "score": -0.10693359375 }, { "rank": 2, "tokenId": 9307, "text": "cat", "score": -4.37109375 }, { "rank": 3, "tokenId": 84136, "text": "Adult", "score": -6.17578125 }, { "rank": 4, "tokenId": 37515, "text": "Dog", "score": -11.2109375 }, { "rank": 5, "tokenId": 87607, "text": "Lion", "score": -12.4453125 }, { "rank": 6, "tokenId": 82625, "text": "adult", "score": -14.4140625 }, { "rank": 7, "tokenId": 216597, "text": "Elephant", "score": -16.625 }, { "rank": 8, "tokenId": 36196, "text": "CAT", "score": -16.71875 } ] } }, "winnerTextsByMode": { "exact": "Cat", "f32_forward": "Cat", "f32_reverse": "Cat", "f32_tree64": "Cat", "f16_forward": "Cat" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Cat" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "lang-arabic-greeting-oneword-d", "text": "Choose one word only.\nمرحبا، الجواب هو", "stableTopK": [ { "rank": 1, "tokenId": 236873, "text": "م", "logit": 16.05583381652832, "prob": 0.7746948003768921 }, { "rank": 2, "tokenId": 1483, "text": "ال", "logit": 14.736116409301758, "prob": 0.20700682699680328 }, { "rank": 3, "tokenId": 113151, "text": "جواب", "logit": 11.517236709594727, "prob": 0.008280239999294281 }, { "rank": 4, "tokenId": 237143, "text": "أ", "logit": 11.192549705505371, "prob": 0.005984573159366846 }, { "rank": 5, "tokenId": 1018, "text": "**", "logit": 9.244464874267578, "prob": 0.0008530818158760667 }, { "rank": 6, "tokenId": 25845, "text": "اج", "logit": 9.112665176391602, "prob": 0.0007477403269149363 }, { "rank": 7, "tokenId": 125378, "text": "الج", "logit": 8.469173431396484, "prob": 0.00039290348649956286 }, { "rank": 8, "tokenId": 236989, "text": "ح", "logit": 8.242532730102539, "prob": 0.00031322549330070615 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 236873, "winnerText": "م", "winnerScore": 17.921896531142792, "winnerGap": 1.7925193880256245, "top": [ { "rank": 1, "tokenId": 236873, "text": "م", "score": 17.921896531142792 }, { "rank": 2, "tokenId": 1483, "text": "ال", "score": 16.129377143117168 }, { "rank": 3, "tokenId": 113151, "text": "جواب", "score": 12.139057868573957 }, { "rank": 4, "tokenId": 237143, "text": "أ", "score": 11.760066982053075 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 9.554959313166556 }, { "rank": 6, "tokenId": 25845, "text": "اج", "score": 9.409551739151503 }, { "rank": 7, "tokenId": 125378, "text": "الج", "score": 8.705569320306736 }, { "rank": 8, "tokenId": 236989, "text": "ح", "score": 8.459869588551754 } ] }, "f32_forward": { "winnerTokenId": 236873, "winnerText": "م", "winnerScore": 17.921897888183594, "winnerGap": 1.7925167083740234, "top": [ { "rank": 1, "tokenId": 236873, "text": "م", "score": 17.921897888183594 }, { "rank": 2, "tokenId": 1483, "text": "ال", "score": 16.12938117980957 }, { "rank": 3, "tokenId": 113151, "text": "جواب", "score": 12.13906192779541 }, { "rank": 4, "tokenId": 237143, "text": "أ", "score": 11.760065078735352 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 9.554977416992188 }, { "rank": 6, "tokenId": 25845, "text": "اج", "score": 9.409555435180664 }, { "rank": 7, "tokenId": 125378, "text": "الج", "score": 8.70556354522705 }, { "rank": 8, "tokenId": 236989, "text": "ح", "score": 8.459869384765625 } ] }, "f32_reverse": { "winnerTokenId": 236873, "winnerText": "م", "winnerScore": 17.921903610229492, "winnerGap": 1.7925243377685547, "top": [ { "rank": 1, "tokenId": 236873, "text": "م", "score": 17.921903610229492 }, { "rank": 2, "tokenId": 1483, "text": "ال", "score": 16.129379272460938 }, { "rank": 3, "tokenId": 113151, "text": "جواب", "score": 12.139052391052246 }, { "rank": 4, "tokenId": 237143, "text": "أ", "score": 11.760066986083984 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 9.554956436157227 }, { "rank": 6, "tokenId": 25845, "text": "اج", "score": 9.409554481506348 }, { "rank": 7, "tokenId": 125378, "text": "الج", "score": 8.705567359924316 }, { "rank": 8, "tokenId": 236989, "text": "ح", "score": 8.45986557006836 } ] }, "f32_tree64": { "winnerTokenId": 236873, "winnerText": "م", "winnerScore": 17.921894073486328, "winnerGap": 1.7925167083740234, "top": [ { "rank": 1, "tokenId": 236873, "text": "م", "score": 17.921894073486328 }, { "rank": 2, "tokenId": 1483, "text": "ال", "score": 16.129377365112305 }, { "rank": 3, "tokenId": 113151, "text": "جواب", "score": 12.139059066772461 }, { "rank": 4, "tokenId": 237143, "text": "أ", "score": 11.760066986083984 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 9.554959297180176 }, { "rank": 6, "tokenId": 25845, "text": "اج", "score": 9.409552574157715 }, { "rank": 7, "tokenId": 125378, "text": "الج", "score": 8.705570220947266 }, { "rank": 8, "tokenId": 236989, "text": "ح", "score": 8.459869384765625 } ] }, "f16_forward": { "winnerTokenId": 1483, "winnerText": "ال", "winnerScore": 16.125, "winnerGap": 2.2421875, "top": [ { "rank": 1, "tokenId": 1483, "text": "ال", "score": 16.125 }, { "rank": 2, "tokenId": 236873, "text": "م", "score": 13.8828125 }, { "rank": 3, "tokenId": 113151, "text": "جواب", "score": 12.3125 }, { "rank": 4, "tokenId": 237143, "text": "أ", "score": 11.6875 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 9.6171875 }, { "rank": 6, "tokenId": 159238, "text": "واحد", "score": 8.1796875 }, { "rank": 7, "tokenId": 235099, "text": "Jaw", "score": 7.8515625 }, { "rank": 8, "tokenId": 125378, "text": "الج", "score": 7.703125 } ] } }, "winnerTextsByMode": { "exact": "م", "f32_forward": "م", "f32_reverse": "م", "f32_tree64": "م", "f16_forward": "ال" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "م", "ال" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 236873, 150345 ], "decodedText": "مرحبا", "steps": [ { "step": 0, "tokenId": 236873, "text": "م" }, { "step": 1, "tokenId": 150345, "text": "رحبا" } ] }, "f16_forward": { "tokenIds": [ 1483, 113151 ], "decodedText": "الجواب", "steps": [ { "step": 0, "tokenId": 1483, "text": "ال" }, { "step": 1, "tokenId": 113151, "text": "جواب" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "lang-chinese-simple-oneword-a", "text": "Answer with exactly one word only.\n对或错：地球是圆的。", "stableTopK": [ { "rank": 1, "tokenId": 237392, "text": "对", "logit": 2.829413890838623, "prob": 0.8940641283988953 }, { "rank": 2, "tokenId": 238836, "text": "错", "logit": 0.6327219009399414, "prob": 0.09939338266849518 }, { "rank": 3, "tokenId": 237783, "text": "真", "logit": -2.136245012283325, "prob": 0.0062346262857317924 }, { "rank": 4, "tokenId": 49324, "text": "正确", "logit": -5.3401079177856445, "prob": 0.00025315728271380067 }, { "rank": 5, "tokenId": 237631, "text": "正", "logit": -8.665889739990234, "prob": 0.00000909958816919243 }, { "rank": 6, "tokenId": 239126, "text": "假", "logit": -9.202333450317383, "prob": 0.000005321660864865407 }, { "rank": 7, "tokenId": 237026, "text": "是", "logit": -9.239140510559082, "prob": 0.000005129346845933469 }, { "rank": 8, "tokenId": 36525, "text": "错误", "logit": -9.340675354003906, "prob": 0.000004634106971934671 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 237392, "winnerText": "对", "winnerScore": 2.8378473437983187, "winnerGap": 2.205034385548852, "top": [ { "rank": 1, "tokenId": 237392, "text": "对", "score": 2.8378473437983187 }, { "rank": 2, "tokenId": 238836, "text": "错", "score": 0.6328129582494668 }, { "rank": 3, "tokenId": 237783, "text": "真", "score": -2.1398662747187567 }, { "rank": 4, "tokenId": 49324, "text": "正确", "score": -5.397607650076185 }, { "rank": 5, "tokenId": 237631, "text": "正", "score": -8.919755731733346 }, { "rank": 6, "tokenId": 239126, "text": "假", "score": -9.508430992452698 }, { "rank": 7, "tokenId": 237026, "text": "是", "score": -9.549084087092826 }, { "rank": 8, "tokenId": 36525, "text": "错误", "score": -9.661384446591828 } ] }, "f32_forward": { "winnerTokenId": 237392, "winnerText": "对", "winnerScore": 2.8378500938415527, "winnerGap": 2.205032527446747, "top": [ { "rank": 1, "tokenId": 237392, "text": "对", "score": 2.8378500938415527 }, { "rank": 2, "tokenId": 238836, "text": "错", "score": 0.6328175663948059 }, { "rank": 3, "tokenId": 237783, "text": "真", "score": -2.1398658752441406 }, { "rank": 4, "tokenId": 49324, "text": "正确", "score": -5.397603511810303 }, { "rank": 5, "tokenId": 237631, "text": "正", "score": -8.91976261138916 }, { "rank": 6, "tokenId": 239126, "text": "假", "score": -9.508435249328613 }, { "rank": 7, "tokenId": 237026, "text": "是", "score": -9.549078941345215 }, { "rank": 8, "tokenId": 36525, "text": "错误", "score": -9.661393165588379 } ] }, "f32_reverse": { "winnerTokenId": 237392, "winnerText": "对", "winnerScore": 2.8378498554229736, "winnerGap": 2.2050323486328125, "top": [ { "rank": 1, "tokenId": 237392, "text": "对", "score": 2.8378498554229736 }, { "rank": 2, "tokenId": 238836, "text": "错", "score": 0.6328175067901611 }, { "rank": 3, "tokenId": 237783, "text": "真", "score": -2.1398658752441406 }, { "rank": 4, "tokenId": 49324, "text": "正确", "score": -5.397604942321777 }, { "rank": 5, "tokenId": 237631, "text": "正", "score": -8.919755935668945 }, { "rank": 6, "tokenId": 239126, "text": "假", "score": -9.508430480957031 }, { "rank": 7, "tokenId": 237026, "text": "是", "score": -9.549101829528809 }, { "rank": 8, "tokenId": 36525, "text": "错误", "score": -9.661381721496582 } ] }, "f32_tree64": { "winnerTokenId": 237392, "winnerText": "对", "winnerScore": 2.8378474712371826, "winnerGap": 2.2050329446792603, "top": [ { "rank": 1, "tokenId": 237392, "text": "对", "score": 2.8378474712371826 }, { "rank": 2, "tokenId": 238836, "text": "错", "score": 0.6328145265579224 }, { "rank": 3, "tokenId": 237783, "text": "真", "score": -2.1398656368255615 }, { "rank": 4, "tokenId": 49324, "text": "正确", "score": -5.39760684967041 }, { "rank": 5, "tokenId": 237631, "text": "正", "score": -8.919755935668945 }, { "rank": 6, "tokenId": 239126, "text": "假", "score": -9.508431434631348 }, { "rank": 7, "tokenId": 237026, "text": "是", "score": -9.549083709716797 }, { "rank": 8, "tokenId": 36525, "text": "错误", "score": -9.661383628845215 } ] }, "f16_forward": { "winnerTokenId": 237392, "winnerText": "对", "winnerScore": 2.896484375, "winnerGap": 2.2490234375, "top": [ { "rank": 1, "tokenId": 237392, "text": "对", "score": 2.896484375 }, { "rank": 2, "tokenId": 238836, "text": "错", "score": 0.6474609375 }, { "rank": 3, "tokenId": 237783, "text": "真", "score": -2.1796875 }, { "rank": 4, "tokenId": 238430, "text": "确", "score": -3.4453125 }, { "rank": 5, "tokenId": 49324, "text": "正确", "score": -4.40234375 }, { "rank": 6, "tokenId": 238184, "text": "才", "score": -5.4921875 }, { "rank": 7, "tokenId": 237607, "text": "实", "score": -6.05078125 }, { "rank": 8, "tokenId": 237973, "text": "题", "score": -7.01171875 } ] } }, "winnerTextsByMode": { "exact": "对", "f32_forward": "对", "f32_reverse": "对", "f32_tree64": "对", "f16_forward": "对" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "对" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "lang-emoji-sentiment-oneword-b", "text": "Respond with one word only.\nRate this: 😊 — positive or negative?", "stableTopK": [ { "rank": 1, "tokenId": 20809, "text": "Neutral", "logit": 6.5876946449279785, "prob": 0.5756987929344177 }, { "rank": 2, "tokenId": 55144, "text": "Amb", "logit": 5.6058669090271, "prob": 0.2156715840101242 }, { "rank": 3, "tokenId": 63702, "text": "Negative", "logit": 4.452619552612305, "prob": 0.06806816160678864 }, { "rank": 4, "tokenId": 2805, "text": "Un", "logit": 4.241360664367676, "prob": 0.05510555952787399 }, { "rank": 5, "tokenId": 236797, "text": "N", "logit": 3.4440019130706787, "prob": 0.024826008826494217 }, { "rank": 6, "tokenId": 902, "text": "In", "logit": 2.450321674346924, "prob": 0.009190879762172699 }, { "rank": 7, "tokenId": 2704, "text": "Ind", "logit": 2.399524450302124, "prob": 0.008735667914152145 }, { "rank": 8, "tokenId": 58686, "text": "Cannot", "logit": 2.3496241569519043, "prob": 0.008310453034937382 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 20809, "winnerText": "Neutral", "winnerScore": 6.696747475304633, "winnerGap": 1.0242277437540208, "top": [ { "rank": 1, "tokenId": 20809, "text": "Neutral", "score": 6.696747475304633 }, { "rank": 2, "tokenId": 55144, "text": "Amb", "score": 5.672519731550612 }, { "rank": 3, "tokenId": 63702, "text": "Negative", "score": 4.485750257452594 }, { "rank": 4, "tokenId": 2805, "text": "Un", "score": 4.269962424193971 }, { "rank": 5, "tokenId": 236797, "text": "N", "score": 3.459251541142372 }, { "rank": 6, "tokenId": 902, "text": "In", "score": 2.455791811828055 }, { "rank": 7, "tokenId": 2704, "text": "Ind", "score": 2.4046608153201454 }, { "rank": 8, "tokenId": 58686, "text": "Cannot", "score": 2.354443365215456 } ] }, "f32_forward": { "winnerTokenId": 20809, "winnerText": "Neutral", "winnerScore": 6.69674015045166, "winnerGap": 1.024214744567871, "top": [ { "rank": 1, "tokenId": 20809, "text": "Neutral", "score": 6.69674015045166 }, { "rank": 2, "tokenId": 55144, "text": "Amb", "score": 5.672525405883789 }, { "rank": 3, "tokenId": 63702, "text": "Negative", "score": 4.485754489898682 }, { "rank": 4, "tokenId": 2805, "text": "Un", "score": 4.269954204559326 }, { "rank": 5, "tokenId": 236797, "text": "N", "score": 3.4592504501342773 }, { "rank": 6, "tokenId": 902, "text": "In", "score": 2.45579195022583 }, { "rank": 7, "tokenId": 2704, "text": "Ind", "score": 2.404658555984497 }, { "rank": 8, "tokenId": 58686, "text": "Cannot", "score": 2.354444980621338 } ] }, "f32_reverse": { "winnerTokenId": 20809, "winnerText": "Neutral", "winnerScore": 6.696746826171875, "winnerGap": 1.0242271423339844, "top": [ { "rank": 1, "tokenId": 20809, "text": "Neutral", "score": 6.696746826171875 }, { "rank": 2, "tokenId": 55144, "text": "Amb", "score": 5.672519683837891 }, { "rank": 3, "tokenId": 63702, "text": "Negative", "score": 4.485748291015625 }, { "rank": 4, "tokenId": 2805, "text": "Un", "score": 4.269961833953857 }, { "rank": 5, "tokenId": 236797, "text": "N", "score": 3.4592559337615967 }, { "rank": 6, "tokenId": 902, "text": "In", "score": 2.4557948112487793 }, { "rank": 7, "tokenId": 2704, "text": "Ind", "score": 2.404663324356079 }, { "rank": 8, "tokenId": 58686, "text": "Cannot", "score": 2.354443073272705 } ] }, "f32_tree64": { "winnerTokenId": 20809, "winnerText": "Neutral", "winnerScore": 6.696748733520508, "winnerGap": 1.0242295265197754, "top": [ { "rank": 1, "tokenId": 20809, "text": "Neutral", "score": 6.696748733520508 }, { "rank": 2, "tokenId": 55144, "text": "Amb", "score": 5.672519207000732 }, { "rank": 3, "tokenId": 63702, "text": "Negative", "score": 4.485750198364258 }, { "rank": 4, "tokenId": 2805, "text": "Un", "score": 4.269962787628174 }, { "rank": 5, "tokenId": 236797, "text": "N", "score": 3.4592514038085938 }, { "rank": 6, "tokenId": 902, "text": "In", "score": 2.455791473388672 }, { "rank": 7, "tokenId": 2704, "text": "Ind", "score": 2.404661178588867 }, { "rank": 8, "tokenId": 58686, "text": "Cannot", "score": 2.3544435501098633 } ] }, "f16_forward": { "winnerTokenId": 20809, "winnerText": "Neutral", "winnerScore": 6.62890625, "winnerGap": 0.89453125, "top": [ { "rank": 1, "tokenId": 20809, "text": "Neutral", "score": 6.62890625 }, { "rank": 2, "tokenId": 55144, "text": "Amb", "score": 5.734375 }, { "rank": 3, "tokenId": 63702, "text": "Negative", "score": 4.41796875 }, { "rank": 4, "tokenId": 2805, "text": "Un", "score": 4.2421875 }, { "rank": 5, "tokenId": 236797, "text": "N", "score": 3.4375 }, { "rank": 6, "tokenId": 41387, "text": "Unknown", "score": 3.32421875 }, { "rank": 7, "tokenId": 902, "text": "In", "score": 2.470703125 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "score": 2.44140625 } ] } }, "winnerTextsByMode": { "exact": "Neutral", "f32_forward": "Neutral", "f32_reverse": "Neutral", "f32_tree64": "Neutral", "f16_forward": "Neutral" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Neutral" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "lang-french-color-oneword-c", "text": "No explanation. One word only.\nLe ciel est", "stableTopK": [ { "rank": 1, "tokenId": 59796, "text": "Ble", "logit": 14.431000709533691, "prob": 0.9938902258872986 }, { "rank": 2, "tokenId": 1148, "text": "ble", "logit": 9.32410717010498, "prob": 0.006017868872731924 }, { "rank": 3, "tokenId": 19204, "text": "Variable", "logit": 3.8011231422424316, "prob": 0.000024034841771936044 }, { "rank": 4, "tokenId": 43868, "text": "Nu", "logit": 3.047137498855591, "prob": 0.000011308095054118894 }, { "rank": 5, "tokenId": 78134, "text": "Ou", "logit": 2.7374861240386963, "prob": 0.000008296779924421571 }, { "rank": 6, "tokenId": 169952, "text": "Azur", "logit": 2.542616367340088, "prob": 0.000006827767265349394 }, { "rank": 7, "tokenId": 236798, "text": "L", "logit": 2.5061404705047607, "prob": 0.000006583205959032057 }, { "rank": 8, "tokenId": 3508, "text": "Bl", "logit": 2.348942756652832, "prob": 0.000005625580342893954 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 59796, "winnerText": "Ble", "winnerScore": 15.729835408430077, "winnerGap": 6.086789501619936, "top": [ { "rank": 1, "tokenId": 59796, "text": "Ble", "score": 15.729835408430077 }, { "rank": 2, "tokenId": 1148, "text": "ble", "score": 9.64304590681014 }, { "rank": 3, "tokenId": 19204, "text": "Variable", "score": 3.82166291711075 }, { "rank": 4, "tokenId": 43868, "text": "Nu", "score": 3.057681577847992 }, { "rank": 5, "tokenId": 78134, "text": "Ou", "score": 2.745120816761432 }, { "rank": 6, "tokenId": 169952, "text": "Azur", "score": 2.548731235553735 }, { "rank": 7, "tokenId": 236798, "text": "L", "score": 2.511993672127659 }, { "rank": 8, "tokenId": 3508, "text": "Bl", "score": 2.3537598111872846 } ] }, "f32_forward": { "winnerTokenId": 59796, "winnerText": "Ble", "winnerScore": 15.729844093322754, "winnerGap": 6.086800575256348, "top": [ { "rank": 1, "tokenId": 59796, "text": "Ble", "score": 15.729844093322754 }, { "rank": 2, "tokenId": 1148, "text": "ble", "score": 9.643043518066406 }, { "rank": 3, "tokenId": 19204, "text": "Variable", "score": 3.8216586112976074 }, { "rank": 4, "tokenId": 43868, "text": "Nu", "score": 3.057682514190674 }, { "rank": 5, "tokenId": 78134, "text": "Ou", "score": 2.7451207637786865 }, { "rank": 6, "tokenId": 169952, "text": "Azur", "score": 2.54872727394104 }, { "rank": 7, "tokenId": 236798, "text": "L", "score": 2.511998414993286 }, { "rank": 8, "tokenId": 3508, "text": "Bl", "score": 2.353750228881836 } ] }, "f32_reverse": { "winnerTokenId": 59796, "winnerText": "Ble", "winnerScore": 15.729829788208008, "winnerGap": 6.086785316467285, "top": [ { "rank": 1, "tokenId": 59796, "text": "Ble", "score": 15.729829788208008 }, { "rank": 2, "tokenId": 1148, "text": "ble", "score": 9.643044471740723 }, { "rank": 3, "tokenId": 19204, "text": "Variable", "score": 3.8216633796691895 }, { "rank": 4, "tokenId": 43868, "text": "Nu", "score": 3.0576863288879395 }, { "rank": 5, "tokenId": 78134, "text": "Ou", "score": 2.7451226711273193 }, { "rank": 6, "tokenId": 169952, "text": "Azur", "score": 2.5487313270568848 }, { "rank": 7, "tokenId": 236798, "text": "L", "score": 2.511993169784546 }, { "rank": 8, "tokenId": 3508, "text": "Bl", "score": 2.3537583351135254 } ] }, "f32_tree64": { "winnerTokenId": 59796, "winnerText": "Ble", "winnerScore": 15.72983455657959, "winnerGap": 6.086789131164551, "top": [ { "rank": 1, "tokenId": 59796, "text": "Ble", "score": 15.72983455657959 }, { "rank": 2, "tokenId": 1148, "text": "ble", "score": 9.643045425415039 }, { "rank": 3, "tokenId": 19204, "text": "Variable", "score": 3.821661949157715 }, { "rank": 4, "tokenId": 43868, "text": "Nu", "score": 3.057680606842041 }, { "rank": 5, "tokenId": 78134, "text": "Ou", "score": 2.7451207637786865 }, { "rank": 6, "tokenId": 169952, "text": "Azur", "score": 2.5487303733825684 }, { "rank": 7, "tokenId": 236798, "text": "L", "score": 2.5119919776916504 }, { "rank": 8, "tokenId": 3508, "text": "Bl", "score": 2.353759527206421 } ] }, "f16_forward": { "winnerTokenId": 59796, "winnerText": "Ble", "winnerScore": 15.796875, "winnerGap": 6.0859375, "top": [ { "rank": 1, "tokenId": 59796, "text": "Ble", "score": 15.796875 }, { "rank": 2, "tokenId": 1148, "text": "ble", "score": 9.7109375 }, { "rank": 3, "tokenId": 19204, "text": "Variable", "score": 3.947265625 }, { "rank": 4, "tokenId": 236798, "text": "L", "score": 2.849609375 }, { "rank": 5, "tokenId": 169952, "text": "Azur", "score": 2.6015625 }, { "rank": 6, "tokenId": 3508, "text": "Bl", "score": 2.3203125 }, { "rank": 7, "tokenId": 78134, "text": "Ou", "score": 1.759765625 }, { "rank": 8, "tokenId": 27957, "text": "Inf", "score": 1.2841796875 } ] } }, "winnerTextsByMode": { "exact": "Ble", "f32_forward": "Ble", "f32_reverse": "Ble", "f32_tree64": "Ble", "f16_forward": "Ble" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Ble" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "lang-german-logic-oneword-d", "text": "Choose one word only.\nIst das richtig oder falsch?", "stableTopK": [ { "rank": 1, "tokenId": 236811, "text": "F", "logit": 15.554506301879883, "prob": 0.9857329726219177 }, { "rank": 2, "tokenId": 1018, "text": "**", "logit": 10.550742149353027, "prob": 0.006616862490773201 }, { "rank": 3, "tokenId": 236794, "text": "R", "logit": 9.27432918548584, "prob": 0.001846345723606646 }, { "rank": 4, "tokenId": 236760, "text": "f", "logit": 9.079171180725098, "prob": 0.0015189971309155226 }, { "rank": 5, "tokenId": 6855, "text": "Ne", "logit": 8.657135009765625, "prob": 0.000996022135950625 }, { "rank": 6, "tokenId": 13034, "text": "Die", "logit": 8.23022747039795, "prob": 0.0006499282317236066 }, { "rank": 7, "tokenId": 236824, "text": "W", "logit": 8.096529960632324, "prob": 0.0005685927462764084 }, { "rank": 8, "tokenId": 7168, "text": "Fe", "logit": 7.039182186126709, "prob": 0.00019751543004531413 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 236811, "winnerText": "F", "winnerScore": 17.227902749118336, "winnerGap": 6.206727303999344, "top": [ { "rank": 1, "tokenId": 236811, "text": "F", "score": 17.227902749118336 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 11.021175445118992 }, { "rank": 3, "tokenId": 236794, "text": "R", "score": 9.587972216260084 }, { "rank": 4, "tokenId": 236760, "text": "f", "score": 9.372665327928708 }, { "rank": 5, "tokenId": 6855, "text": "Ne", "score": 8.910204544698047 }, { "rank": 6, "tokenId": 13034, "text": "Die", "score": 8.446558322168784 }, { "rank": 7, "tokenId": 236824, "text": "W", "score": 8.302172455825172 }, { "rank": 8, "tokenId": 7168, "text": "Fe", "score": 7.1728063561479445 } ] }, "f32_forward": { "winnerTokenId": 236811, "winnerText": "F", "winnerScore": 17.227914810180664, "winnerGap": 6.206748962402344, "top": [ { "rank": 1, "tokenId": 236811, "text": "F", "score": 17.227914810180664 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 11.02116584777832 }, { "rank": 3, "tokenId": 236794, "text": "R", "score": 9.587966918945312 }, { "rank": 4, "tokenId": 236760, "text": "f", "score": 9.372661590576172 }, { "rank": 5, "tokenId": 6855, "text": "Ne", "score": 8.910200119018555 }, { "rank": 6, "tokenId": 13034, "text": "Die", "score": 8.446556091308594 }, { "rank": 7, "tokenId": 236824, "text": "W", "score": 8.302172660827637 }, { "rank": 8, "tokenId": 7168, "text": "Fe", "score": 7.172804832458496 } ] }, "f32_reverse": { "winnerTokenId": 236811, "winnerText": "F", "winnerScore": 17.227893829345703, "winnerGap": 6.206721305847168, "top": [ { "rank": 1, "tokenId": 236811, "text": "F", "score": 17.227893829345703 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 11.021172523498535 }, { "rank": 3, "tokenId": 236794, "text": "R", "score": 9.587965965270996 }, { "rank": 4, "tokenId": 236760, "text": "f", "score": 9.372668266296387 }, { "rank": 5, "tokenId": 6855, "text": "Ne", "score": 8.910199165344238 }, { "rank": 6, "tokenId": 13034, "text": "Die", "score": 8.446556091308594 }, { "rank": 7, "tokenId": 236824, "text": "W", "score": 8.302169799804688 }, { "rank": 8, "tokenId": 7168, "text": "Fe", "score": 7.172806739807129 } ] }, "f32_tree64": { "winnerTokenId": 236811, "winnerText": "F", "winnerScore": 17.227901458740234, "winnerGap": 6.206725120544434, "top": [ { "rank": 1, "tokenId": 236811, "text": "F", "score": 17.227901458740234 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 11.0211763381958 }, { "rank": 3, "tokenId": 236794, "text": "R", "score": 9.587971687316895 }, { "rank": 4, "tokenId": 236760, "text": "f", "score": 9.372665405273438 }, { "rank": 5, "tokenId": 6855, "text": "Ne", "score": 8.91020393371582 }, { "rank": 6, "tokenId": 13034, "text": "Die", "score": 8.44655704498291 }, { "rank": 7, "tokenId": 236824, "text": "W", "score": 8.30217170715332 }, { "rank": 8, "tokenId": 7168, "text": "Fe", "score": 7.172806739807129 } ] }, "f16_forward": { "winnerTokenId": 236811, "winnerText": "F", "winnerScore": 17.171875, "winnerGap": 7.765625, "top": [ { "rank": 1, "tokenId": 236811, "text": "F", "score": 17.171875 }, { "rank": 2, "tokenId": 236760, "text": "f", "score": 9.40625 }, { "rank": 3, "tokenId": 6855, "text": "Ne", "score": 8.8671875 }, { "rank": 4, "tokenId": 13034, "text": "Die", "score": 8.4375 }, { "rank": 5, "tokenId": 22943, "text": "Das", "score": 7.2421875 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 7.01953125 }, { "rank": 7, "tokenId": 90481, "text": "Ist", "score": 6.953125 }, { "rank": 8, "tokenId": 9277, "text": "False", "score": 6.59375 } ] } }, "winnerTextsByMode": { "exact": "F", "f32_forward": "F", "f32_reverse": "F", "f32_tree64": "F", "f16_forward": "F" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "F" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "lang-hindi-answer-oneword-a", "text": "Answer with exactly one word only.\nसही या गलत: पानी गीला है।", "stableTopK": [ { "rank": 1, "tokenId": 201488, "text": "सही", "logit": 12.19961166381836, "prob": 0.9995647072792053 }, { "rank": 2, "tokenId": 236831, "text": "स", "logit": 4.3019795417785645, "prob": 0.00037146065733395517 }, { "rank": 3, "tokenId": 97664, "text": "सच", "logit": 2.364941120147705, "prob": 0.00005353869346436113 }, { "rank": 4, "tokenId": 18517, "text": " सही", "logit": -0.16284973919391632, "prob": 0.000004274271759641124 }, { "rank": 5, "tokenId": 4339, "text": "True", "logit": -1.1222387552261353, "prob": 0.0000016375885252273292 }, { "rank": 6, "tokenId": 236751, "text": "s", "logit": -1.6701185703277588, "prob": 9.468116672906035e-7 }, { "rank": 7, "tokenId": 40904, "text": "सत", "logit": -2.0708773136138916, "prob": 6.341854827951465e-7 }, { "rank": 8, "tokenId": 236773, "text": "S", "logit": -2.497724771499634, "prob": 4.138460383273923e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 201488, "winnerText": "सही", "winnerScore": 12.947854399907694, "winnerGap": 8.616018021802994, "top": [ { "rank": 1, "tokenId": 201488, "text": "सही", "score": 12.947854399907694 }, { "rank": 2, "tokenId": 236831, "text": "स", "score": 4.331836378104701 }, { "rank": 3, "tokenId": 97664, "text": "सच", "score": 2.36985572177003 }, { "rank": 4, "tokenId": 18517, "text": " सही", "score": -0.1628503236790202 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -1.1227615510041602 }, { "rank": 6, "tokenId": 236751, "text": "s", "score": -1.6718508609851703 }, { "rank": 7, "tokenId": 40904, "text": "सत", "score": -2.0741763804135536 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": -2.50352231865757 } ] }, "f32_forward": { "winnerTokenId": 201488, "winnerText": "सही", "winnerScore": 12.947843551635742, "winnerGap": 8.61601209640503, "top": [ { "rank": 1, "tokenId": 201488, "text": "सही", "score": 12.947843551635742 }, { "rank": 2, "tokenId": 236831, "text": "स", "score": 4.331831455230713 }, { "rank": 3, "tokenId": 97664, "text": "सच", "score": 2.3698506355285645 }, { "rank": 4, "tokenId": 18517, "text": " सही", "score": -0.16285569965839386 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -1.122758150100708 }, { "rank": 6, "tokenId": 236751, "text": "s", "score": -1.6718519926071167 }, { "rank": 7, "tokenId": 40904, "text": "सत", "score": -2.074174404144287 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": -2.503523349761963 } ] }, "f32_reverse": { "winnerTokenId": 201488, "winnerText": "सही", "winnerScore": 12.94785213470459, "winnerGap": 8.616013526916504, "top": [ { "rank": 1, "tokenId": 201488, "text": "सही", "score": 12.94785213470459 }, { "rank": 2, "tokenId": 236831, "text": "स", "score": 4.331838607788086 }, { "rank": 3, "tokenId": 97664, "text": "सच", "score": 2.369856834411621 }, { "rank": 4, "tokenId": 18517, "text": " सही", "score": -0.1628551483154297 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -1.1227589845657349 }, { "rank": 6, "tokenId": 236751, "text": "s", "score": -1.671849250793457 }, { "rank": 7, "tokenId": 40904, "text": "सत", "score": -2.0741820335388184 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": -2.5035269260406494 } ] }, "f32_tree64": { "winnerTokenId": 201488, "winnerText": "सही", "winnerScore": 12.947854042053223, "winnerGap": 8.61601734161377, "top": [ { "rank": 1, "tokenId": 201488, "text": "सही", "score": 12.947854042053223 }, { "rank": 2, "tokenId": 236831, "text": "स", "score": 4.331836700439453 }, { "rank": 3, "tokenId": 97664, "text": "सच", "score": 2.369856357574463 }, { "rank": 4, "tokenId": 18517, "text": " सही", "score": -0.16284918785095215 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -1.1227607727050781 }, { "rank": 6, "tokenId": 236751, "text": "s", "score": -1.6718506813049316 }, { "rank": 7, "tokenId": 40904, "text": "सत", "score": -2.0741753578186035 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": -2.5035228729248047 } ] }, "f16_forward": { "winnerTokenId": 201488, "winnerText": "सही", "winnerScore": 12.96875, "winnerGap": 8.6640625, "top": [ { "rank": 1, "tokenId": 201488, "text": "सही", "score": 12.96875 }, { "rank": 2, "tokenId": 236831, "text": "स", "score": 4.3046875 }, { "rank": 3, "tokenId": 97664, "text": "सच", "score": 2.412109375 }, { "rank": 4, "tokenId": 18517, "text": " सही", "score": -0.17529296875 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -1.12890625 }, { "rank": 6, "tokenId": 236751, "text": "s", "score": -1.619140625 }, { "rank": 7, "tokenId": 236926, "text": "श", "score": -1.625 }, { "rank": 8, "tokenId": 40904, "text": "सत", "score": -2.05859375 } ] } }, "winnerTextsByMode": { "exact": "सही", "f32_forward": "सही", "f32_reverse": "सही", "f32_tree64": "सही", "f16_forward": "सही" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "सही" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "lang-italian-open-oneword-b", "text": "Respond with one word only.\nLa vita è", "stableTopK": [ { "rank": 1, "tokenId": 1558, "text": "Com", "logit": 9.903369903564453, "prob": 0.5736725330352783 }, { "rank": 2, "tokenId": 56331, "text": "compl", "logit": 9.374505996704102, "prob": 0.33805033564567566 }, { "rank": 3, "tokenId": 94971, "text": "Compl", "logit": 7.31682825088501, "prob": 0.04318603500723839 }, { "rank": 4, "tokenId": 854, "text": "com", "logit": 6.769125461578369, "prob": 0.02497347630560398 }, { "rank": 5, "tokenId": 203552, "text": "Vita", "logit": 5.607251167297363, "prob": 0.007814180105924606 }, { "rank": 6, "tokenId": 2078, "text": "inc", "logit": 5.275609016418457, "prob": 0.005608581937849522 }, { "rank": 7, "tokenId": 31766, "text": "sf", "logit": 3.9069595336914062, "prob": 0.001427105744369328 }, { "rank": 8, "tokenId": 43208, "text": "Inc", "logit": 3.221402168273926, "prob": 0.0007189891766756773 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 1558, "winnerText": "Com", "winnerScore": 10.288631735192695, "winnerGap": 0.5897742314308481, "top": [ { "rank": 1, "tokenId": 1558, "text": "Com", "score": 10.288631735192695 }, { "rank": 2, "tokenId": 56331, "text": "compl", "score": 9.698857503761847 }, { "rank": 3, "tokenId": 94971, "text": "Compl", "score": 7.467316569086137 }, { "rank": 4, "tokenId": 854, "text": "com", "score": 6.887648972152183 }, { "rank": 5, "tokenId": 203552, "text": "Vita", "score": 5.673949840845324 }, { "rank": 6, "tokenId": 2078, "text": "inc", "score": 5.331021199506726 }, { "rank": 7, "tokenId": 31766, "text": "sf", "score": 3.9292751162578305 }, { "rank": 8, "tokenId": 43208, "text": "Inc", "score": 3.233868173060266 } ] }, "f32_forward": { "winnerTokenId": 1558, "winnerText": "Com", "winnerScore": 10.2886323928833, "winnerGap": 0.5897789001464844, "top": [ { "rank": 1, "tokenId": 1558, "text": "Com", "score": 10.2886323928833 }, { "rank": 2, "tokenId": 56331, "text": "compl", "score": 9.698853492736816 }, { "rank": 3, "tokenId": 94971, "text": "Compl", "score": 7.467349529266357 }, { "rank": 4, "tokenId": 854, "text": "com", "score": 6.887652397155762 }, { "rank": 5, "tokenId": 203552, "text": "Vita", "score": 5.673947334289551 }, { "rank": 6, "tokenId": 2078, "text": "inc", "score": 5.331033706665039 }, { "rank": 7, "tokenId": 31766, "text": "sf", "score": 3.929273843765259 }, { "rank": 8, "tokenId": 43208, "text": "Inc", "score": 3.233858823776245 } ] }, "f32_reverse": { "winnerTokenId": 1558, "winnerText": "Com", "winnerScore": 10.288631439208984, "winnerGap": 0.5897693634033203, "top": [ { "rank": 1, "tokenId": 1558, "text": "Com", "score": 10.288631439208984 }, { "rank": 2, "tokenId": 56331, "text": "compl", "score": 9.698862075805664 }, { "rank": 3, "tokenId": 94971, "text": "Compl", "score": 7.467328071594238 }, { "rank": 4, "tokenId": 854, "text": "com", "score": 6.887648582458496 }, { "rank": 5, "tokenId": 203552, "text": "Vita", "score": 5.673948287963867 }, { "rank": 6, "tokenId": 2078, "text": "inc", "score": 5.331020355224609 }, { "rank": 7, "tokenId": 31766, "text": "sf", "score": 3.9292726516723633 }, { "rank": 8, "tokenId": 43208, "text": "Inc", "score": 3.2338521480560303 } ] }, "f32_tree64": { "winnerTokenId": 1558, "winnerText": "Com", "winnerScore": 10.288631439208984, "winnerGap": 0.5897750854492188, "top": [ { "rank": 1, "tokenId": 1558, "text": "Com", "score": 10.288631439208984 }, { "rank": 2, "tokenId": 56331, "text": "compl", "score": 9.698856353759766 }, { "rank": 3, "tokenId": 94971, "text": "Compl", "score": 7.467314720153809 }, { "rank": 4, "tokenId": 854, "text": "com", "score": 6.88764762878418 }, { "rank": 5, "tokenId": 203552, "text": "Vita", "score": 5.673947811126709 }, { "rank": 6, "tokenId": 2078, "text": "inc", "score": 5.331020355224609 }, { "rank": 7, "tokenId": 31766, "text": "sf", "score": 3.9292736053466797 }, { "rank": 8, "tokenId": 43208, "text": "Inc", "score": 3.2338693141937256 } ] }, "f16_forward": { "winnerTokenId": 1558, "winnerText": "Com", "winnerScore": 10.328125, "winnerGap": 0.4140625, "top": [ { "rank": 1, "tokenId": 1558, "text": "Com", "score": 10.328125 }, { "rank": 2, "tokenId": 56331, "text": "compl", "score": 9.9140625 }, { "rank": 3, "tokenId": 94971, "text": "Compl", "score": 7.3984375 }, { "rank": 4, "tokenId": 203552, "text": "Vita", "score": 5.66796875 }, { "rank": 5, "tokenId": 2078, "text": "inc", "score": 5.27734375 }, { "rank": 6, "tokenId": 31766, "text": "sf", "score": 3.982421875 }, { "rank": 7, "tokenId": 43208, "text": "Inc", "score": 3.265625 }, { "rank": 8, "tokenId": 854, "text": "com", "score": 2.974609375 } ] } }, "winnerTextsByMode": { "exact": "Com", "f32_forward": "Com", "f32_reverse": "Com", "f32_tree64": "Com", "f16_forward": "Com" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Com" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "lang-japanese-yes-oneword-c", "text": "No explanation. One word only.\nはい or いいえ: the sky is blue.", "stableTopK": [ { "rank": 1, "tokenId": 45621, "text": "はい", "logit": 15.793691635131836, "prob": 0.9940725564956665 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "logit": 10.592906951904297, "prob": 0.00547956395894289 }, { "rank": 3, "tokenId": 72119, "text": "Hai", "logit": 7.411078453063965, "prob": 0.00022745497699361295 }, { "rank": 4, "tokenId": 99075, "text": "Evet", "logit": 6.789294719696045, "prob": 0.0001221400743816048 }, { "rank": 5, "tokenId": 18137, "text": "いい", "logit": 4.75555419921875, "prob": 0.000015981440810719505 }, { "rank": 6, "tokenId": 27577, "text": "Ha", "logit": 4.368228912353516, "prob": 0.000010849324098671786 }, { "rank": 7, "tokenId": 33411, "text": "Ja", "logit": 4.266433238983154, "prob": 0.000009799262443266343 }, { "rank": 8, "tokenId": 7962, "text": "あり", "logit": 4.154669284820557, "prob": 0.000008763042387727182 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 45621, "winnerText": "はい", "winnerScore": 17.556907550981798, "winnerGap": 6.487584721033738, "top": [ { "rank": 1, "tokenId": 45621, "text": "はい", "score": 17.556907550981798 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 11.06932282994806 }, { "rank": 3, "tokenId": 72119, "text": "Hai", "score": 7.567610293236795 }, { "rank": 4, "tokenId": 99075, "text": "Evet", "score": 6.908902129855932 }, { "rank": 5, "tokenId": 18137, "text": "いい", "score": 4.795997163921982 }, { "rank": 6, "tokenId": 27577, "text": "Ha", "score": 4.399500517581625 }, { "rank": 7, "tokenId": 33411, "text": "Ja", "score": 4.295550611185475 }, { "rank": 8, "tokenId": 7962, "text": "あり", "score": 4.181536666225497 } ] }, "f32_forward": { "winnerTokenId": 45621, "winnerText": "はい", "winnerScore": 17.556915283203125, "winnerGap": 6.487588882446289, "top": [ { "rank": 1, "tokenId": 45621, "text": "はい", "score": 17.556915283203125 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 11.069326400756836 }, { "rank": 3, "tokenId": 72119, "text": "Hai", "score": 7.567607879638672 }, { "rank": 4, "tokenId": 99075, "text": "Evet", "score": 6.90890645980835 }, { "rank": 5, "tokenId": 18137, "text": "いい", "score": 4.796000003814697 }, { "rank": 6, "tokenId": 27577, "text": "Ha", "score": 4.399489402770996 }, { "rank": 7, "tokenId": 33411, "text": "Ja", "score": 4.295543670654297 }, { "rank": 8, "tokenId": 7962, "text": "あり", "score": 4.181537628173828 } ] }, "f32_reverse": { "winnerTokenId": 45621, "winnerText": "はい", "winnerScore": 17.556909561157227, "winnerGap": 6.487584114074707, "top": [ { "rank": 1, "tokenId": 45621, "text": "はい", "score": 17.556909561157227 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 11.06932544708252 }, { "rank": 3, "tokenId": 72119, "text": "Hai", "score": 7.567608833312988 }, { "rank": 4, "tokenId": 99075, "text": "Evet", "score": 6.908904075622559 }, { "rank": 5, "tokenId": 18137, "text": "いい", "score": 4.795994758605957 }, { "rank": 6, "tokenId": 27577, "text": "Ha", "score": 4.399500846862793 }, { "rank": 7, "tokenId": 33411, "text": "Ja", "score": 4.295547008514404 }, { "rank": 8, "tokenId": 7962, "text": "あり", "score": 4.181536674499512 } ] }, "f32_tree64": { "winnerTokenId": 45621, "winnerText": "はい", "winnerScore": 17.556907653808594, "winnerGap": 6.487585067749023, "top": [ { "rank": 1, "tokenId": 45621, "text": "はい", "score": 17.556907653808594 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 11.06932258605957 }, { "rank": 3, "tokenId": 72119, "text": "Hai", "score": 7.567610740661621 }, { "rank": 4, "tokenId": 99075, "text": "Evet", "score": 6.908902645111084 }, { "rank": 5, "tokenId": 18137, "text": "いい", "score": 4.79599666595459 }, { "rank": 6, "tokenId": 27577, "text": "Ha", "score": 4.399499893188477 }, { "rank": 7, "tokenId": 33411, "text": "Ja", "score": 4.2955498695373535 }, { "rank": 8, "tokenId": 7962, "text": "あり", "score": 4.181535720825195 } ] }, "f16_forward": { "winnerTokenId": 45621, "winnerText": "はい", "winnerScore": 17.53125, "winnerGap": 6.53125, "top": [ { "rank": 1, "tokenId": 45621, "text": "はい", "score": 17.53125 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 11 }, { "rank": 3, "tokenId": 72119, "text": "Hai", "score": 7.578125 }, { "rank": 4, "tokenId": 99075, "text": "Evet", "score": 6.95703125 }, { "rank": 5, "tokenId": 18137, "text": "いい", "score": 4.7734375 }, { "rank": 6, "tokenId": 42750, "text": "Hay", "score": 4.484375 }, { "rank": 7, "tokenId": 33411, "text": "Ja", "score": 4.33203125 }, { "rank": 8, "tokenId": 7962, "text": "あり", "score": 4.1953125 } ] } }, "winnerTextsByMode": { "exact": "はい", "f32_forward": "はい", "f32_reverse": "はい", "f32_tree64": "はい", "f16_forward": "はい" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "はい" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "lang-korean-question-oneword-d", "text": "Choose one word only.\n맞습니까? 예 또는 아니오:", "stableTopK": [ { "rank": 1, "tokenId": 238643, "text": "예", "logit": 19.95676040649414, "prob": 0.9995193481445312 }, { "rank": 2, "tokenId": 237534, "text": "아", "logit": 11.863698959350586, "prob": 0.0003055056731682271 }, { "rank": 3, "tokenId": 238789, "text": "네", "logit": 10.445368766784668, "prob": 0.00007396841101581231 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "logit": 9.933717727661133, "prob": 0.000044344429625198245 }, { "rank": 5, "tokenId": 1018, "text": "**", "logit": 9.595959663391113, "prob": 0.00003163389192195609 }, { "rank": 6, "tokenId": 238378, "text": "음", "logit": 7.3340888023376465, "prob": 0.0000032948419175227173 }, { "rank": 7, "tokenId": 239307, "text": "질", "logit": 7.056977272033691, "prob": 0.0000024973910512926523 }, { "rank": 8, "tokenId": 238671, "text": "없", "logit": 6.895171642303467, "prob": 0.00000212429722523666 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 238643, "winnerText": "예", "winnerScore": 24.06387529352358, "winnerGap": 11.516319793951038, "top": [ { "rank": 1, "tokenId": 238643, "text": "예", "score": 24.06387529352358 }, { "rank": 2, "tokenId": 237534, "text": "아", "score": 12.547555499572542 }, { "rank": 3, "tokenId": 238789, "text": "네", "score": 10.901097631897215 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 10.322702088608084 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 9.944911201328736 }, { "rank": 6, "tokenId": 238378, "text": "음", "score": 7.485672886290573 }, { "rank": 7, "tokenId": 239307, "text": "질", "score": 7.191641623350833 }, { "rank": 8, "tokenId": 238671, "text": "없", "score": 7.020589714516884 } ] }, "f32_forward": { "winnerTokenId": 238643, "winnerText": "예", "winnerScore": 24.06386375427246, "winnerGap": 11.516312599182129, "top": [ { "rank": 1, "tokenId": 238643, "text": "예", "score": 24.06386375427246 }, { "rank": 2, "tokenId": 237534, "text": "아", "score": 12.547551155090332 }, { "rank": 3, "tokenId": 238789, "text": "네", "score": 10.901090621948242 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 10.322705268859863 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 9.944900512695312 }, { "rank": 6, "tokenId": 238378, "text": "음", "score": 7.485672473907471 }, { "rank": 7, "tokenId": 239307, "text": "질", "score": 7.1916422843933105 }, { "rank": 8, "tokenId": 238671, "text": "없", "score": 7.020599365234375 } ] }, "f32_reverse": { "winnerTokenId": 238643, "winnerText": "예", "winnerScore": 24.06385040283203, "winnerGap": 11.516298294067383, "top": [ { "rank": 1, "tokenId": 238643, "text": "예", "score": 24.06385040283203 }, { "rank": 2, "tokenId": 237534, "text": "아", "score": 12.547552108764648 }, { "rank": 3, "tokenId": 238789, "text": "네", "score": 10.901098251342773 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 10.3226957321167 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 9.944911003112793 }, { "rank": 6, "tokenId": 238378, "text": "음", "score": 7.485673427581787 }, { "rank": 7, "tokenId": 239307, "text": "질", "score": 7.191641807556152 }, { "rank": 8, "tokenId": 238671, "text": "없", "score": 7.020590782165527 } ] }, "f32_tree64": { "winnerTokenId": 238643, "winnerText": "예", "winnerScore": 24.063875198364258, "winnerGap": 11.516319274902344, "top": [ { "rank": 1, "tokenId": 238643, "text": "예", "score": 24.063875198364258 }, { "rank": 2, "tokenId": 237534, "text": "아", "score": 12.547555923461914 }, { "rank": 3, "tokenId": 238789, "text": "네", "score": 10.901098251342773 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 10.322701454162598 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 9.944911003112793 }, { "rank": 6, "tokenId": 238378, "text": "음", "score": 7.4856719970703125 }, { "rank": 7, "tokenId": 239307, "text": "질", "score": 7.191641330718994 }, { "rank": 8, "tokenId": 238671, "text": "없", "score": 7.020589828491211 } ] }, "f16_forward": { "winnerTokenId": 238643, "winnerText": "예", "winnerScore": 24.203125, "winnerGap": 11.484375, "top": [ { "rank": 1, "tokenId": 238643, "text": "예", "score": 24.203125 }, { "rank": 2, "tokenId": 237534, "text": "아", "score": 12.71875 }, { "rank": 3, "tokenId": 238789, "text": "네", "score": 10.90625 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 10.375 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 10.015625 }, { "rank": 6, "tokenId": 238378, "text": "음", "score": 7.53125 }, { "rank": 7, "tokenId": 239307, "text": "질", "score": 7.1171875 }, { "rank": 8, "tokenId": 238671, "text": "없", "score": 7.08984375 } ] } }, "winnerTextsByMode": { "exact": "예", "f32_forward": "예", "f32_reverse": "예", "f32_tree64": "예", "f16_forward": "예" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "예" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "lang-mixed-code-switch-oneword-a", "text": "Answer with exactly one word only.\nThat was très", "stableTopK": [ { "rank": 1, "tokenId": 60881, "text": "Excellent", "logit": 2.2073521614074707, "prob": 0.650748610496521 }, { "rank": 2, "tokenId": 143521, "text": "excellent", "logit": 1.1611213684082031, "prob": 0.22858145833015442 }, { "rank": 3, "tokenId": 118263, "text": "bien", "logit": -0.60904860496521, "prob": 0.03892834484577179 }, { "rank": 4, "tokenId": 11947, "text": "Good", "logit": -0.6637130379676819, "prob": 0.03685746714472771 }, { "rank": 5, "tokenId": 118961, "text": "Merci", "logit": -1.971967101097107, "prob": 0.009962261654436588 }, { "rank": 6, "tokenId": 15466, "text": "good", "logit": -2.251514196395874, "prob": 0.0075327265076339245 }, { "rank": 7, "tokenId": 47330, "text": "Bien", "logit": -2.493192434310913, "prob": 0.005915516521781683 }, { "rank": 8, "tokenId": 47779, "text": "excel", "logit": -3.800654411315918, "prob": 0.0016001812182366848 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 60881, "winnerText": "Excellent", "winnerScore": 2.211345920261241, "winnerGap": 1.0496377209971666, "top": [ { "rank": 1, "tokenId": 60881, "text": "Excellent", "score": 2.211345920261241 }, { "rank": 2, "tokenId": 143521, "text": "excellent", "score": 1.1617081992640745 }, { "rank": 3, "tokenId": 118263, "text": "bien", "score": -0.609135750777881 }, { "rank": 4, "tokenId": 11947, "text": "Good", "score": -0.6638226806656746 }, { "rank": 5, "tokenId": 118961, "text": "Merci", "score": -1.9748131043749666 }, { "rank": 6, "tokenId": 15466, "text": "good", "score": -2.2557556074167735 }, { "rank": 7, "tokenId": 47330, "text": "Bien", "score": -2.498951929872817 }, { "rank": 8, "tokenId": 47779, "text": "excel", "score": -3.8211909711654286 } ] }, "f32_forward": { "winnerTokenId": 60881, "winnerText": "Excellent", "winnerScore": 2.211348056793213, "winnerGap": 1.0496381521224976, "top": [ { "rank": 1, "tokenId": 60881, "text": "Excellent", "score": 2.211348056793213 }, { "rank": 2, "tokenId": 143521, "text": "excellent", "score": 1.1617099046707153 }, { "rank": 3, "tokenId": 118263, "text": "bien", "score": -0.6091359853744507 }, { "rank": 4, "tokenId": 11947, "text": "Good", "score": -0.6638239622116089 }, { "rank": 5, "tokenId": 118961, "text": "Merci", "score": -1.974812388420105 }, { "rank": 6, "tokenId": 15466, "text": "good", "score": -2.255760431289673 }, { "rank": 7, "tokenId": 47330, "text": "Bien", "score": -2.4989542961120605 }, { "rank": 8, "tokenId": 47779, "text": "excel", "score": -3.8211898803710938 } ] }, "f32_reverse": { "winnerTokenId": 60881, "winnerText": "Excellent", "winnerScore": 2.211341142654419, "winnerGap": 1.0496331453323364, "top": [ { "rank": 1, "tokenId": 60881, "text": "Excellent", "score": 2.211341142654419 }, { "rank": 2, "tokenId": 143521, "text": "excellent", "score": 1.1617079973220825 }, { "rank": 3, "tokenId": 118263, "text": "bien", "score": -0.6091421246528625 }, { "rank": 4, "tokenId": 11947, "text": "Good", "score": -0.6638252139091492 }, { "rank": 5, "tokenId": 118961, "text": "Merci", "score": -1.974813461303711 }, { "rank": 6, "tokenId": 15466, "text": "good", "score": -2.255758047103882 }, { "rank": 7, "tokenId": 47330, "text": "Bien", "score": -2.4989445209503174 }, { "rank": 8, "tokenId": 47779, "text": "excel", "score": -3.821193218231201 } ] }, "f32_tree64": { "winnerTokenId": 60881, "winnerText": "Excellent", "winnerScore": 2.21134614944458, "winnerGap": 1.0496383905410767, "top": [ { "rank": 1, "tokenId": 60881, "text": "Excellent", "score": 2.21134614944458 }, { "rank": 2, "tokenId": 143521, "text": "excellent", "score": 1.1617077589035034 }, { "rank": 3, "tokenId": 118263, "text": "bien", "score": -0.6091363430023193 }, { "rank": 4, "tokenId": 11947, "text": "Good", "score": -0.6638232469558716 }, { "rank": 5, "tokenId": 118961, "text": "Merci", "score": -1.9748132228851318 }, { "rank": 6, "tokenId": 15466, "text": "good", "score": -2.2557544708251953 }, { "rank": 7, "tokenId": 47330, "text": "Bien", "score": -2.498950958251953 }, { "rank": 8, "tokenId": 47779, "text": "excel", "score": -3.82119083404541 } ] }, "f16_forward": { "winnerTokenId": 60881, "winnerText": "Excellent", "winnerScore": 2.271484375, "winnerGap": 1.1064453125, "top": [ { "rank": 1, "tokenId": 60881, "text": "Excellent", "score": 2.271484375 }, { "rank": 2, "tokenId": 143521, "text": "excellent", "score": 1.1650390625 }, { "rank": 3, "tokenId": 118263, "text": "bien", "score": -0.62548828125 }, { "rank": 4, "tokenId": 11947, "text": "Good", "score": -0.671875 }, { "rank": 5, "tokenId": 118961, "text": "Merci", "score": -1.9736328125 }, { "rank": 6, "tokenId": 15466, "text": "good", "score": -2.208984375 }, { "rank": 7, "tokenId": 47330, "text": "Bien", "score": -2.53515625 }, { "rank": 8, "tokenId": 589, "text": "ter", "score": -3.13671875 } ] } }, "winnerTextsByMode": { "exact": "Excellent", "f32_forward": "Excellent", "f32_reverse": "Excellent", "f32_tree64": "Excellent", "f16_forward": "Excellent" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Excellent" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "lang-mixed-spanglish-oneword-b", "text": "Respond with one word only.\nThat's not bueno, it's actually", "stableTopK": [ { "rank": 1, "tokenId": 4339, "text": "True", "logit": 5.224925994873047, "prob": 0.7587286829948425 }, { "rank": 2, "tokenId": 34318, "text": "Correct", "logit": 3.2412400245666504, "prob": 0.10437167435884476 }, { "rank": 3, "tokenId": 58459, "text": "Wrong", "logit": 2.3056788444519043, "prob": 0.04095185548067093 }, { "rank": 4, "tokenId": 68755, "text": "Actually", "logit": 2.0704073905944824, "prob": 0.0323665551841259 }, { "rank": 5, "tokenId": 65994, "text": "Better", "logit": 0.49231454730033875, "prob": 0.006679431069642305 }, { "rank": 6, "tokenId": 3397, "text": "true", "logit": 0.4026913046836853, "prob": 0.006106840446591377 }, { "rank": 7, "tokenId": 19058, "text": "Okay", "logit": 0.4022217094898224, "prob": 0.0061039733700454235 }, { "rank": 8, "tokenId": 19448, "text": "correct", "logit": 0.3872837424278259, "prob": 0.006013470236212015 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 5.2787357474272625, "winnerGap": 2.0247973698214192, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 5.2787357474272625 }, { "rank": 2, "tokenId": 34318, "text": "Correct", "score": 3.2539383776058433 }, { "rank": 3, "tokenId": 58459, "text": "Wrong", "score": 2.3102350562146032 }, { "rank": 4, "tokenId": 68755, "text": "Actually", "score": 2.0737053787827 }, { "rank": 5, "tokenId": 65994, "text": "Better", "score": 0.4923611035035709 }, { "rank": 6, "tokenId": 3397, "text": "true", "score": 0.40271580655684147 }, { "rank": 7, "tokenId": 19058, "text": "Okay", "score": 0.4022466712008601 }, { "rank": 8, "tokenId": 19448, "text": "correct", "score": 0.38730634615497106 } ] }, "f32_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 5.278736114501953, "winnerGap": 2.024806022644043, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 5.278736114501953 }, { "rank": 2, "tokenId": 34318, "text": "Correct", "score": 3.25393009185791 }, { "rank": 3, "tokenId": 58459, "text": "Wrong", "score": 2.3102335929870605 }, { "rank": 4, "tokenId": 68755, "text": "Actually", "score": 2.0737006664276123 }, { "rank": 5, "tokenId": 65994, "text": "Better", "score": 0.49235984683036804 }, { "rank": 6, "tokenId": 3397, "text": "true", "score": 0.4027121663093567 }, { "rank": 7, "tokenId": 19058, "text": "Okay", "score": 0.4022435247898102 }, { "rank": 8, "tokenId": 19448, "text": "correct", "score": 0.3873060941696167 } ] }, "f32_reverse": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 5.278735160827637, "winnerGap": 2.024797201156616, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 5.278735160827637 }, { "rank": 2, "tokenId": 34318, "text": "Correct", "score": 3.2539379596710205 }, { "rank": 3, "tokenId": 58459, "text": "Wrong", "score": 2.3102362155914307 }, { "rank": 4, "tokenId": 68755, "text": "Actually", "score": 2.0737106800079346 }, { "rank": 5, "tokenId": 65994, "text": "Better", "score": 0.492360383272171 }, { "rank": 6, "tokenId": 3397, "text": "true", "score": 0.4027111232280731 }, { "rank": 7, "tokenId": 19058, "text": "Okay", "score": 0.4022441506385803 }, { "rank": 8, "tokenId": 19448, "text": "correct", "score": 0.38730478286743164 } ] }, "f32_tree64": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 5.278735637664795, "winnerGap": 2.024797201156616, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 5.278735637664795 }, { "rank": 2, "tokenId": 34318, "text": "Correct", "score": 3.2539384365081787 }, { "rank": 3, "tokenId": 58459, "text": "Wrong", "score": 2.3102355003356934 }, { "rank": 4, "tokenId": 68755, "text": "Actually", "score": 2.073704719543457 }, { "rank": 5, "tokenId": 65994, "text": "Better", "score": 0.4923610985279083 }, { "rank": 6, "tokenId": 3397, "text": "true", "score": 0.40271568298339844 }, { "rank": 7, "tokenId": 19058, "text": "Okay", "score": 0.4022458791732788 }, { "rank": 8, "tokenId": 19448, "text": "correct", "score": 0.3873065710067749 } ] }, "f16_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 5.25390625, "winnerGap": 2.91015625, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 5.25390625 }, { "rank": 2, "tokenId": 58459, "text": "Wrong", "score": 2.34375 }, { "rank": 3, "tokenId": 68755, "text": "Actually", "score": 2.193359375 }, { "rank": 4, "tokenId": 65994, "text": "Better", "score": 0.54638671875 }, { "rank": 5, "tokenId": 19058, "text": "Okay", "score": 0.472412109375 }, { "rank": 6, "tokenId": 19448, "text": "correct", "score": 0.4208984375 }, { "rank": 7, "tokenId": 3397, "text": "true", "score": 0.371826171875 }, { "rank": 8, "tokenId": 11407, "text": "Then", "score": -0.315673828125 } ] } }, "winnerTextsByMode": { "exact": "True", "f32_forward": "True", "f32_reverse": "True", "f32_tree64": "True", "f16_forward": "True" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "True" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "lang-portuguese-stem-oneword-c", "text": "No explanation. One word only.\nA resposta correta é", "stableTopK": [ { "rank": 1, "tokenId": 9028, "text": "Sim", "logit": 14.17142105102539, "prob": 0.2997512221336365 }, { "rank": 2, "tokenId": 3778, "text": "Pr", "logit": 13.54032039642334, "prob": 0.15946942567825317 }, { "rank": 3, "tokenId": 236776, "text": "A", "logit": 13.392650604248047, "prob": 0.13757681846618652 }, { "rank": 4, "tokenId": 69250, "text": "Continu", "logit": 13.112421035766602, "prob": 0.10395445674657822 }, { "rank": 5, "tokenId": 195748, "text": "Esper", "logit": 12.876502990722656, "prob": 0.08210794627666473 }, { "rank": 6, "tokenId": 105494, "text": "Resposta", "logit": 12.446630477905273, "prob": 0.05341877415776253 }, { "rank": 7, "tokenId": 4724, "text": "Ent", "logit": 11.73401927947998, "prob": 0.02619454264640808 }, { "rank": 8, "tokenId": 34230, "text": "Continue", "logit": 11.227110862731934, "prob": 0.01577841304242611 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 9028, "winnerText": "Sim", "winnerScore": 15.39390718454527, "winnerGap": 0.8023021813944293, "top": [ { "rank": 1, "tokenId": 9028, "text": "Sim", "score": 15.39390718454527 }, { "rank": 2, "tokenId": 3778, "text": "Pr", "score": 14.59160500315084 }, { "rank": 3, "tokenId": 236776, "text": "A", "score": 14.40667175102513 }, { "rank": 4, "tokenId": 69250, "text": "Continu", "score": 14.058489988986977 }, { "rank": 5, "tokenId": 195748, "text": "Esper", "score": 13.768087813918985 }, { "rank": 6, "tokenId": 105494, "text": "Resposta", "score": 13.245021386855317 }, { "rank": 7, "tokenId": 4724, "text": "Ent", "score": 12.394141343919586 }, { "rank": 8, "tokenId": 34230, "text": "Continue", "score": 11.800239606621702 } ] }, "f32_forward": { "winnerTokenId": 9028, "winnerText": "Sim", "winnerScore": 15.39390754699707, "winnerGap": 0.8023147583007812, "top": [ { "rank": 1, "tokenId": 9028, "text": "Sim", "score": 15.39390754699707 }, { "rank": 2, "tokenId": 3778, "text": "Pr", "score": 14.591592788696289 }, { "rank": 3, "tokenId": 236776, "text": "A", "score": 14.406661033630371 }, { "rank": 4, "tokenId": 69250, "text": "Continu", "score": 14.058514595031738 }, { "rank": 5, "tokenId": 195748, "text": "Esper", "score": 13.768101692199707 }, { "rank": 6, "tokenId": 105494, "text": "Resposta", "score": 13.245019912719727 }, { "rank": 7, "tokenId": 4724, "text": "Ent", "score": 12.394132614135742 }, { "rank": 8, "tokenId": 34230, "text": "Continue", "score": 11.800233840942383 } ] }, "f32_reverse": { "winnerTokenId": 9028, "winnerText": "Sim", "winnerScore": 15.393908500671387, "winnerGap": 0.8022985458374023, "top": [ { "rank": 1, "tokenId": 9028, "text": "Sim", "score": 15.393908500671387 }, { "rank": 2, "tokenId": 3778, "text": "Pr", "score": 14.591609954833984 }, { "rank": 3, "tokenId": 236776, "text": "A", "score": 14.406682014465332 }, { "rank": 4, "tokenId": 69250, "text": "Continu", "score": 14.058490753173828 }, { "rank": 5, "tokenId": 195748, "text": "Esper", "score": 13.768099784851074 }, { "rank": 6, "tokenId": 105494, "text": "Resposta", "score": 13.245018005371094 }, { "rank": 7, "tokenId": 4724, "text": "Ent", "score": 12.394142150878906 }, { "rank": 8, "tokenId": 34230, "text": "Continue", "score": 11.800235748291016 } ] }, "f32_tree64": { "winnerTokenId": 9028, "winnerText": "Sim", "winnerScore": 15.39390754699707, "winnerGap": 0.802302360534668, "top": [ { "rank": 1, "tokenId": 9028, "text": "Sim", "score": 15.39390754699707 }, { "rank": 2, "tokenId": 3778, "text": "Pr", "score": 14.591605186462402 }, { "rank": 3, "tokenId": 236776, "text": "A", "score": 14.406671524047852 }, { "rank": 4, "tokenId": 69250, "text": "Continu", "score": 14.058490753173828 }, { "rank": 5, "tokenId": 195748, "text": "Esper", "score": 13.768087387084961 }, { "rank": 6, "tokenId": 105494, "text": "Resposta", "score": 13.245020866394043 }, { "rank": 7, "tokenId": 4724, "text": "Ent", "score": 12.394142150878906 }, { "rank": 8, "tokenId": 34230, "text": "Continue", "score": 11.800238609313965 } ] }, "f16_forward": { "winnerTokenId": 9028, "winnerText": "Sim", "winnerScore": 15.640625, "winnerGap": 0.984375, "top": [ { "rank": 1, "tokenId": 9028, "text": "Sim", "score": 15.640625 }, { "rank": 2, "tokenId": 3778, "text": "Pr", "score": 14.65625 }, { "rank": 3, "tokenId": 236776, "text": "A", "score": 14.4296875 }, { "rank": 4, "tokenId": 69250, "text": "Continu", "score": 14.03125 }, { "rank": 5, "tokenId": 195748, "text": "Esper", "score": 13.6328125 }, { "rank": 6, "tokenId": 105494, "text": "Resposta", "score": 13.2734375 }, { "rank": 7, "tokenId": 4724, "text": "Ent", "score": 12.4453125 }, { "rank": 8, "tokenId": 34230, "text": "Continue", "score": 11.828125 } ] } }, "winnerTextsByMode": { "exact": "Sim", "f32_forward": "Sim", "f32_reverse": "Sim", "f32_tree64": "Sim", "f16_forward": "Sim" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Sim" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "lang-russian-choice-oneword-d", "text": "Choose one word only.\nПравда или ложь: огонь холодный.", "stableTopK": [ { "rank": 1, "tokenId": 148195, "text": "Ло", "logit": 16.058639526367188, "prob": 0.9953905344009399 }, { "rank": 2, "tokenId": 102473, "text": "Нет", "logit": 9.451462745666504, "prob": 0.0013444141950458288 }, { "rank": 3, "tokenId": 132523, "text": "лож", "logit": 9.421920776367188, "prob": 0.0013052785070613027 }, { "rank": 4, "tokenId": 1018, "text": "**", "logit": 9.31903076171875, "prob": 0.0011776563478633761 }, { "rank": 5, "tokenId": 26450, "text": "Не", "logit": 8.056392669677734, "prob": 0.0003331668849568814 }, { "rank": 6, "tokenId": 237612, "text": "Л", "logit": 6.473593235015869, "prob": 0.00006843224400654435 }, { "rank": 7, "tokenId": 113258, "text": "ЛО", "logit": 6.136550426483154, "prob": 0.00004885229282081127 }, { "rank": 8, "tokenId": 237134, "text": "П", "logit": 5.751087188720703, "prob": 0.00003322617703815922 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 148195, "winnerText": "Ло", "winnerScore": 17.9258362807495, "winnerGap": 8.141613637647291, "top": [ { "rank": 1, "tokenId": 148195, "text": "Ло", "score": 17.9258362807495 }, { "rank": 2, "tokenId": 102473, "text": "Нет", "score": 9.78422264310221 }, { "rank": 3, "tokenId": 132523, "text": "лож", "score": 9.751436346477078 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 9.63742052091019 }, { "rank": 5, "tokenId": 26450, "text": "Не", "score": 8.25889619624773 }, { "rank": 6, "tokenId": 237612, "text": "Л", "score": 6.576976514149365 }, { "rank": 7, "tokenId": 113258, "text": "ЛО", "score": 6.224345816970397 }, { "rank": 8, "tokenId": 237134, "text": "П", "score": 5.823132380949251 } ] }, "f32_forward": { "winnerTokenId": 148195, "winnerText": "Ло", "winnerScore": 17.925851821899414, "winnerGap": 8.141630172729492, "top": [ { "rank": 1, "tokenId": 148195, "text": "Ло", "score": 17.925851821899414 }, { "rank": 2, "tokenId": 102473, "text": "Нет", "score": 9.784221649169922 }, { "rank": 3, "tokenId": 132523, "text": "лож", "score": 9.75143814086914 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 9.637429237365723 }, { "rank": 5, "tokenId": 26450, "text": "Не", "score": 8.258899688720703 }, { "rank": 6, "tokenId": 237612, "text": "Л", "score": 6.576977729797363 }, { "rank": 7, "tokenId": 113258, "text": "ЛО", "score": 6.2243547439575195 }, { "rank": 8, "tokenId": 237134, "text": "П", "score": 5.823139667510986 } ] }, "f32_reverse": { "winnerTokenId": 148195, "winnerText": "Ло", "winnerScore": 17.925832748413086, "winnerGap": 8.141605377197266, "top": [ { "rank": 1, "tokenId": 148195, "text": "Ло", "score": 17.925832748413086 }, { "rank": 2, "tokenId": 102473, "text": "Нет", "score": 9.78422737121582 }, { "rank": 3, "tokenId": 132523, "text": "лож", "score": 9.751429557800293 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 9.637421607971191 }, { "rank": 5, "tokenId": 26450, "text": "Не", "score": 8.258904457092285 }, { "rank": 6, "tokenId": 237612, "text": "Л", "score": 6.576974868774414 }, { "rank": 7, "tokenId": 113258, "text": "ЛО", "score": 6.224347114562988 }, { "rank": 8, "tokenId": 237134, "text": "П", "score": 5.823132514953613 } ] }, "f32_tree64": { "winnerTokenId": 148195, "winnerText": "Ло", "winnerScore": 17.92583465576172, "winnerGap": 8.14161205291748, "top": [ { "rank": 1, "tokenId": 148195, "text": "Ло", "score": 17.92583465576172 }, { "rank": 2, "tokenId": 102473, "text": "Нет", "score": 9.784222602844238 }, { "rank": 3, "tokenId": 132523, "text": "лож", "score": 9.751436233520508 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 9.637420654296875 }, { "rank": 5, "tokenId": 26450, "text": "Не", "score": 8.258895874023438 }, { "rank": 6, "tokenId": 237612, "text": "Л", "score": 6.576976776123047 }, { "rank": 7, "tokenId": 113258, "text": "ЛО", "score": 6.2243452072143555 }, { "rank": 8, "tokenId": 237134, "text": "П", "score": 5.823131561279297 } ] }, "f16_forward": { "winnerTokenId": 148195, "winnerText": "Ло", "winnerScore": 18.125, "winnerGap": 8.2578125, "top": [ { "rank": 1, "tokenId": 148195, "text": "Ло", "score": 18.125 }, { "rank": 2, "tokenId": 102473, "text": "Нет", "score": 9.8671875 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 9.7578125 }, { "rank": 4, "tokenId": 26450, "text": "Не", "score": 8.3203125 }, { "rank": 5, "tokenId": 237612, "text": "Л", "score": 6.6484375 }, { "rank": 6, "tokenId": 113258, "text": "ЛО", "score": 6.29296875 }, { "rank": 7, "tokenId": 237248, "text": "О", "score": 5.890625 }, { "rank": 8, "tokenId": 237134, "text": "П", "score": 5.88671875 } ] } }, "winnerTextsByMode": { "exact": "Ло", "f32_forward": "Ло", "f32_reverse": "Ло", "f32_tree64": "Ло", "f16_forward": "Ло" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Ло" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "lang-spanish-answer-oneword-a", "text": "Answer with exactly one word only.\nLa respuesta es sí o no. Es", "stableTopK": [ { "rank": 1, "tokenId": 3771, "text": "No", "logit": 17.360862731933594, "prob": 0.9751322269439697 }, { "rank": 2, "tokenId": 88449, "text": "S�", "logit": 13.59980297088623, "prob": 0.022680677473545074 }, { "rank": 3, "tokenId": 38021, "text": "Sin", "logit": 10.65077018737793, "prob": 0.0011882486287504435 }, { "rank": 4, "tokenId": 1904, "text": "no", "logit": 9.743096351623535, "prob": 0.00047941275988705456 }, { "rank": 5, "tokenId": 237665, "text": "�", "logit": 7.56147575378418, "prob": 0.000054105796152725816 }, { "rank": 6, "tokenId": 12014, "text": "Si", "logit": 7.533742427825928, "prob": 0.00005262588456389494 }, { "rank": 7, "tokenId": 236797, "text": "N", "logit": 7.152017593383789, "prob": 0.00003592678694985807 }, { "rank": 8, "tokenId": 230380, "text": "Ning", "logit": 7.138147354125977, "prob": 0.00003543191633070819 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 19.814972387294844, "winnerGap": 5.148579306774755, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 19.814972387294844 }, { "rank": 2, "tokenId": 88449, "text": "S�", "score": 14.66639308052009 }, { "rank": 3, "tokenId": 38021, "text": "Sin", "score": 11.135482444773784 }, { "rank": 4, "tokenId": 1904, "text": "no", "score": 10.109104476318569 }, { "rank": 5, "tokenId": 237665, "text": "�", "score": 7.727994588616241 }, { "rank": 6, "tokenId": 12014, "text": "Si", "score": 7.698385824354869 }, { "rank": 7, "tokenId": 236797, "text": "N", "score": 7.292328261602854 }, { "rank": 8, "tokenId": 230380, "text": "Ning", "score": 7.277625292446402 } ] }, "f32_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 19.814945220947266, "winnerGap": 5.148553848266602, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 19.814945220947266 }, { "rank": 2, "tokenId": 88449, "text": "S�", "score": 14.666391372680664 }, { "rank": 3, "tokenId": 38021, "text": "Sin", "score": 11.135467529296875 }, { "rank": 4, "tokenId": 1904, "text": "no", "score": 10.109111785888672 }, { "rank": 5, "tokenId": 237665, "text": "�", "score": 7.728000640869141 }, { "rank": 6, "tokenId": 12014, "text": "Si", "score": 7.698392391204834 }, { "rank": 7, "tokenId": 236797, "text": "N", "score": 7.292324542999268 }, { "rank": 8, "tokenId": 230380, "text": "Ning", "score": 7.277628421783447 } ] }, "f32_reverse": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 19.814952850341797, "winnerGap": 5.148564338684082, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 19.814952850341797 }, { "rank": 2, "tokenId": 88449, "text": "S�", "score": 14.666388511657715 }, { "rank": 3, "tokenId": 38021, "text": "Sin", "score": 11.135485649108887 }, { "rank": 4, "tokenId": 1904, "text": "no", "score": 10.109105110168457 }, { "rank": 5, "tokenId": 237665, "text": "�", "score": 7.727996349334717 }, { "rank": 6, "tokenId": 12014, "text": "Si", "score": 7.698383808135986 }, { "rank": 7, "tokenId": 236797, "text": "N", "score": 7.292331695556641 }, { "rank": 8, "tokenId": 230380, "text": "Ning", "score": 7.277625560760498 } ] }, "f32_tree64": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 19.814971923828125, "winnerGap": 5.148580551147461, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 19.814971923828125 }, { "rank": 2, "tokenId": 88449, "text": "S�", "score": 14.666391372680664 }, { "rank": 3, "tokenId": 38021, "text": "Sin", "score": 11.135482788085938 }, { "rank": 4, "tokenId": 1904, "text": "no", "score": 10.109103202819824 }, { "rank": 5, "tokenId": 237665, "text": "�", "score": 7.7279953956604 }, { "rank": 6, "tokenId": 12014, "text": "Si", "score": 7.698386192321777 }, { "rank": 7, "tokenId": 236797, "text": "N", "score": 7.292328357696533 }, { "rank": 8, "tokenId": 230380, "text": "Ning", "score": 7.277624607086182 } ] }, "f16_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 19.53125, "winnerGap": 4.8515625, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 19.53125 }, { "rank": 2, "tokenId": 88449, "text": "S�", "score": 14.6796875 }, { "rank": 3, "tokenId": 38021, "text": "Sin", "score": 11.3203125 }, { "rank": 4, "tokenId": 1904, "text": "no", "score": 10.0078125 }, { "rank": 5, "tokenId": 12014, "text": "Si", "score": 7.76171875 }, { "rank": 6, "tokenId": 106, "text": "", "score": 6.73828125 }, { "rank": 7, "tokenId": 81880, "text": "s�", "score": 6.6171875 }, { "rank": 8, "tokenId": 31049, "text": "Depend", "score": 6.3671875 } ] } }, "winnerTextsByMode": { "exact": "No", "f32_forward": "No", "f32_reverse": "No", "f32_tree64": "No", "f16_forward": "No" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "No" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "lang-swahili-stem-oneword-b", "text": "Respond with one word only.\nJibu sahihi ni", "stableTopK": [ { "rank": 1, "tokenId": 236776, "text": "A", "logit": 15.36073112487793, "prob": 0.3156351149082184 }, { "rank": 2, "tokenId": 236797, "text": "N", "logit": 14.266483306884766, "prob": 0.10567190498113632 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "logit": 14.155978202819824, "prob": 0.09461669623851776 }, { "rank": 4, "tokenId": 9366, "text": "Please", "logit": 13.497368812561035, "prob": 0.04897081479430199 }, { "rank": 5, "tokenId": 236855, "text": "K", "logit": 13.46823501586914, "prob": 0.04756469279527664 }, { "rank": 6, "tokenId": 236774, "text": "T", "logit": 13.394927024841309, "prob": 0.04420256242156029 }, { "rank": 7, "tokenId": 236799, "text": "B", "logit": 13.222310066223145, "prob": 0.03719468042254448 }, { "rank": 8, "tokenId": 236773, "text": "S", "logit": 13.153977394104004, "prob": 0.0347379632294178 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 236776, "winnerText": "A", "winnerScore": 16.964096341979182, "winnerGap": 1.4475793295968558, "top": [ { "rank": 1, "tokenId": 236776, "text": "A", "score": 16.964096341979182 }, { "rank": 2, "tokenId": 236797, "text": "N", "score": 15.516517012382327 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": 15.374041570351196 }, { "rank": 4, "tokenId": 9366, "text": "Please", "score": 14.537708872040756 }, { "rank": 5, "tokenId": 236855, "text": "K", "score": 14.501205372533091 }, { "rank": 6, "tokenId": 236774, "text": "T", "score": 14.409510546998904 }, { "rank": 7, "tokenId": 236799, "text": "B", "score": 14.194601730322995 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": 14.109895870151277 } ] }, "f32_forward": { "winnerTokenId": 236776, "winnerText": "A", "winnerScore": 16.964113235473633, "winnerGap": 1.4475927352905273, "top": [ { "rank": 1, "tokenId": 236776, "text": "A", "score": 16.964113235473633 }, { "rank": 2, "tokenId": 236797, "text": "N", "score": 15.516520500183105 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": 15.374040603637695 }, { "rank": 4, "tokenId": 9366, "text": "Please", "score": 14.537705421447754 }, { "rank": 5, "tokenId": 236855, "text": "K", "score": 14.501213073730469 }, { "rank": 6, "tokenId": 236774, "text": "T", "score": 14.40950870513916 }, { "rank": 7, "tokenId": 236799, "text": "B", "score": 14.194604873657227 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": 14.109885215759277 } ] }, "f32_reverse": { "winnerTokenId": 236776, "winnerText": "A", "winnerScore": 16.964094161987305, "winnerGap": 1.4475688934326172, "top": [ { "rank": 1, "tokenId": 236776, "text": "A", "score": 16.964094161987305 }, { "rank": 2, "tokenId": 236797, "text": "N", "score": 15.516525268554688 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": 15.374041557312012 }, { "rank": 4, "tokenId": 9366, "text": "Please", "score": 14.537705421447754 }, { "rank": 5, "tokenId": 236855, "text": "K", "score": 14.501205444335938 }, { "rank": 6, "tokenId": 236774, "text": "T", "score": 14.409512519836426 }, { "rank": 7, "tokenId": 236799, "text": "B", "score": 14.194581031799316 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": 14.109903335571289 } ] }, "f32_tree64": { "winnerTokenId": 236776, "winnerText": "A", "winnerScore": 16.964096069335938, "winnerGap": 1.4475793838500977, "top": [ { "rank": 1, "tokenId": 236776, "text": "A", "score": 16.964096069335938 }, { "rank": 2, "tokenId": 236797, "text": "N", "score": 15.51651668548584 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": 15.374041557312012 }, { "rank": 4, "tokenId": 9366, "text": "Please", "score": 14.537708282470703 }, { "rank": 5, "tokenId": 236855, "text": "K", "score": 14.501205444335938 }, { "rank": 6, "tokenId": 236774, "text": "T", "score": 14.409510612487793 }, { "rank": 7, "tokenId": 236799, "text": "B", "score": 14.194602012634277 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": 14.109894752502441 } ] }, "f16_forward": { "winnerTokenId": 236776, "winnerText": "A", "winnerScore": 17.0625, "winnerGap": 1.53125, "top": [ { "rank": 1, "tokenId": 236776, "text": "A", "score": 17.0625 }, { "rank": 2, "tokenId": 236797, "text": "N", "score": 15.53125 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": 15.5 }, { "rank": 4, "tokenId": 9366, "text": "Please", "score": 14.59375 }, { "rank": 5, "tokenId": 236774, "text": "T", "score": 14.5546875 }, { "rank": 6, "tokenId": 236855, "text": "K", "score": 14.5078125 }, { "rank": 7, "tokenId": 236799, "text": "B", "score": 14.1875 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": 14.0234375 } ] } }, "winnerTextsByMode": { "exact": "A", "f32_forward": "A", "f32_reverse": "A", "f32_tree64": "A", "f16_forward": "A" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "A" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "lang-turkish-bool-oneword-c", "text": "No explanation. One word only.\nDoğru mu yanlış mı: su ıslaktır.", "stableTopK": [ { "rank": 1, "tokenId": 6294, "text": "Do", "logit": 18.385986328125, "prob": 0.9834952354431152 }, { "rank": 2, "tokenId": 128135, "text": "Yan", "logit": 14.22293758392334, "prob": 0.01530323177576065 }, { "rank": 3, "tokenId": 99075, "text": "Evet", "logit": 11.38431167602539, "prob": 0.0008953309734351933 }, { "rank": 4, "tokenId": 33338, "text": "Ger", "logit": 8.560774803161621, "prob": 0.000053178628149908036 }, { "rank": 5, "tokenId": 42750, "text": "Hay", "logit": 8.203880310058594, "prob": 0.00003721686516655609 }, { "rank": 6, "tokenId": 2320, "text": "do", "logit": 7.790743827819824, "prob": 0.000024621635020594113 }, { "rank": 7, "tokenId": 57456, "text": "Tek", "logit": 7.671677589416504, "prob": 0.00002185783341701608 }, { "rank": 8, "tokenId": 4339, "text": "True", "logit": 7.0589165687561035, "prob": 0.00001184372649731813 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 6294, "winnerText": "Do", "winnerScore": 21.404972156414686, "winnerGap": 5.944670332220305, "top": [ { "rank": 1, "tokenId": 6294, "text": "Do", "score": 21.404972156414686 }, { "rank": 2, "tokenId": 128135, "text": "Yan", "score": 15.460301824194381 }, { "rank": 3, "tokenId": 99075, "text": "Evet", "score": 11.983463856157572 }, { "rank": 4, "tokenId": 33338, "text": "Ger", "score": 8.805204062567583 }, { "rank": 5, "tokenId": 42750, "text": "Hay", "score": 8.418075447248905 }, { "rank": 6, "tokenId": 2320, "text": "do", "score": 7.973325695668986 }, { "rank": 7, "tokenId": 57456, "text": "Tek", "score": 7.845792336009826 }, { "rank": 8, "tokenId": 4339, "text": "True", "score": 7.193693508541543 } ] }, "f32_forward": { "winnerTokenId": 6294, "winnerText": "Do", "winnerScore": 21.40496826171875, "winnerGap": 5.944680213928223, "top": [ { "rank": 1, "tokenId": 6294, "text": "Do", "score": 21.40496826171875 }, { "rank": 2, "tokenId": 128135, "text": "Yan", "score": 15.460288047790527 }, { "rank": 3, "tokenId": 99075, "text": "Evet", "score": 11.983452796936035 }, { "rank": 4, "tokenId": 33338, "text": "Ger", "score": 8.80520248413086 }, { "rank": 5, "tokenId": 42750, "text": "Hay", "score": 8.4180908203125 }, { "rank": 6, "tokenId": 2320, "text": "do", "score": 7.97332239151001 }, { "rank": 7, "tokenId": 57456, "text": "Tek", "score": 7.845789432525635 }, { "rank": 8, "tokenId": 4339, "text": "True", "score": 7.19370174407959 } ] }, "f32_reverse": { "winnerTokenId": 6294, "winnerText": "Do", "winnerScore": 21.40498161315918, "winnerGap": 5.944685935974121, "top": [ { "rank": 1, "tokenId": 6294, "text": "Do", "score": 21.40498161315918 }, { "rank": 2, "tokenId": 128135, "text": "Yan", "score": 15.460295677185059 }, { "rank": 3, "tokenId": 99075, "text": "Evet", "score": 11.98346996307373 }, { "rank": 4, "tokenId": 33338, "text": "Ger", "score": 8.805204391479492 }, { "rank": 5, "tokenId": 42750, "text": "Hay", "score": 8.418070793151855 }, { "rank": 6, "tokenId": 2320, "text": "do", "score": 7.97332763671875 }, { "rank": 7, "tokenId": 57456, "text": "Tek", "score": 7.845794677734375 }, { "rank": 8, "tokenId": 4339, "text": "True", "score": 7.193691730499268 } ] }, "f32_tree64": { "winnerTokenId": 6294, "winnerText": "Do", "winnerScore": 21.404972076416016, "winnerGap": 5.944670677185059, "top": [ { "rank": 1, "tokenId": 6294, "text": "Do", "score": 21.404972076416016 }, { "rank": 2, "tokenId": 128135, "text": "Yan", "score": 15.460301399230957 }, { "rank": 3, "tokenId": 99075, "text": "Evet", "score": 11.983464241027832 }, { "rank": 4, "tokenId": 33338, "text": "Ger", "score": 8.805203437805176 }, { "rank": 5, "tokenId": 42750, "text": "Hay", "score": 8.418075561523438 }, { "rank": 6, "tokenId": 2320, "text": "do", "score": 7.973326206207275 }, { "rank": 7, "tokenId": 57456, "text": "Tek", "score": 7.8457932472229 }, { "rank": 8, "tokenId": 4339, "text": "True", "score": 7.193693161010742 } ] }, "f16_forward": { "winnerTokenId": 6294, "winnerText": "Do", "winnerScore": 21.4375, "winnerGap": 5.9140625, "top": [ { "rank": 1, "tokenId": 6294, "text": "Do", "score": 21.4375 }, { "rank": 2, "tokenId": 128135, "text": "Yan", "score": 15.5234375 }, { "rank": 3, "tokenId": 99075, "text": "Evet", "score": 12.078125 }, { "rank": 4, "tokenId": 42750, "text": "Hay", "score": 8.46875 }, { "rank": 5, "tokenId": 2320, "text": "do", "score": 8.0390625 }, { "rank": 6, "tokenId": 4339, "text": "True", "score": 7.296875 }, { "rank": 7, "tokenId": 106, "text": "", "score": 7.1328125 }, { "rank": 8, "tokenId": 238416, "text": "�", "score": 6.77734375 } ] } }, "winnerTextsByMode": { "exact": "Do", "f32_forward": "Do", "f32_reverse": "Do", "f32_tree64": "Do", "f16_forward": "Do" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Do" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "light-true-false-both-oneword-d", "text": "Choose one word only.\nTrue, false, or both: light is fundamentally a wave", "stableTopK": [ { "rank": 1, "tokenId": 4339, "text": "True", "logit": 12.875286102294922, "prob": 0.959497332572937 }, { "rank": 2, "tokenId": 27413, "text": "both", "logit": 8.962235450744629, "prob": 0.019170235842466354 }, { "rank": 3, "tokenId": 22186, "text": "Both", "logit": 8.791983604431152, "prob": 0.01616918109357357 }, { "rank": 4, "tokenId": 1018, "text": "**", "logit": 7.176557540893555, "prob": 0.003214529249817133 }, { "rank": 5, "tokenId": 3397, "text": "true", "logit": 6.483299255371094, "prob": 0.0016070861602202058 }, { "rank": 6, "tokenId": 9277, "text": "False", "logit": 4.236281871795654, "prob": 0.00016989160212688148 }, { "rank": 7, "tokenId": 10784, "text": "Yes", "logit": 1.5966264009475708, "prob": 0.000012127857189625502 }, { "rank": 8, "tokenId": 58930, "text": "either", "logit": 1.2767003774642944, "prob": 0.000008807283848000225 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.766594645507752, "winnerGap": 4.522488927195013, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.766594645507752 }, { "rank": 2, "tokenId": 27413, "text": "both", "score": 9.244105718312738 }, { "rank": 3, "tokenId": 22186, "text": "Both", "score": 9.057517760612527 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 7.318354277462809 }, { "rank": 5, "tokenId": 3397, "text": "true", "score": 6.587157476474459 }, { "rank": 6, "tokenId": 9277, "text": "False", "score": 4.2647829480491595 }, { "rank": 7, "tokenId": 10784, "text": "Yes", "score": 1.5981367017873964 }, { "rank": 8, "tokenId": 58930, "text": "either", "score": 1.2774722936302068 } ] }, "f32_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.766599655151367, "winnerGap": 4.522495269775391, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.766599655151367 }, { "rank": 2, "tokenId": 27413, "text": "both", "score": 9.244104385375977 }, { "rank": 3, "tokenId": 22186, "text": "Both", "score": 9.057514190673828 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 7.318358898162842 }, { "rank": 5, "tokenId": 3397, "text": "true", "score": 6.587164878845215 }, { "rank": 6, "tokenId": 9277, "text": "False", "score": 4.26478385925293 }, { "rank": 7, "tokenId": 10784, "text": "Yes", "score": 1.5981347560882568 }, { "rank": 8, "tokenId": 58930, "text": "either", "score": 1.2774674892425537 } ] }, "f32_reverse": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.766587257385254, "winnerGap": 4.52248477935791, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.766587257385254 }, { "rank": 2, "tokenId": 27413, "text": "both", "score": 9.244102478027344 }, { "rank": 3, "tokenId": 22186, "text": "Both", "score": 9.057518005371094 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 7.318350315093994 }, { "rank": 5, "tokenId": 3397, "text": "true", "score": 6.587154865264893 }, { "rank": 6, "tokenId": 9277, "text": "False", "score": 4.264784812927246 }, { "rank": 7, "tokenId": 10784, "text": "Yes", "score": 1.5981377363204956 }, { "rank": 8, "tokenId": 58930, "text": "either", "score": 1.2774710655212402 } ] }, "f32_tree64": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.766593933105469, "winnerGap": 4.522487640380859, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.766593933105469 }, { "rank": 2, "tokenId": 27413, "text": "both", "score": 9.24410629272461 }, { "rank": 3, "tokenId": 22186, "text": "Both", "score": 9.057518005371094 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 7.318355560302734 }, { "rank": 5, "tokenId": 3397, "text": "true", "score": 6.587157249450684 }, { "rank": 6, "tokenId": 9277, "text": "False", "score": 4.264782905578613 }, { "rank": 7, "tokenId": 10784, "text": "Yes", "score": 1.5981369018554688 }, { "rank": 8, "tokenId": 58930, "text": "either", "score": 1.2774721384048462 } ] }, "f16_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 12.78125, "winnerGap": 3.5390625, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 12.78125 }, { "rank": 2, "tokenId": 27413, "text": "both", "score": 9.2421875 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 7.33203125 }, { "rank": 4, "tokenId": 3397, "text": "true", "score": 6.66015625 }, { "rank": 5, "tokenId": 22186, "text": "Both", "score": 5.12890625 }, { "rank": 6, "tokenId": 9277, "text": "False", "score": 4.2734375 }, { "rank": 7, "tokenId": 10784, "text": "Yes", "score": 1.6005859375 }, { "rank": 8, "tokenId": 58930, "text": "either", "score": 1.296875 } ] } }, "winnerTextsByMode": { "exact": "True", "f32_forward": "True", "f32_reverse": "True", "f32_tree64": "True", "f16_forward": "True" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "True" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "macro-allow-block-choice-oneword-a", "text": "Answer with exactly one word only.\nChoose exactly one word: allow or block.\nAn unexpected email attachment with macros should", "stableTopK": [ { "rank": 1, "tokenId": 5088, "text": "block", "logit": 7.376798629760742, "prob": 0.9996992945671082 }, { "rank": 2, "tokenId": 10809, "text": "Block", "logit": -0.7389419078826904, "prob": 0.00029870879370719194 }, { "rank": 3, "tokenId": 8021, "text": "allow", "logit": -5.96421480178833, "prob": 0.000001606722321412235 }, { "rank": 4, "tokenId": 27122, "text": "Allow", "logit": -8.301785469055176, "prob": 1.5514822848672338e-7 }, { "rank": 5, "tokenId": 124174, "text": "ブロック", "logit": -9.078320503234863, "prob": 7.136773660931794e-8 }, { "rank": 6, "tokenId": 70961, "text": " ब्लॉक", "logit": -9.872418403625488, "prob": 3.225741807000304e-8 }, { "rank": 7, "tokenId": 61836, "text": "BLOCK", "logit": -10.031744003295898, "prob": 2.7506501965035568e-8 }, { "rank": 8, "tokenId": 3355, "text": " block", "logit": -10.803169250488281, "prob": 1.2717731756595185e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 5088, "winnerText": "block", "winnerScore": 7.531107790448829, "winnerGap": 8.270196903005152, "top": [ { "rank": 1, "tokenId": 5088, "text": "block", "score": 7.531107790448829 }, { "rank": 2, "tokenId": 10809, "text": "Block", "score": -0.7390891125563233 }, { "rank": 3, "tokenId": 8021, "text": "allow", "score": -6.044706588665292 }, { "rank": 4, "tokenId": 27122, "text": "Allow", "score": -8.523999376006437 }, { "rank": 5, "tokenId": 124174, "text": "ブロック", "score": -9.37172793905168 }, { "rank": 6, "tokenId": 70961, "text": " ब्लॉक", "score": -10.253912122227604 }, { "rank": 7, "tokenId": 61836, "text": "BLOCK", "score": -10.432931926174604 }, { "rank": 8, "tokenId": 3355, "text": " block", "score": -11.310223146187923 } ] }, "f32_forward": { "winnerTokenId": 5088, "winnerText": "block", "winnerScore": 7.531106472015381, "winnerGap": 8.270197212696075, "top": [ { "rank": 1, "tokenId": 5088, "text": "block", "score": 7.531106472015381 }, { "rank": 2, "tokenId": 10809, "text": "Block", "score": -0.7390907406806946 }, { "rank": 3, "tokenId": 8021, "text": "allow", "score": -6.044711589813232 }, { "rank": 4, "tokenId": 27122, "text": "Allow", "score": -8.523998260498047 }, { "rank": 5, "tokenId": 124174, "text": "ブロック", "score": -9.37172794342041 }, { "rank": 6, "tokenId": 70961, "text": " ब्लॉक", "score": -10.253907203674316 }, { "rank": 7, "tokenId": 61836, "text": "BLOCK", "score": -10.432937622070312 }, { "rank": 8, "tokenId": 3355, "text": " block", "score": -11.310222625732422 } ] }, "f32_reverse": { "winnerTokenId": 5088, "winnerText": "block", "winnerScore": 7.5311055183410645, "winnerGap": 8.270194232463837, "top": [ { "rank": 1, "tokenId": 5088, "text": "block", "score": 7.5311055183410645 }, { "rank": 2, "tokenId": 10809, "text": "Block", "score": -0.7390887141227722 }, { "rank": 3, "tokenId": 8021, "text": "allow", "score": -6.044707298278809 }, { "rank": 4, "tokenId": 27122, "text": "Allow", "score": -8.52401065826416 }, { "rank": 5, "tokenId": 124174, "text": "ブロック", "score": -9.371723175048828 }, { "rank": 6, "tokenId": 70961, "text": " ब्लॉक", "score": -10.253923416137695 }, { "rank": 7, "tokenId": 61836, "text": "BLOCK", "score": -10.432920455932617 }, { "rank": 8, "tokenId": 3355, "text": " block", "score": -11.310213088989258 } ] }, "f32_tree64": { "winnerTokenId": 5088, "winnerText": "block", "winnerScore": 7.531108856201172, "winnerGap": 8.270198583602905, "top": [ { "rank": 1, "tokenId": 5088, "text": "block", "score": 7.531108856201172 }, { "rank": 2, "tokenId": 10809, "text": "Block", "score": -0.7390897274017334 }, { "rank": 3, "tokenId": 8021, "text": "allow", "score": -6.044707298278809 }, { "rank": 4, "tokenId": 27122, "text": "Allow", "score": -8.52400016784668 }, { "rank": 5, "tokenId": 124174, "text": "ブロック", "score": -9.371726989746094 }, { "rank": 6, "tokenId": 70961, "text": " ब्लॉक", "score": -10.253911972045898 }, { "rank": 7, "tokenId": 61836, "text": "BLOCK", "score": -10.432929992675781 }, { "rank": 8, "tokenId": 3355, "text": " block", "score": -11.310222625732422 } ] }, "f16_forward": { "winnerTokenId": 5088, "winnerText": "block", "winnerScore": 7.53125, "winnerGap": 12.57421875, "top": [ { "rank": 1, "tokenId": 5088, "text": "block", "score": 7.53125 }, { "rank": 2, "tokenId": 10809, "text": "Block", "score": -5.04296875 }, { "rank": 3, "tokenId": 8021, "text": "allow", "score": -6.03515625 }, { "rank": 4, "tokenId": 56201, "text": "allows", "score": -7.609375 }, { "rank": 5, "tokenId": 27122, "text": "Allow", "score": -8.5546875 }, { "rank": 6, "tokenId": 124174, "text": "ブロック", "score": -9.4296875 }, { "rank": 7, "tokenId": 70961, "text": " ब्लॉक", "score": -10.2578125 }, { "rank": 8, "tokenId": 61836, "text": "BLOCK", "score": -10.4140625 } ] } }, "winnerTextsByMode": { "exact": "block", "f32_forward": "block", "f32_reverse": "block", "f32_tree64": "block", "f16_forward": "block" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "block" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "masterpiece-yes-no-contested-oneword-b", "text": "Respond with one word only.\nYes, no, or contested: a masterpiece can be immoral", "stableTopK": [ { "rank": 1, "tokenId": 1584, "text": "Cont", "logit": 18.90180015563965, "prob": 0.858832061290741 }, { "rank": 2, "tokenId": 1848, "text": "cont", "logit": 17.087127685546875, "prob": 0.1398962289094925 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "logit": 12.221489906311035, "prob": 0.0010781676974147558 }, { "rank": 4, "tokenId": 3771, "text": "No", "logit": 9.741861343383789, "prob": 0.0000903227919479832 }, { "rank": 5, "tokenId": 140272, "text": "contest", "logit": 9.480810165405273, "prob": 0.00006957036384847015 }, { "rank": 6, "tokenId": 4443, "text": "yes", "logit": 7.025800704956055, "prob": 0.000005973477072984679 }, { "rank": 7, "tokenId": 69626, "text": "Possible", "logit": 6.273370265960693, "prob": 0.0000028148213004897116 }, { "rank": 8, "tokenId": 6700, "text": "Am", "logit": 5.505339622497559, "prob": 0.000001305868181589176 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 1584, "winnerText": "Cont", "winnerScore": 22.245466733677617, "winnerGap": 2.8388465811152486, "top": [ { "rank": 1, "tokenId": 1584, "text": "Cont", "score": 22.245466733677617 }, { "rank": 2, "tokenId": 1848, "text": "cont", "score": 19.40662015256237 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 12.974086047740983 }, { "rank": 4, "tokenId": 3771, "text": "No", "score": 10.10773115110263 }, { "rank": 5, "tokenId": 140272, "text": "contest", "score": 9.81680987635815 }, { "rank": 6, "tokenId": 4443, "text": "yes", "score": 7.158645351328801 }, { "rank": 7, "tokenId": 69626, "text": "Possible", "score": 6.36728366209789 }, { "rank": 8, "tokenId": 6700, "text": "Am", "score": 5.568416627598506 } ] }, "f32_forward": { "winnerTokenId": 1584, "winnerText": "Cont", "winnerScore": 22.2454833984375, "winnerGap": 2.8388671875, "top": [ { "rank": 1, "tokenId": 1584, "text": "Cont", "score": 22.2454833984375 }, { "rank": 2, "tokenId": 1848, "text": "cont", "score": 19.4066162109375 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 12.97408390045166 }, { "rank": 4, "tokenId": 3771, "text": "No", "score": 10.107739448547363 }, { "rank": 5, "tokenId": 140272, "text": "contest", "score": 9.816816329956055 }, { "rank": 6, "tokenId": 4443, "text": "yes", "score": 7.158661842346191 }, { "rank": 7, "tokenId": 69626, "text": "Possible", "score": 6.367289066314697 }, { "rank": 8, "tokenId": 6700, "text": "Am", "score": 5.568411827087402 } ] }, "f32_reverse": { "winnerTokenId": 1584, "winnerText": "Cont", "winnerScore": 22.2454776763916, "winnerGap": 2.8388423919677734, "top": [ { "rank": 1, "tokenId": 1584, "text": "Cont", "score": 22.2454776763916 }, { "rank": 2, "tokenId": 1848, "text": "cont", "score": 19.406635284423828 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 12.974092483520508 }, { "rank": 4, "tokenId": 3771, "text": "No", "score": 10.107731819152832 }, { "rank": 5, "tokenId": 140272, "text": "contest", "score": 9.816810607910156 }, { "rank": 6, "tokenId": 4443, "text": "yes", "score": 7.1586432456970215 }, { "rank": 7, "tokenId": 69626, "text": "Possible", "score": 6.367287635803223 }, { "rank": 8, "tokenId": 6700, "text": "Am", "score": 5.568419456481934 } ] }, "f32_tree64": { "winnerTokenId": 1584, "winnerText": "Cont", "winnerScore": 22.245468139648438, "winnerGap": 2.838846206665039, "top": [ { "rank": 1, "tokenId": 1584, "text": "Cont", "score": 22.245468139648438 }, { "rank": 2, "tokenId": 1848, "text": "cont", "score": 19.4066219329834 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 12.97408676147461 }, { "rank": 4, "tokenId": 3771, "text": "No", "score": 10.107731819152832 }, { "rank": 5, "tokenId": 140272, "text": "contest", "score": 9.816810607910156 }, { "rank": 6, "tokenId": 4443, "text": "yes", "score": 7.1586456298828125 }, { "rank": 7, "tokenId": 69626, "text": "Possible", "score": 6.367282390594482 }, { "rank": 8, "tokenId": 6700, "text": "Am", "score": 5.568417072296143 } ] }, "f16_forward": { "winnerTokenId": 1584, "winnerText": "Cont", "winnerScore": 22.140625, "winnerGap": 2.78125, "top": [ { "rank": 1, "tokenId": 1584, "text": "Cont", "score": 22.140625 }, { "rank": 2, "tokenId": 1848, "text": "cont", "score": 19.359375 }, { "rank": 3, "tokenId": 140272, "text": "contest", "score": 9.9453125 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": 8.9609375 }, { "rank": 5, "tokenId": 4443, "text": "yes", "score": 7.22265625 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": 6.12890625 }, { "rank": 7, "tokenId": 6700, "text": "Am", "score": 5.6171875 }, { "rank": 8, "tokenId": 7084, "text": "Open", "score": 5.40625 } ] } }, "winnerTextsByMode": { "exact": "Cont", "f32_forward": "Cont", "f32_reverse": "Cont", "f32_tree64": "Cont", "f16_forward": "Cont" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Cont" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "math-one-plus-one-oneword-c", "text": "No explanation. One word only.\n1 + 1 =", "stableTopK": [ { "rank": 1, "tokenId": 11634, "text": "Two", "logit": 12.068686485290527, "prob": 0.9994345307350159 }, { "rank": 2, "tokenId": 236778, "text": "2", "logit": 4.551098346710205, "prob": 0.0005431344034150243 }, { "rank": 3, "tokenId": 236795, "text": "$", "logit": 0.4781089723110199, "prob": 0.000009247631169273518 }, { "rank": 4, "tokenId": 166056, "text": "Deux", "logit": -0.02909325622022152, "prob": 0.000005568719643633813 }, { "rank": 5, "tokenId": 13498, "text": "two", "logit": -0.49535706639289856, "prob": 0.000003493490567052504 }, { "rank": 6, "tokenId": 237725, "text": "二", "logit": -1.3947972059249878, "prob": 0.0000014211426559995743 }, { "rank": 7, "tokenId": 106, "text": "", "logit": -2.3818652629852295, "prob": 5.296139988786308e-7 }, { "rank": 8, "tokenId": 3292, "text": "$\\", "logit": -2.605886459350586, "prob": 4.2331953409302514e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 11634, "winnerText": "Two", "winnerScore": 12.79132691875131, "winnerGap": 8.204826799776455, "top": [ { "rank": 1, "tokenId": 11634, "text": "Two", "score": 12.79132691875131 }, { "rank": 2, "tokenId": 236778, "text": "2", "score": 4.586500118974854 }, { "rank": 3, "tokenId": 236795, "text": "$", "score": 0.47814892211791804 }, { "rank": 4, "tokenId": 166056, "text": "Deux", "score": -0.029090169367736785 }, { "rank": 5, "tokenId": 13498, "text": "two", "score": -0.4953951005228845 }, { "rank": 6, "tokenId": 237725, "text": "二", "score": -1.3958019812842224 }, { "rank": 7, "tokenId": 106, "text": "", "score": -2.386892662697882 }, { "rank": 8, "tokenId": 3292, "text": "$\\", "score": -2.612471741041496 } ] }, "f32_forward": { "winnerTokenId": 11634, "winnerText": "Two", "winnerScore": 12.791314125061035, "winnerGap": 8.20481824874878, "top": [ { "rank": 1, "tokenId": 11634, "text": "Two", "score": 12.791314125061035 }, { "rank": 2, "tokenId": 236778, "text": "2", "score": 4.586495876312256 }, { "rank": 3, "tokenId": 236795, "text": "$", "score": 0.47815027832984924 }, { "rank": 4, "tokenId": 166056, "text": "Deux", "score": -0.029087834060192108 }, { "rank": 5, "tokenId": 13498, "text": "two", "score": -0.49539613723754883 }, { "rank": 6, "tokenId": 237725, "text": "二", "score": -1.395802617073059 }, { "rank": 7, "tokenId": 106, "text": "", "score": -2.386894702911377 }, { "rank": 8, "tokenId": 3292, "text": "$\\", "score": -2.61246919631958 } ] }, "f32_reverse": { "winnerTokenId": 11634, "winnerText": "Two", "winnerScore": 12.7913179397583, "winnerGap": 8.204817771911621, "top": [ { "rank": 1, "tokenId": 11634, "text": "Two", "score": 12.7913179397583 }, { "rank": 2, "tokenId": 236778, "text": "2", "score": 4.58650016784668 }, { "rank": 3, "tokenId": 236795, "text": "$", "score": 0.4781501591205597 }, { "rank": 4, "tokenId": 166056, "text": "Deux", "score": -0.02909011021256447 }, { "rank": 5, "tokenId": 13498, "text": "two", "score": -0.49539482593536377 }, { "rank": 6, "tokenId": 237725, "text": "二", "score": -1.3958008289337158 }, { "rank": 7, "tokenId": 106, "text": "", "score": -2.3868916034698486 }, { "rank": 8, "tokenId": 3292, "text": "$\\", "score": -2.612473964691162 } ] }, "f32_tree64": { "winnerTokenId": 11634, "winnerText": "Two", "winnerScore": 12.791326522827148, "winnerGap": 8.204826831817627, "top": [ { "rank": 1, "tokenId": 11634, "text": "Two", "score": 12.791326522827148 }, { "rank": 2, "tokenId": 236778, "text": "2", "score": 4.5864996910095215 }, { "rank": 3, "tokenId": 236795, "text": "$", "score": 0.4781484603881836 }, { "rank": 4, "tokenId": 166056, "text": "Deux", "score": -0.029090404510498047 }, { "rank": 5, "tokenId": 13498, "text": "two", "score": -0.4953956604003906 }, { "rank": 6, "tokenId": 237725, "text": "二", "score": -1.3958015441894531 }, { "rank": 7, "tokenId": 106, "text": "", "score": -2.386892557144165 }, { "rank": 8, "tokenId": 3292, "text": "$\\", "score": -2.6124722957611084 } ] }, "f16_forward": { "winnerTokenId": 11634, "winnerText": "Two", "winnerScore": 8.578125, "winnerGap": 3.9453125, "top": [ { "rank": 1, "tokenId": 11634, "text": "Two", "score": 8.578125 }, { "rank": 2, "tokenId": 236778, "text": "2", "score": 4.6328125 }, { "rank": 3, "tokenId": 166056, "text": "Deux", "score": -0.07110595703125 }, { "rank": 4, "tokenId": 13498, "text": "two", "score": -0.453857421875 }, { "rank": 5, "tokenId": 236795, "text": "$", "score": -0.51171875 }, { "rank": 6, "tokenId": 134008, "text": "TWO", "score": -1.1640625 }, { "rank": 7, "tokenId": 237725, "text": "二", "score": -1.412109375 }, { "rank": 8, "tokenId": 8512, "text": " Two", "score": -1.564453125 } ] } }, "winnerTextsByMode": { "exact": "Two", "f32_forward": "Two", "f32_reverse": "Two", "f32_tree64": "Two", "f16_forward": "Two" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Two" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "medical-public-private-oneword-d", "text": "Choose one word only.\nA patient's medical record should be public or private. It should be", "stableTopK": [ { "rank": 1, "tokenId": 7128, "text": "private", "logit": 16.72978973388672, "prob": 0.542540967464447 }, { "rank": 2, "tokenId": 29335, "text": "Private", "logit": 15.886890411376953, "prob": 0.2335425466299057 }, { "rank": 3, "tokenId": 1018, "text": "**", "logit": 15.834759712219238, "prob": 0.2216797024011612 }, { "rank": 4, "tokenId": 15928, "text": "Conf", "logit": 10.466185569763184, "prob": 0.0010331986704841256 }, { "rank": 5, "tokenId": 10331, "text": "conf", "logit": 10.396147727966309, "prob": 0.0009633115842007101 }, { "rank": 6, "tokenId": 91082, "text": "PRIVATE", "logit": 8.02587604522705, "prob": 0.00009002660226542503 }, { "rank": 7, "tokenId": 52581, "text": "Privacy", "logit": 6.909182548522949, "prob": 0.000029471146262949333 }, { "rank": 8, "tokenId": 52420, "text": "Patient", "logit": 6.0465087890625, "prob": 0.000012437772056728136 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 7128, "winnerText": "private", "winnerScore": 18.882901632728128, "winnerGap": 1.1967719180783405, "top": [ { "rank": 1, "tokenId": 7128, "text": "private", "score": 18.882901632728128 }, { "rank": 2, "tokenId": 29335, "text": "Private", "score": 17.686129714649788 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 17.613776409047265 }, { "rank": 4, "tokenId": 15928, "text": "Conf", "score": 10.924804034516633 }, { "rank": 5, "tokenId": 10331, "text": "conf", "score": 10.845124592093299 }, { "rank": 6, "tokenId": 91082, "text": "PRIVATE", "score": 8.226020571102033 }, { "rank": 7, "tokenId": 52581, "text": "Privacy", "score": 7.035378349392971 }, { "rank": 8, "tokenId": 52420, "text": "Patient", "score": 6.130440791917323 } ] }, "f32_forward": { "winnerTokenId": 7128, "winnerText": "private", "winnerScore": 18.88290786743164, "winnerGap": 1.1967697143554688, "top": [ { "rank": 1, "tokenId": 7128, "text": "private", "score": 18.88290786743164 }, { "rank": 2, "tokenId": 29335, "text": "Private", "score": 17.686138153076172 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 17.613773345947266 }, { "rank": 4, "tokenId": 15928, "text": "Conf", "score": 10.924809455871582 }, { "rank": 5, "tokenId": 10331, "text": "conf", "score": 10.845129013061523 }, { "rank": 6, "tokenId": 91082, "text": "PRIVATE", "score": 8.226035118103027 }, { "rank": 7, "tokenId": 52581, "text": "Privacy", "score": 7.035374641418457 }, { "rank": 8, "tokenId": 52420, "text": "Patient", "score": 6.130443096160889 } ] }, "f32_reverse": { "winnerTokenId": 7128, "winnerText": "private", "winnerScore": 18.882890701293945, "winnerGap": 1.1967601776123047, "top": [ { "rank": 1, "tokenId": 7128, "text": "private", "score": 18.882890701293945 }, { "rank": 2, "tokenId": 29335, "text": "Private", "score": 17.68613052368164 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 17.61376953125 }, { "rank": 4, "tokenId": 15928, "text": "Conf", "score": 10.9248046875 }, { "rank": 5, "tokenId": 10331, "text": "conf", "score": 10.845124244689941 }, { "rank": 6, "tokenId": 91082, "text": "PRIVATE", "score": 8.226019859313965 }, { "rank": 7, "tokenId": 52581, "text": "Privacy", "score": 7.03537654876709 }, { "rank": 8, "tokenId": 52420, "text": "Patient", "score": 6.1304402351379395 } ] }, "f32_tree64": { "winnerTokenId": 7128, "winnerText": "private", "winnerScore": 18.88290023803711, "winnerGap": 1.1967697143554688, "top": [ { "rank": 1, "tokenId": 7128, "text": "private", "score": 18.88290023803711 }, { "rank": 2, "tokenId": 29335, "text": "Private", "score": 17.68613052368164 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 17.61377716064453 }, { "rank": 4, "tokenId": 15928, "text": "Conf", "score": 10.9248046875 }, { "rank": 5, "tokenId": 10331, "text": "conf", "score": 10.845125198364258 }, { "rank": 6, "tokenId": 91082, "text": "PRIVATE", "score": 8.226019859313965 }, { "rank": 7, "tokenId": 52581, "text": "Privacy", "score": 7.035379886627197 }, { "rank": 8, "tokenId": 52420, "text": "Patient", "score": 6.130441665649414 } ] }, "f16_forward": { "winnerTokenId": 7128, "winnerText": "private", "winnerScore": 19.015625, "winnerGap": 1.234375, "top": [ { "rank": 1, "tokenId": 7128, "text": "private", "score": 19.015625 }, { "rank": 2, "tokenId": 29335, "text": "Private", "score": 17.78125 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 16.640625 }, { "rank": 4, "tokenId": 10331, "text": "conf", "score": 11.1328125 }, { "rank": 5, "tokenId": 15928, "text": "Conf", "score": 10.921875 }, { "rank": 6, "tokenId": 52581, "text": "Privacy", "score": 7.06640625 }, { "rank": 7, "tokenId": 51274, "text": "Medical", "score": 6.12109375 }, { "rank": 8, "tokenId": 52420, "text": "Patient", "score": 6.06640625 } ] } }, "winnerTextsByMode": { "exact": "private", "f32_forward": "private", "f32_reverse": "private", "f32_tree64": "private", "f16_forward": "private" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "private" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "medical-public-private-choice-oneword-a", "text": "Answer with exactly one word only.\nChoose exactly one word: public or private.\nA patient's medical record should remain", "stableTopK": [ { "rank": 1, "tokenId": 7128, "text": "private", "logit": 5.758550643920898, "prob": 0.9999691843986511 }, { "rank": 2, "tokenId": 2153, "text": "public", "logit": -4.900664329528809, "prob": 0.00002348271846130956 }, { "rank": 3, "tokenId": 29335, "text": "Private", "logit": -6.265419006347656, "prob": 0.000005998504093440715 }, { "rank": 4, "tokenId": 10331, "text": "conf", "logit": -8.152435302734375, "prob": 9.089126820072124e-7 }, { "rank": 5, "tokenId": 25641, "text": "patient", "logit": -10.208562850952148, "prob": 1.1629401086565849e-7 }, { "rank": 6, "tokenId": 1018, "text": "**", "logit": -10.834418296813965, "prob": 6.219446646582583e-8 }, { "rank": 7, "tokenId": 29568, "text": "priv", "logit": -11.654979705810547, "prob": 2.7377039302223238e-8 }, { "rank": 8, "tokenId": 91082, "text": "PRIVATE", "logit": -11.882745742797852, "prob": 2.1800625304990717e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 7128, "winnerText": "private", "winnerScore": 5.830881182594382, "winnerGap": 10.775846718966227, "top": [ { "rank": 1, "tokenId": 7128, "text": "private", "score": 5.830881182594382 }, { "rank": 2, "tokenId": 2153, "text": "public", "score": -4.9449655363718445 }, { "rank": 3, "tokenId": 29335, "text": "Private", "score": -6.3589727273474494 }, { "rank": 4, "tokenId": 10331, "text": "conf", "score": -8.362501921636934 }, { "rank": 5, "tokenId": 25641, "text": "patient", "score": -10.632451144030476 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -11.34614077321266 }, { "rank": 7, "tokenId": 29568, "text": "priv", "score": -12.300936985274667 }, { "rank": 8, "tokenId": 91082, "text": "PRIVATE", "score": -12.570164688685708 } ] }, "f32_forward": { "winnerTokenId": 7128, "winnerText": "private", "winnerScore": 5.830885410308838, "winnerGap": 10.775845527648926, "top": [ { "rank": 1, "tokenId": 7128, "text": "private", "score": 5.830885410308838 }, { "rank": 2, "tokenId": 2153, "text": "public", "score": -4.944960117340088 }, { "rank": 3, "tokenId": 29335, "text": "Private", "score": -6.358981132507324 }, { "rank": 4, "tokenId": 10331, "text": "conf", "score": -8.362506866455078 }, { "rank": 5, "tokenId": 25641, "text": "patient", "score": -10.6324462890625 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -11.346144676208496 }, { "rank": 7, "tokenId": 29568, "text": "priv", "score": -12.300936698913574 }, { "rank": 8, "tokenId": 91082, "text": "PRIVATE", "score": -12.57015323638916 } ] }, "f32_reverse": { "winnerTokenId": 7128, "winnerText": "private", "winnerScore": 5.83088493347168, "winnerGap": 10.775853157043457, "top": [ { "rank": 1, "tokenId": 7128, "text": "private", "score": 5.83088493347168 }, { "rank": 2, "tokenId": 2153, "text": "public", "score": -4.944968223571777 }, { "rank": 3, "tokenId": 29335, "text": "Private", "score": -6.358964920043945 }, { "rank": 4, "tokenId": 10331, "text": "conf", "score": -8.362506866455078 }, { "rank": 5, "tokenId": 25641, "text": "patient", "score": -10.63244915008545 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -11.346149444580078 }, { "rank": 7, "tokenId": 29568, "text": "priv", "score": -12.300946235656738 }, { "rank": 8, "tokenId": 91082, "text": "PRIVATE", "score": -12.570160865783691 } ] }, "f32_tree64": { "winnerTokenId": 7128, "winnerText": "private", "winnerScore": 5.830880165100098, "winnerGap": 10.775845527648926, "top": [ { "rank": 1, "tokenId": 7128, "text": "private", "score": 5.830880165100098 }, { "rank": 2, "tokenId": 2153, "text": "public", "score": -4.944965362548828 }, { "rank": 3, "tokenId": 29335, "text": "Private", "score": -6.358971118927002 }, { "rank": 4, "tokenId": 10331, "text": "conf", "score": -8.36250114440918 }, { "rank": 5, "tokenId": 25641, "text": "patient", "score": -10.632452011108398 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -11.346139907836914 }, { "rank": 7, "tokenId": 29568, "text": "priv", "score": -12.30093765258789 }, { "rank": 8, "tokenId": 91082, "text": "PRIVATE", "score": -12.570162773132324 } ] }, "f16_forward": { "winnerTokenId": 7128, "winnerText": "private", "winnerScore": 5.85546875, "winnerGap": 10.8046875, "top": [ { "rank": 1, "tokenId": 7128, "text": "private", "score": 5.85546875 }, { "rank": 2, "tokenId": 2153, "text": "public", "score": -4.94921875 }, { "rank": 3, "tokenId": 29335, "text": "Private", "score": -6.38671875 }, { "rank": 4, "tokenId": 25641, "text": "patient", "score": -6.59765625 }, { "rank": 5, "tokenId": 236865, "text": "#", "score": -8.515625 }, { "rank": 6, "tokenId": 10331, "text": "conf", "score": -8.578125 }, { "rank": 7, "tokenId": 2318, "text": "\\[", "score": -9.109375 }, { "rank": 8, "tokenId": 16602, "text": "health", "score": -9.2421875 } ] } }, "winnerTextsByMode": { "exact": "private", "f32_forward": "private", "f32_reverse": "private", "f32_tree64": "private", "f16_forward": "private" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "private" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "memories-yes-no-unclear-oneword-b", "text": "Respond with one word only.\nYes, no, or unclear: if your memories are copied perfectly into another body, you survive", "stableTopK": [ { "rank": 1, "tokenId": 2805, "text": "Un", "logit": 13.535883903503418, "prob": 0.96974116563797 }, { "rank": 2, "tokenId": 602, "text": "un", "logit": 9.909904479980469, "prob": 0.025817427784204483 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "logit": 7.714859962463379, "prob": 0.002874863799661398 }, { "rank": 4, "tokenId": 3771, "text": "No", "logit": 6.117172718048096, "prob": 0.0005817689234390855 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "logit": 5.5745086669921875, "prob": 0.0003381228307262063 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "logit": 4.869178771972656, "prob": 0.00016701428103260696 }, { "rank": 7, "tokenId": 22631, "text": "Clear", "logit": 4.652632713317871, "prob": 0.00013449584366753697 }, { "rank": 8, "tokenId": 21288, "text": "Subject", "logit": 2.6884090900421143, "prob": 0.000018865024685510434 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 14.586036371858613, "winnerGap": 4.290075994350852, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 14.586036371858613 }, { "rank": 2, "tokenId": 602, "text": "un", "score": 10.295960377507761 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 7.892014803197137 }, { "rank": 4, "tokenId": 3771, "text": "No", "score": 6.204132196029185 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 5.640028326319815 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": 4.912624414135809 }, { "rank": 7, "tokenId": 22631, "text": "Clear", "score": 4.690481216971849 }, { "rank": 8, "tokenId": 21288, "text": "Subject", "score": 2.6956420187982992 } ] }, "f32_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 14.586041450500488, "winnerGap": 4.290083885192871, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 14.586041450500488 }, { "rank": 2, "tokenId": 602, "text": "un", "score": 10.295957565307617 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 7.892021179199219 }, { "rank": 4, "tokenId": 3771, "text": "No", "score": 6.2041401863098145 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 5.640028476715088 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": 4.912631988525391 }, { "rank": 7, "tokenId": 22631, "text": "Clear", "score": 4.690484046936035 }, { "rank": 8, "tokenId": 21288, "text": "Subject", "score": 2.695647716522217 } ] }, "f32_reverse": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 14.58603286743164, "winnerGap": 4.290081024169922, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 14.58603286743164 }, { "rank": 2, "tokenId": 602, "text": "un", "score": 10.295951843261719 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 7.892012119293213 }, { "rank": 4, "tokenId": 3771, "text": "No", "score": 6.20413064956665 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 5.64002799987793 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": 4.912622451782227 }, { "rank": 7, "tokenId": 22631, "text": "Clear", "score": 4.690484046936035 }, { "rank": 8, "tokenId": 21288, "text": "Subject", "score": 2.695634603500366 } ] }, "f32_tree64": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 14.586036682128906, "winnerGap": 4.290075302124023, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 14.586036682128906 }, { "rank": 2, "tokenId": 602, "text": "un", "score": 10.295961380004883 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 7.892014980316162 }, { "rank": 4, "tokenId": 3771, "text": "No", "score": 6.204133033752441 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 5.640028953552246 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": 4.912623405456543 }, { "rank": 7, "tokenId": 22631, "text": "Clear", "score": 4.690481185913086 }, { "rank": 8, "tokenId": 21288, "text": "Subject", "score": 2.6956417560577393 } ] }, "f16_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 14.484375, "winnerGap": 4.25, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 14.484375 }, { "rank": 2, "tokenId": 602, "text": "un", "score": 10.234375 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 7.90234375 }, { "rank": 4, "tokenId": 3771, "text": "No", "score": 6.09765625 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 5.671875 }, { "rank": 6, "tokenId": 22631, "text": "Clear", "score": 4.73828125 }, { "rank": 7, "tokenId": 31410, "text": " unclear", "score": 2.220703125 }, { "rank": 8, "tokenId": 14977, "text": "Question", "score": 2.064453125 } ] } }, "winnerTextsByMode": { "exact": "Un", "f32_forward": "Un", "f32_reverse": "Un", "f32_tree64": "Un", "f16_forward": "Un" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Un" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "mercy-cruelty-both-oneword-c", "text": "No explanation. One word only.\nMercy, cruelty, or both: sedating a dying animal is", "stableTopK": [ { "rank": 1, "tokenId": 236788, "text": "E", "logit": 13.807656288146973, "prob": 0.9466965198516846 }, { "rank": 2, "tokenId": 201053, "text": "Ethics", "logit": 10.38254165649414, "prob": 0.030810760334134102 }, { "rank": 3, "tokenId": 31728, "text": "Eth", "logit": 9.39914321899414, "prob": 0.011524387635290623 }, { "rank": 4, "tokenId": 91899, "text": "Merc", "logit": 8.247836112976074, "prob": 0.0036442785058170557 }, { "rank": 5, "tokenId": 71869, "text": "Cru", "logit": 7.798874378204346, "prob": 0.00232610828243196 }, { "rank": 6, "tokenId": 168154, "text": "Compass", "logit": 7.745566368103027, "prob": 0.0022053553257137537 }, { "rank": 7, "tokenId": 236791, "text": "P", "logit": 6.363778114318848, "prob": 0.0005538288387469947 }, { "rank": 8, "tokenId": 236774, "text": "T", "logit": 6.346890449523926, "prob": 0.0005445544957183301 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 236788, "winnerText": "E", "winnerScore": 14.929053654240242, "winnerGap": 4.099381795008478, "top": [ { "rank": 1, "tokenId": 236788, "text": "E", "score": 14.929053654240242 }, { "rank": 2, "tokenId": 201053, "text": "Ethics", "score": 10.829671859231764 }, { "rank": 3, "tokenId": 31728, "text": "Eth", "score": 9.726172169830125 }, { "rank": 4, "tokenId": 91899, "text": "Merc", "score": 8.46560599626359 }, { "rank": 5, "tokenId": 71869, "text": "Cru", "score": 7.982042362633194 }, { "rank": 6, "tokenId": 168154, "text": "Compass", "score": 7.924900774763174 }, { "rank": 7, "tokenId": 236791, "text": "P", "score": 6.4618915216625314 }, { "rank": 8, "tokenId": 236774, "text": "T", "score": 6.444210189438529 } ] }, "f32_forward": { "winnerTokenId": 236788, "winnerText": "E", "winnerScore": 14.929059982299805, "winnerGap": 4.099401473999023, "top": [ { "rank": 1, "tokenId": 236788, "text": "E", "score": 14.929059982299805 }, { "rank": 2, "tokenId": 201053, "text": "Ethics", "score": 10.829658508300781 }, { "rank": 3, "tokenId": 31728, "text": "Eth", "score": 9.726174354553223 }, { "rank": 4, "tokenId": 91899, "text": "Merc", "score": 8.46561050415039 }, { "rank": 5, "tokenId": 71869, "text": "Cru", "score": 7.982039928436279 }, { "rank": 6, "tokenId": 168154, "text": "Compass", "score": 7.924893856048584 }, { "rank": 7, "tokenId": 236791, "text": "P", "score": 6.46189546585083 }, { "rank": 8, "tokenId": 236774, "text": "T", "score": 6.444226264953613 } ] }, "f32_reverse": { "winnerTokenId": 236788, "winnerText": "E", "winnerScore": 14.92906379699707, "winnerGap": 4.099391937255859, "top": [ { "rank": 1, "tokenId": 236788, "text": "E", "score": 14.92906379699707 }, { "rank": 2, "tokenId": 201053, "text": "Ethics", "score": 10.829671859741211 }, { "rank": 3, "tokenId": 31728, "text": "Eth", "score": 9.726166725158691 }, { "rank": 4, "tokenId": 91899, "text": "Merc", "score": 8.46560001373291 }, { "rank": 5, "tokenId": 71869, "text": "Cru", "score": 7.982044219970703 }, { "rank": 6, "tokenId": 168154, "text": "Compass", "score": 7.924901008605957 }, { "rank": 7, "tokenId": 236791, "text": "P", "score": 6.461894512176514 }, { "rank": 8, "tokenId": 236774, "text": "T", "score": 6.444212913513184 } ] }, "f32_tree64": { "winnerTokenId": 236788, "winnerText": "E", "winnerScore": 14.92905330657959, "winnerGap": 4.099381446838379, "top": [ { "rank": 1, "tokenId": 236788, "text": "E", "score": 14.92905330657959 }, { "rank": 2, "tokenId": 201053, "text": "Ethics", "score": 10.829671859741211 }, { "rank": 3, "tokenId": 31728, "text": "Eth", "score": 9.726171493530273 }, { "rank": 4, "tokenId": 91899, "text": "Merc", "score": 8.465605735778809 }, { "rank": 5, "tokenId": 71869, "text": "Cru", "score": 7.982043266296387 }, { "rank": 6, "tokenId": 168154, "text": "Compass", "score": 7.924900054931641 }, { "rank": 7, "tokenId": 236791, "text": "P", "score": 6.461892127990723 }, { "rank": 8, "tokenId": 236774, "text": "T", "score": 6.444210052490234 } ] }, "f16_forward": { "winnerTokenId": 236788, "winnerText": "E", "winnerScore": 15, "winnerGap": 5.2578125, "top": [ { "rank": 1, "tokenId": 236788, "text": "E", "score": 15 }, { "rank": 2, "tokenId": 31728, "text": "Eth", "score": 9.7421875 }, { "rank": 3, "tokenId": 71869, "text": "Cru", "score": 8.078125 }, { "rank": 4, "tokenId": 201053, "text": "Ethics", "score": 6.77734375 }, { "rank": 5, "tokenId": 236774, "text": "T", "score": 6.52734375 }, { "rank": 6, "tokenId": 236791, "text": "P", "score": 6.5 }, { "rank": 7, "tokenId": 2035, "text": "Tr", "score": 5.80078125 }, { "rank": 8, "tokenId": 236744, "text": "e", "score": 5.3203125 } ] } }, "winnerTextsByMode": { "exact": "E", "f32_forward": "E", "f32_reverse": "E", "f32_tree64": "E", "f16_forward": "E" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "E" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "mercy-cruelty-explicit-oneword-d", "text": "Choose one word only.\nAnswer with exactly one word: mercy or cruelty. Question: Sedating a dying animal is Answer:", "stableTopK": [ { "rank": 1, "tokenId": 1042, "text": "mer", "logit": 21.07633399963379, "prob": 0.9722557663917542 }, { "rank": 2, "tokenId": 91899, "text": "Merc", "logit": 17.516681671142578, "prob": 0.027659427374601364 }, { "rank": 3, "tokenId": 60032, "text": "merc", "logit": 10.780150413513184, "prob": 0.0000328250062011648 }, { "rank": 4, "tokenId": 73633, "text": "cru", "logit": 10.612071990966797, "prob": 0.00002774656968540512 }, { "rank": 5, "tokenId": 71869, "text": "Cru", "logit": 9.602476119995117, "prob": 0.00001010991218208801 }, { "rank": 6, "tokenId": 40474, "text": " mercy", "logit": 8.740706443786621, "prob": 0.000004270567387720803 }, { "rank": 7, "tokenId": 17639, "text": "Mer", "logit": 8.435559272766113, "prob": 0.0000031474717161472654 }, { "rank": 8, "tokenId": 1018, "text": "**", "logit": 7.232456207275391, "prob": 9.450631068830262e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 1042, "winnerText": "mer", "winnerScore": 26.169222722506312, "winnerGap": 6.118906813485108, "top": [ { "rank": 1, "tokenId": 1042, "text": "mer", "score": 26.169222722506312 }, { "rank": 2, "tokenId": 91899, "text": "Merc", "score": 20.050315909021204 }, { "rank": 3, "tokenId": 60032, "text": "merc", "score": 11.283780390344578 }, { "rank": 4, "tokenId": 73633, "text": "cru", "score": 11.091222025637105 }, { "rank": 5, "tokenId": 71869, "text": "Cru", "score": 9.952174021187957 }, { "rank": 6, "tokenId": 40474, "text": " mercy", "score": 9.001438924512378 }, { "rank": 7, "tokenId": 17639, "text": "Mer", "score": 8.66905930515907 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": 7.377673383718168 } ] }, "f32_forward": { "winnerTokenId": 1042, "winnerText": "mer", "winnerScore": 26.16925048828125, "winnerGap": 6.118919372558594, "top": [ { "rank": 1, "tokenId": 1042, "text": "mer", "score": 26.16925048828125 }, { "rank": 2, "tokenId": 91899, "text": "Merc", "score": 20.050331115722656 }, { "rank": 3, "tokenId": 60032, "text": "merc", "score": 11.283784866333008 }, { "rank": 4, "tokenId": 73633, "text": "cru", "score": 11.091218948364258 }, { "rank": 5, "tokenId": 71869, "text": "Cru", "score": 9.952176094055176 }, { "rank": 6, "tokenId": 40474, "text": " mercy", "score": 9.001443862915039 }, { "rank": 7, "tokenId": 17639, "text": "Mer", "score": 8.669058799743652 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": 7.3776679039001465 } ] }, "f32_reverse": { "winnerTokenId": 1042, "winnerText": "mer", "winnerScore": 26.16922378540039, "winnerGap": 6.118894577026367, "top": [ { "rank": 1, "tokenId": 1042, "text": "mer", "score": 26.16922378540039 }, { "rank": 2, "tokenId": 91899, "text": "Merc", "score": 20.050329208374023 }, { "rank": 3, "tokenId": 60032, "text": "merc", "score": 11.28378677368164 }, { "rank": 4, "tokenId": 73633, "text": "cru", "score": 11.091224670410156 }, { "rank": 5, "tokenId": 71869, "text": "Cru", "score": 9.95217227935791 }, { "rank": 6, "tokenId": 40474, "text": " mercy", "score": 9.001440048217773 }, { "rank": 7, "tokenId": 17639, "text": "Mer", "score": 8.669059753417969 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": 7.377681255340576 } ] }, "f32_tree64": { "winnerTokenId": 1042, "winnerText": "mer", "winnerScore": 26.169219970703125, "winnerGap": 6.118904113769531, "top": [ { "rank": 1, "tokenId": 1042, "text": "mer", "score": 26.169219970703125 }, { "rank": 2, "tokenId": 91899, "text": "Merc", "score": 20.050315856933594 }, { "rank": 3, "tokenId": 60032, "text": "merc", "score": 11.283781051635742 }, { "rank": 4, "tokenId": 73633, "text": "cru", "score": 11.09122085571289 }, { "rank": 5, "tokenId": 71869, "text": "Cru", "score": 9.95217514038086 }, { "rank": 6, "tokenId": 40474, "text": " mercy", "score": 9.00143814086914 }, { "rank": 7, "tokenId": 17639, "text": "Mer", "score": 8.669057846069336 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": 7.377671718597412 } ] }, "f16_forward": { "winnerTokenId": 1042, "winnerText": "mer", "winnerScore": 26.375, "winnerGap": 6.140625, "top": [ { "rank": 1, "tokenId": 1042, "text": "mer", "score": 26.375 }, { "rank": 2, "tokenId": 91899, "text": "Merc", "score": 20.234375 }, { "rank": 3, "tokenId": 60032, "text": "merc", "score": 11.3671875 }, { "rank": 4, "tokenId": 73633, "text": "cru", "score": 11.09375 }, { "rank": 5, "tokenId": 40474, "text": " mercy", "score": 9.015625 }, { "rank": 6, "tokenId": 17639, "text": "Mer", "score": 8.8125 }, { "rank": 7, "tokenId": 1808, "text": "hes", "score": 7.24609375 }, { "rank": 8, "tokenId": 14727, "text": "kind", "score": 6.9375 } ] } }, "winnerTextsByMode": { "exact": "mer", "f32_forward": "mer", "f32_reverse": "mer", "f32_tree64": "mer", "f16_forward": "mer" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "mer" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "methanol-safe-unsafe-oneword-a", "text": "Answer with exactly one word only.\nDrinking methanol is safe or unsafe. It is", "stableTopK": [ { "rank": 1, "tokenId": 2805, "text": "Un", "logit": 3.3953945636749268, "prob": 0.9999915957450867 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "logit": -8.3182954788208, "prob": 0.000008180981240002438 }, { "rank": 3, "tokenId": 38699, "text": "Una", "logit": -12.86439037322998, "prob": 8.678835428099774e-8 }, { "rank": 4, "tokenId": 236536, "text": "Dangerous", "logit": -13.422564506530762, "prob": 4.966489441926569e-8 }, { "rank": 5, "tokenId": 46308, "text": "Une", "logit": -14.57194995880127, "prob": 1.5735398761762553e-8 }, { "rank": 6, "tokenId": 236836, "text": "U", "logit": -14.767036437988281, "prob": 1.2946511418476803e-8 }, { "rank": 7, "tokenId": 1018, "text": "**", "logit": -15.796948432922363, "prob": 4.622401572618173e-9 }, { "rank": 8, "tokenId": 26546, "text": "Very", "logit": -16.071962356567383, "prob": 3.510998602962445e-9 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 3.4100012928257684, "winnerGap": 11.951879876969187, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 3.4100012928257684 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": -8.541878584143419 }, { "rank": 3, "tokenId": 38699, "text": "Una", "score": -13.753233792213578 }, { "rank": 4, "tokenId": 236536, "text": "Dangerous", "score": -14.444049757801132 }, { "rank": 5, "tokenId": 46308, "text": "Une", "score": -15.913759674983766 }, { "rank": 6, "tokenId": 236836, "text": "U", "score": -16.17015590120323 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -17.561404447293683 }, { "rank": 8, "tokenId": 26546, "text": "Very", "score": -17.94450859926873 } ] }, "f32_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 3.410007953643799, "winnerGap": 11.951870441436768, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 3.410007953643799 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": -8.541862487792969 }, { "rank": 3, "tokenId": 38699, "text": "Una", "score": -13.753230094909668 }, { "rank": 4, "tokenId": 236536, "text": "Dangerous", "score": -14.444045066833496 }, { "rank": 5, "tokenId": 46308, "text": "Une", "score": -15.91374683380127 }, { "rank": 6, "tokenId": 236836, "text": "U", "score": -16.17015838623047 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -17.561420440673828 }, { "rank": 8, "tokenId": 26546, "text": "Very", "score": -17.94450569152832 } ] }, "f32_reverse": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 3.410004138946533, "winnerGap": 11.95188856124878, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 3.410004138946533 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": -8.541884422302246 }, { "rank": 3, "tokenId": 38699, "text": "Una", "score": -13.753230094909668 }, { "rank": 4, "tokenId": 236536, "text": "Dangerous", "score": -14.444062232971191 }, { "rank": 5, "tokenId": 46308, "text": "Une", "score": -15.913761138916016 }, { "rank": 6, "tokenId": 236836, "text": "U", "score": -16.170162200927734 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -17.561416625976562 }, { "rank": 8, "tokenId": 26546, "text": "Very", "score": -17.944528579711914 } ] }, "f32_tree64": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 3.4100003242492676, "winnerGap": 11.951878070831299, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 3.4100003242492676 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": -8.541877746582031 }, { "rank": 3, "tokenId": 38699, "text": "Una", "score": -13.753232955932617 }, { "rank": 4, "tokenId": 236536, "text": "Dangerous", "score": -14.444049835205078 }, { "rank": 5, "tokenId": 46308, "text": "Une", "score": -15.913759231567383 }, { "rank": 6, "tokenId": 236836, "text": "U", "score": -16.170156478881836 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -17.56140899658203 }, { "rank": 8, "tokenId": 26546, "text": "Very", "score": -17.944507598876953 } ] }, "f16_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 3.4921875, "winnerGap": 12.046875, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 3.4921875 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": -8.5546875 }, { "rank": 3, "tokenId": 38699, "text": "Una", "score": -9.703125 }, { "rank": 4, "tokenId": 236536, "text": "Dangerous", "score": -14.4296875 }, { "rank": 5, "tokenId": 46308, "text": "Une", "score": -14.90625 }, { "rank": 6, "tokenId": 236836, "text": "U", "score": -16.1875 }, { "rank": 7, "tokenId": 188, "text": "
", "score": -16.25 }, { "rank": 8, "tokenId": 236792, "text": "M", "score": -16.359375 } ] } }, "winnerTextsByMode": { "exact": "Un", "f32_forward": "Un", "f32_reverse": "Un", "f32_tree64": "Un", "f16_forward": "Un" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Un" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "methanol-safe-unsafe-choice-oneword-b", "text": "Respond with one word only.\nChoose exactly one word: safe or unsafe.\nDrinking methanol is", "stableTopK": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "logit": 18.9012451171875, "prob": 0.9895439743995667 }, { "rank": 2, "tokenId": 2805, "text": "Un", "logit": 14.350379943847656, "prob": 0.010447670705616474 }, { "rank": 3, "tokenId": 602, "text": "un", "logit": 6.65982723236084, "prob": 0.0000047758958316990174 }, { "rank": 4, "tokenId": 8069, "text": "uns", "logit": 6.360676288604736, "prob": 0.00000354107601197029 }, { "rank": 5, "tokenId": 1018, "text": "**", "logit": 0.43489664793014526, "prob": 9.453700933192977e-9 }, { "rank": 6, "tokenId": 25793, "text": " unsafe", "logit": 0.04750676825642586, "prob": 6.4174217051515825e-9 }, { "rank": 7, "tokenId": 38699, "text": "Una", "logit": -0.9783888459205627, "prob": 2.300486912076849e-9 }, { "rank": 8, "tokenId": 46308, "text": "Une", "logit": -1.3944931030273438, "prob": 1.517427627995005e-9 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 22.244550338343316, "winnerGap": 6.619430005116925, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 22.244550338343316 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 15.625120333226391 }, { "rank": 3, "tokenId": 602, "text": "un", "score": 6.772582095887088 }, { "rank": 4, "tokenId": 8069, "text": "uns", "score": 6.458645715730398 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 0.4349256298284345 }, { "rank": 6, "tokenId": 25793, "text": " unsafe", "score": 0.04751215460464664 }, { "rank": 7, "tokenId": 38699, "text": "Una", "score": -0.9787345593411949 }, { "rank": 8, "tokenId": 46308, "text": "Une", "score": -1.3954937741328273 } ] }, "f32_forward": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 22.244556427001953, "winnerGap": 6.619442939758301, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 22.244556427001953 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 15.625113487243652 }, { "rank": 3, "tokenId": 602, "text": "un", "score": 6.772573947906494 }, { "rank": 4, "tokenId": 8069, "text": "uns", "score": 6.458649158477783 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 0.4349238872528076 }, { "rank": 6, "tokenId": 25793, "text": " unsafe", "score": 0.04751265421509743 }, { "rank": 7, "tokenId": 38699, "text": "Una", "score": -0.9787337779998779 }, { "rank": 8, "tokenId": 46308, "text": "Une", "score": -1.3954920768737793 } ] }, "f32_reverse": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 22.24456787109375, "winnerGap": 6.619448661804199, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 22.24456787109375 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 15.62511920928955 }, { "rank": 3, "tokenId": 602, "text": "un", "score": 6.772583961486816 }, { "rank": 4, "tokenId": 8069, "text": "uns", "score": 6.458641052246094 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 0.43492114543914795 }, { "rank": 6, "tokenId": 25793, "text": " unsafe", "score": 0.04751317948102951 }, { "rank": 7, "tokenId": 38699, "text": "Una", "score": -0.978736400604248 }, { "rank": 8, "tokenId": 46308, "text": "Une", "score": -1.3954957723617554 } ] }, "f32_tree64": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 22.244552612304688, "winnerGap": 6.61943244934082, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 22.244552612304688 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 15.625120162963867 }, { "rank": 3, "tokenId": 602, "text": "un", "score": 6.772582530975342 }, { "rank": 4, "tokenId": 8069, "text": "uns", "score": 6.458646297454834 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 0.43492722511291504 }, { "rank": 6, "tokenId": 25793, "text": " unsafe", "score": 0.04751014709472656 }, { "rank": 7, "tokenId": 38699, "text": "Una", "score": -0.9787354469299316 }, { "rank": 8, "tokenId": 46308, "text": "Une", "score": -1.3954923152923584 } ] }, "f16_forward": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 22.453125, "winnerGap": 6.7265625, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 22.453125 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 15.7265625 }, { "rank": 3, "tokenId": 602, "text": "un", "score": 6.76953125 }, { "rank": 4, "tokenId": 8069, "text": "uns", "score": 6.47265625 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 0.4501953125 }, { "rank": 6, "tokenId": 25793, "text": " unsafe", "score": 0.08251953125 }, { "rank": 7, "tokenId": 38699, "text": "Una", "score": -0.94287109375 }, { "rank": 8, "tokenId": 46308, "text": "Une", "score": -1.341796875 } ] } }, "winnerTextsByMode": { "exact": "unsafe", "f32_forward": "unsafe", "f32_reverse": "unsafe", "f32_tree64": "unsafe", "f16_forward": "unsafe" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "unsafe" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "na-diamonds-lab-oneword-c", "text": "No explanation. One word only.\nAnswer with exactly one word: natural or artificial. Question: A diamond grown in a laboratory is Answer:", "stableTopK": [ { "rank": 1, "tokenId": 141316, "text": "artificial", "logit": 20.77895164489746, "prob": 0.9925692081451416 }, { "rank": 2, "tokenId": 118870, "text": "Artificial", "logit": 15.883371353149414, "prob": 0.007423988543450832 }, { "rank": 3, "tokenId": 661, "text": "art", "logit": 8.723072052001953, "prob": 0.000005767117727373261 }, { "rank": 4, "tokenId": 23014, "text": "natural", "logit": 6.628944396972656, "prob": 7.103799930519017e-7 }, { "rank": 5, "tokenId": 9175, "text": "Art", "logit": 5.419210910797119, "prob": 2.118898549952064e-7 }, { "rank": 6, "tokenId": 16477, "text": " artificial", "logit": 3.208763599395752, "prob": 2.3234060009258428e-8 }, { "rank": 7, "tokenId": 1018, "text": "**", "logit": 2.8026463985443115, "prob": 1.547927475087363e-8 }, { "rank": 8, "tokenId": 39905, "text": "Natural", "logit": 2.4022445678710938, "prob": 1.0371900671657386e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 141316, "winnerText": "artificial", "winnerScore": 25.58989390391733, "winnerGap": 7.908651513967037, "top": [ { "rank": 1, "tokenId": 141316, "text": "artificial", "score": 25.58989390391733 }, { "rank": 2, "tokenId": 118870, "text": "Artificial", "score": 17.68124238995029 }, { "rank": 3, "tokenId": 661, "text": "art", "score": 8.982178259247071 }, { "rank": 4, "tokenId": 23014, "text": "natural", "score": 6.740103750784311 }, { "rank": 5, "tokenId": 9175, "text": "Art", "score": 5.4793376900019695 }, { "rank": 6, "tokenId": 16477, "text": " artificial", "score": 3.221084962407744 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": 2.810841047051957 }, { "rank": 8, "tokenId": 39905, "text": "Natural", "score": 2.40739558671358 } ] }, "f32_forward": { "winnerTokenId": 141316, "winnerText": "artificial", "winnerScore": 25.589866638183594, "winnerGap": 7.9086151123046875, "top": [ { "rank": 1, "tokenId": 141316, "text": "artificial", "score": 25.589866638183594 }, { "rank": 2, "tokenId": 118870, "text": "Artificial", "score": 17.681251525878906 }, { "rank": 3, "tokenId": 661, "text": "art", "score": 8.982178688049316 }, { "rank": 4, "tokenId": 23014, "text": "natural", "score": 6.7401041984558105 }, { "rank": 5, "tokenId": 9175, "text": "Art", "score": 5.479343414306641 }, { "rank": 6, "tokenId": 16477, "text": " artificial", "score": 3.2210865020751953 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": 2.8108420372009277 }, { "rank": 8, "tokenId": 39905, "text": "Natural", "score": 2.4073972702026367 } ] }, "f32_reverse": { "winnerTokenId": 141316, "winnerText": "artificial", "winnerScore": 25.58988380432129, "winnerGap": 7.908641815185547, "top": [ { "rank": 1, "tokenId": 141316, "text": "artificial", "score": 25.58988380432129 }, { "rank": 2, "tokenId": 118870, "text": "Artificial", "score": 17.681241989135742 }, { "rank": 3, "tokenId": 661, "text": "art", "score": 8.982178688049316 }, { "rank": 4, "tokenId": 23014, "text": "natural", "score": 6.7401041984558105 }, { "rank": 5, "tokenId": 9175, "text": "Art", "score": 5.479349613189697 }, { "rank": 6, "tokenId": 16477, "text": " artificial", "score": 3.2210803031921387 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": 2.810844659805298 }, { "rank": 8, "tokenId": 39905, "text": "Natural", "score": 2.4073963165283203 } ] }, "f32_tree64": { "winnerTokenId": 141316, "winnerText": "artificial", "winnerScore": 25.589893341064453, "winnerGap": 7.908653259277344, "top": [ { "rank": 1, "tokenId": 141316, "text": "artificial", "score": 25.589893341064453 }, { "rank": 2, "tokenId": 118870, "text": "Artificial", "score": 17.68124008178711 }, { "rank": 3, "tokenId": 661, "text": "art", "score": 8.982175827026367 }, { "rank": 4, "tokenId": 23014, "text": "natural", "score": 6.740102767944336 }, { "rank": 5, "tokenId": 9175, "text": "Art", "score": 5.479337692260742 }, { "rank": 6, "tokenId": 16477, "text": " artificial", "score": 3.221086025238037 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": 2.8108420372009277 }, { "rank": 8, "tokenId": 39905, "text": "Natural", "score": 2.407397985458374 } ] }, "f16_forward": { "winnerTokenId": 141316, "winnerText": "artificial", "winnerScore": 25.890625, "winnerGap": 8.234375, "top": [ { "rank": 1, "tokenId": 141316, "text": "artificial", "score": 25.890625 }, { "rank": 2, "tokenId": 118870, "text": "Artificial", "score": 17.65625 }, { "rank": 3, "tokenId": 661, "text": "art", "score": 9.1015625 }, { "rank": 4, "tokenId": 23014, "text": "natural", "score": 6.7890625 }, { "rank": 5, "tokenId": 9175, "text": "Art", "score": 5.53515625 }, { "rank": 6, "tokenId": 16477, "text": " artificial", "score": 3.228515625 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": 2.849609375 }, { "rank": 8, "tokenId": 39905, "text": "Natural", "score": 2.283203125 } ] } }, "winnerTextsByMode": { "exact": "artificial", "f32_forward": "artificial", "f32_reverse": "artificial", "f32_tree64": "artificial", "f16_forward": "artificial" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "artificial" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "name-color-oneword-d", "text": "Choose one word only.\nName a color:", "stableTopK": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "logit": 8.961247444152832, "prob": 0.9971720576286316 }, { "rank": 2, "tokenId": 9264, "text": "Red", "logit": 3.0891904830932617, "prob": 0.0028091061394661665 }, { "rank": 3, "tokenId": 1018, "text": "**", "logit": -1.9228296279907227, "prob": 0.000018701459339354187 }, { "rank": 4, "tokenId": 9503, "text": "blue", "logit": -7.865029335021973, "prob": 4.9114635913838356e-8 }, { "rank": 5, "tokenId": 37154, "text": "Yellow", "logit": -7.972296237945557, "prob": 4.411898757439303e-8 }, { "rank": 6, "tokenId": 149876, "text": "蓝色", "logit": -9.406778335571289, "prob": 1.0510850856348952e-8 }, { "rank": 7, "tokenId": 236829, "text": "*", "logit": -9.793505668640137, "prob": 7.139771440733966e-9 }, { "rank": 8, "tokenId": 151790, "text": "Crim", "logit": -10.8179292678833, "prob": 2.5632018729737638e-9 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 9.243022780183338, "winnerGap": 6.142842565034442, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 9.243022780183338 }, { "rank": 2, "tokenId": 9264, "text": "Red", "score": 3.100180215148896 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": -1.9254677108088742 }, { "rank": 4, "tokenId": 9503, "text": "blue", "score": -8.053042008834682 }, { "rank": 5, "tokenId": 37154, "text": "Yellow", "score": -8.168334319099216 }, { "rank": 6, "tokenId": 149876, "text": "蓝色", "score": -9.734636459739377 }, { "rank": 7, "tokenId": 236829, "text": "*", "score": -10.165494051554871 }, { "rank": 8, "tokenId": 151790, "text": "Crim", "score": -11.327173616799968 } ] }, "f32_forward": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 9.243013381958008, "winnerGap": 6.142838478088379, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 9.243013381958008 }, { "rank": 2, "tokenId": 9264, "text": "Red", "score": 3.100174903869629 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": -1.9254631996154785 }, { "rank": 4, "tokenId": 9503, "text": "blue", "score": -8.053049087524414 }, { "rank": 5, "tokenId": 37154, "text": "Yellow", "score": -8.168340682983398 }, { "rank": 6, "tokenId": 149876, "text": "蓝色", "score": -9.734627723693848 }, { "rank": 7, "tokenId": 236829, "text": "*", "score": -10.165482521057129 }, { "rank": 8, "tokenId": 151790, "text": "Crim", "score": -11.327181816101074 } ] }, "f32_reverse": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 9.243022918701172, "winnerGap": 6.142843961715698, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 9.243022918701172 }, { "rank": 2, "tokenId": 9264, "text": "Red", "score": 3.1001789569854736 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": -1.9254677295684814 }, { "rank": 4, "tokenId": 9503, "text": "blue", "score": -8.053038597106934 }, { "rank": 5, "tokenId": 37154, "text": "Yellow", "score": -8.168330192565918 }, { "rank": 6, "tokenId": 149876, "text": "蓝色", "score": -9.734643936157227 }, { "rank": 7, "tokenId": 236829, "text": "*", "score": -10.165498733520508 }, { "rank": 8, "tokenId": 151790, "text": "Crim", "score": -11.327152252197266 } ] }, "f32_tree64": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 9.243022918701172, "winnerGap": 6.142843008041382, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 9.243022918701172 }, { "rank": 2, "tokenId": 9264, "text": "Red", "score": 3.10017991065979 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": -1.925466537475586 }, { "rank": 4, "tokenId": 9503, "text": "blue", "score": -8.053043365478516 }, { "rank": 5, "tokenId": 37154, "text": "Yellow", "score": -8.1683349609375 }, { "rank": 6, "tokenId": 149876, "text": "蓝色", "score": -9.734636306762695 }, { "rank": 7, "tokenId": 236829, "text": "*", "score": -10.165494918823242 }, { "rank": 8, "tokenId": 151790, "text": "Crim", "score": -11.32717514038086 } ] }, "f16_forward": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 5.34765625, "winnerGap": 2.265625, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 5.34765625 }, { "rank": 2, "tokenId": 9264, "text": "Red", "score": 3.08203125 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": -1.908203125 }, { "rank": 4, "tokenId": 236829, "text": "*", "score": -6.203125 }, { "rank": 5, "tokenId": 236865, "text": "#", "score": -7.93359375 }, { "rank": 6, "tokenId": 9503, "text": "blue", "score": -7.97265625 }, { "rank": 7, "tokenId": 37154, "text": "Yellow", "score": -8.21875 }, { "rank": 8, "tokenId": 149876, "text": "蓝色", "score": -9.828125 } ] } }, "winnerTextsByMode": { "exact": "Blue", "f32_forward": "Blue", "f32_reverse": "Blue", "f32_tree64": "Blue", "f16_forward": "Blue" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Blue" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "natural-artificial-explicit-oneword-a", "text": "Answer with exactly one word only.\nAnswer with exactly one word: natural or artificial. Question: Lab-grown meat is Answer:", "stableTopK": [ { "rank": 1, "tokenId": 141316, "text": "artificial", "logit": 11.710471153259277, "prob": 0.9816314578056335 }, { "rank": 2, "tokenId": 118870, "text": "Artificial", "logit": 7.709706783294678, "prob": 0.017965469509363174 }, { "rank": 3, "tokenId": 23014, "text": "natural", "logit": 3.8556721210479736, "prob": 0.00038076110649853945 }, { "rank": 4, "tokenId": 39905, "text": "Natural", "logit": 0.4356335401535034, "prob": 0.000012455143405531999 }, { "rank": 5, "tokenId": 661, "text": "art", "logit": -0.28469255566596985, "prob": 0.000006060592568246648 }, { "rank": 6, "tokenId": 9175, "text": "Art", "logit": -1.8316349983215332, "prob": 0.0000012902875141662662 }, { "rank": 7, "tokenId": 27413, "text": "both", "logit": -2.5177955627441406, "prob": 6.496669016087253e-7 }, { "rank": 8, "tokenId": 16477, "text": " artificial", "logit": -3.719078302383423, "prob": 1.9542505924619036e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 141316, "winnerText": "artificial", "winnerScore": 12.366355938946121, "winnerGap": 4.4798602315566285, "top": [ { "rank": 1, "tokenId": 141316, "text": "artificial", "score": 12.366355938946121 }, { "rank": 2, "tokenId": 118870, "text": "Artificial", "score": 7.886495707389493 }, { "rank": 3, "tokenId": 23014, "text": "natural", "score": 3.8771151481462383 }, { "rank": 4, "tokenId": 39905, "text": "Natural", "score": 0.4356648913133103 }, { "rank": 5, "tokenId": 661, "text": "art", "score": -0.28469841427897435 }, { "rank": 6, "tokenId": 9175, "text": "Art", "score": -1.8339207809047942 }, { "rank": 7, "tokenId": 27413, "text": "both", "score": -2.523734021488913 }, { "rank": 8, "tokenId": 16477, "text": " artificial", "score": -3.7383188490231145 } ] }, "f32_forward": { "winnerTokenId": 141316, "winnerText": "artificial", "winnerScore": 12.36636734008789, "winnerGap": 4.479875087738037, "top": [ { "rank": 1, "tokenId": 141316, "text": "artificial", "score": 12.36636734008789 }, { "rank": 2, "tokenId": 118870, "text": "Artificial", "score": 7.8864922523498535 }, { "rank": 3, "tokenId": 23014, "text": "natural", "score": 3.877124547958374 }, { "rank": 4, "tokenId": 39905, "text": "Natural", "score": 0.4356604218482971 }, { "rank": 5, "tokenId": 661, "text": "art", "score": -0.28469982743263245 }, { "rank": 6, "tokenId": 9175, "text": "Art", "score": -1.833921194076538 }, { "rank": 7, "tokenId": 27413, "text": "both", "score": -2.523725986480713 }, { "rank": 8, "tokenId": 16477, "text": " artificial", "score": -3.7383158206939697 } ] }, "f32_reverse": { "winnerTokenId": 141316, "winnerText": "artificial", "winnerScore": 12.36635684967041, "winnerGap": 4.479867458343506, "top": [ { "rank": 1, "tokenId": 141316, "text": "artificial", "score": 12.36635684967041 }, { "rank": 2, "tokenId": 118870, "text": "Artificial", "score": 7.886489391326904 }, { "rank": 3, "tokenId": 23014, "text": "natural", "score": 3.8771133422851562 }, { "rank": 4, "tokenId": 39905, "text": "Natural", "score": 0.43565577268600464 }, { "rank": 5, "tokenId": 661, "text": "art", "score": -0.2847006916999817 }, { "rank": 6, "tokenId": 9175, "text": "Art", "score": -1.8339123725891113 }, { "rank": 7, "tokenId": 27413, "text": "both", "score": -2.5237269401550293 }, { "rank": 8, "tokenId": 16477, "text": " artificial", "score": -3.73831844329834 } ] }, "f32_tree64": { "winnerTokenId": 141316, "winnerText": "artificial", "winnerScore": 12.366355895996094, "winnerGap": 4.479858875274658, "top": [ { "rank": 1, "tokenId": 141316, "text": "artificial", "score": 12.366355895996094 }, { "rank": 2, "tokenId": 118870, "text": "Artificial", "score": 7.8864970207214355 }, { "rank": 3, "tokenId": 23014, "text": "natural", "score": 3.8771142959594727 }, { "rank": 4, "tokenId": 39905, "text": "Natural", "score": 0.43566417694091797 }, { "rank": 5, "tokenId": 661, "text": "art", "score": -0.2846989631652832 }, { "rank": 6, "tokenId": 9175, "text": "Art", "score": -1.8339216709136963 }, { "rank": 7, "tokenId": 27413, "text": "both", "score": -2.523733139038086 }, { "rank": 8, "tokenId": 16477, "text": " artificial", "score": -3.7383198738098145 } ] }, "f16_forward": { "winnerTokenId": 141316, "winnerText": "artificial", "winnerScore": 12.3984375, "winnerGap": 4.51953125, "top": [ { "rank": 1, "tokenId": 141316, "text": "artificial", "score": 12.3984375 }, { "rank": 2, "tokenId": 118870, "text": "Artificial", "score": 7.87890625 }, { "rank": 3, "tokenId": 23014, "text": "natural", "score": 4.02734375 }, { "rank": 4, "tokenId": 661, "text": "art", "score": -0.281982421875 }, { "rank": 5, "tokenId": 39905, "text": "Natural", "score": -0.6162109375 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": -2.552734375 }, { "rank": 7, "tokenId": 236746, "text": "a", "score": -3.169921875 }, { "rank": 8, "tokenId": 811, "text": "one", "score": -3.494140625 } ] } }, "winnerTextsByMode": { "exact": "artificial", "f32_forward": "artificial", "f32_reverse": "artificial", "f32_tree64": "artificial", "f16_forward": "artificial" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "artificial" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "newline-oneword-b", "text": "Respond with one word only.\nThe sky is blue\n", "stableTopK": [ { "rank": 1, "tokenId": 4339, "text": "True", "logit": 12.693662643432617, "prob": 0.9941450953483582 }, { "rank": 2, "tokenId": 16520, "text": "Blue", "logit": 7.496328830718994, "prob": 0.005498907063156366 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "logit": 4.4037275314331055, "prob": 0.000249560980591923 }, { "rank": 4, "tokenId": 34318, "text": "Correct", "logit": 3.235837697982788, "prob": 0.00007761909364489838 }, { "rank": 5, "tokenId": 51077, "text": "Indeed", "logit": 0.7624428868293762, "prob": 0.0000065431499933765735 }, { "rank": 6, "tokenId": 5475, "text": "Acc", "logit": 0.5009208917617798, "prob": 0.000005037432856624946 }, { "rank": 7, "tokenId": 74041, "text": "Truth", "logit": 0.4452941119670868, "prob": 0.000004764868208440021 }, { "rank": 8, "tokenId": 3397, "text": "true", "logit": -0.3647055923938751, "prob": 0.00000211969063457218 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.544666046032656, "winnerGap": 5.886197170857454, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.544666046032656 }, { "rank": 2, "tokenId": 16520, "text": "Blue", "score": 7.658468875175203 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 4.435773504113879 }, { "rank": 4, "tokenId": 34318, "text": "Correct", "score": 3.2484778711813433 }, { "rank": 5, "tokenId": 51077, "text": "Indeed", "score": 0.762604832565625 }, { "rank": 6, "tokenId": 5475, "text": "Acc", "score": 0.5009676945380832 }, { "rank": 7, "tokenId": 74041, "text": "Truth", "score": 0.4453286490197712 }, { "rank": 8, "tokenId": 3397, "text": "true", "score": -0.3647245339512182 } ] }, "f32_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.544665336608887, "winnerGap": 5.886195659637451, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.544665336608887 }, { "rank": 2, "tokenId": 16520, "text": "Blue", "score": 7.6584696769714355 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 4.43577766418457 }, { "rank": 4, "tokenId": 34318, "text": "Correct", "score": 3.248479127883911 }, { "rank": 5, "tokenId": 51077, "text": "Indeed", "score": 0.7626049518585205 }, { "rank": 6, "tokenId": 5475, "text": "Acc", "score": 0.500968873500824 }, { "rank": 7, "tokenId": 74041, "text": "Truth", "score": 0.4453280568122864 }, { "rank": 8, "tokenId": 3397, "text": "true", "score": -0.3647250235080719 } ] }, "f32_reverse": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.544661521911621, "winnerGap": 5.886191368103027, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.544661521911621 }, { "rank": 2, "tokenId": 16520, "text": "Blue", "score": 7.658470153808594 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 4.435776710510254 }, { "rank": 4, "tokenId": 34318, "text": "Correct", "score": 3.24847674369812 }, { "rank": 5, "tokenId": 51077, "text": "Indeed", "score": 0.7626049518585205 }, { "rank": 6, "tokenId": 5475, "text": "Acc", "score": 0.5009676218032837 }, { "rank": 7, "tokenId": 74041, "text": "Truth", "score": 0.4453304708003998 }, { "rank": 8, "tokenId": 3397, "text": "true", "score": -0.364724338054657 } ] }, "f32_tree64": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.54466438293457, "winnerGap": 5.886194705963135, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.54466438293457 }, { "rank": 2, "tokenId": 16520, "text": "Blue", "score": 7.6584696769714355 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 4.435772895812988 }, { "rank": 4, "tokenId": 34318, "text": "Correct", "score": 3.248476505279541 }, { "rank": 5, "tokenId": 51077, "text": "Indeed", "score": 0.7626032829284668 }, { "rank": 6, "tokenId": 5475, "text": "Acc", "score": 0.5009679794311523 }, { "rank": 7, "tokenId": 74041, "text": "Truth", "score": 0.4453285336494446 }, { "rank": 8, "tokenId": 3397, "text": "true", "score": -0.3647245168685913 } ] }, "f16_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.515625, "winnerGap": 6.0703125, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.515625 }, { "rank": 2, "tokenId": 16520, "text": "Blue", "score": 7.4453125 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 4.4921875 }, { "rank": 4, "tokenId": 34318, "text": "Correct", "score": 3.263671875 }, { "rank": 5, "tokenId": 51077, "text": "Indeed", "score": 0.8291015625 }, { "rank": 6, "tokenId": 5475, "text": "Acc", "score": 0.53271484375 }, { "rank": 7, "tokenId": 74041, "text": "Truth", "score": 0.45703125 }, { "rank": 8, "tokenId": 3397, "text": "true", "score": -0.32861328125 } ] } }, "winnerTextsByMode": { "exact": "True", "f32_forward": "True", "f32_reverse": "True", "f32_tree64": "True", "f16_forward": "True" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "True" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "one-word-oneword-c", "text": "No explanation. One word only.\nWrite one word:", "stableTopK": [ { "rank": 1, "tokenId": 14713, "text": "Under", "logit": 6.145391464233398, "prob": 0.4735724627971649 }, { "rank": 2, "tokenId": 186741, "text": "Silence", "logit": 5.138437747955322, "prob": 0.17301031947135925 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "logit": 4.686410427093506, "prob": 0.11009282618761063 }, { "rank": 4, "tokenId": 136657, "text": "Silent", "logit": 4.577613353729248, "prob": 0.09874362498521805 }, { "rank": 5, "tokenId": 19058, "text": "Okay", "logit": 3.916276454925537, "prob": 0.050967589020729065 }, { "rank": 6, "tokenId": 34496, "text": "Done", "logit": 3.6286866664886475, "prob": 0.03822922334074974 }, { "rank": 7, "tokenId": 27426, "text": "Begin", "logit": 2.2037456035614014, "prob": 0.009194992482662201 }, { "rank": 8, "tokenId": 34230, "text": "Continue", "logit": 2.1223511695861816, "prob": 0.00847622100263834 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 14713, "winnerText": "Under", "winnerScore": 6.2335789686325285, "winnerGap": 1.0439893891877894, "top": [ { "rank": 1, "tokenId": 14713, "text": "Under", "score": 6.2335789686325285 }, { "rank": 2, "tokenId": 186741, "text": "Silence", "score": 5.189589579444739 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 4.725095059111414 }, { "rank": 4, "tokenId": 136657, "text": "Silent", "score": 4.613643115263855 }, { "rank": 5, "tokenId": 19058, "text": "Okay", "score": 3.9387506674543182 }, { "rank": 6, "tokenId": 34496, "text": "Done", "score": 3.6465423536514034 }, { "rank": 7, "tokenId": 27426, "text": "Begin", "score": 2.207722265777363 }, { "rank": 8, "tokenId": 34230, "text": "Continue", "score": 2.1259054907650183 } ] }, "f32_forward": { "winnerTokenId": 14713, "winnerText": "Under", "winnerScore": 6.233582019805908, "winnerGap": 1.0439958572387695, "top": [ { "rank": 1, "tokenId": 14713, "text": "Under", "score": 6.233582019805908 }, { "rank": 2, "tokenId": 186741, "text": "Silence", "score": 5.189586162567139 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 4.725093841552734 }, { "rank": 4, "tokenId": 136657, "text": "Silent", "score": 4.613640785217285 }, { "rank": 5, "tokenId": 19058, "text": "Okay", "score": 3.938753366470337 }, { "rank": 6, "tokenId": 34496, "text": "Done", "score": 3.64654278755188 }, { "rank": 7, "tokenId": 27426, "text": "Begin", "score": 2.2077245712280273 }, { "rank": 8, "tokenId": 34230, "text": "Continue", "score": 2.1259024143218994 } ] }, "f32_reverse": { "winnerTokenId": 14713, "winnerText": "Under", "winnerScore": 6.233580112457275, "winnerGap": 1.043990135192871, "top": [ { "rank": 1, "tokenId": 14713, "text": "Under", "score": 6.233580112457275 }, { "rank": 2, "tokenId": 186741, "text": "Silence", "score": 5.189589977264404 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 4.725096225738525 }, { "rank": 4, "tokenId": 136657, "text": "Silent", "score": 4.613645553588867 }, { "rank": 5, "tokenId": 19058, "text": "Okay", "score": 3.938750982284546 }, { "rank": 6, "tokenId": 34496, "text": "Done", "score": 3.6465413570404053 }, { "rank": 7, "tokenId": 27426, "text": "Begin", "score": 2.2077219486236572 }, { "rank": 8, "tokenId": 34230, "text": "Continue", "score": 2.125903606414795 } ] }, "f32_tree64": { "winnerTokenId": 14713, "winnerText": "Under", "winnerScore": 6.233578681945801, "winnerGap": 1.0439891815185547, "top": [ { "rank": 1, "tokenId": 14713, "text": "Under", "score": 6.233578681945801 }, { "rank": 2, "tokenId": 186741, "text": "Silence", "score": 5.189589500427246 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 4.725093841552734 }, { "rank": 4, "tokenId": 136657, "text": "Silent", "score": 4.613643646240234 }, { "rank": 5, "tokenId": 19058, "text": "Okay", "score": 3.938751697540283 }, { "rank": 6, "tokenId": 34496, "text": "Done", "score": 3.646543025970459 }, { "rank": 7, "tokenId": 27426, "text": "Begin", "score": 2.2077229022979736 }, { "rank": 8, "tokenId": 34230, "text": "Continue", "score": 2.125906229019165 } ] }, "f16_forward": { "winnerTokenId": 14713, "winnerText": "Under", "winnerScore": 6.234375, "winnerGap": 1.0390625, "top": [ { "rank": 1, "tokenId": 14713, "text": "Under", "score": 6.234375 }, { "rank": 2, "tokenId": 186741, "text": "Silence", "score": 5.1953125 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 4.8046875 }, { "rank": 4, "tokenId": 136657, "text": "Silent", "score": 4.5625 }, { "rank": 5, "tokenId": 19058, "text": "Okay", "score": 3.953125 }, { "rank": 6, "tokenId": 34496, "text": "Done", "score": 3.693359375 }, { "rank": 7, "tokenId": 27426, "text": "Begin", "score": 2.27734375 }, { "rank": 8, "tokenId": 34230, "text": "Continue", "score": 2.146484375 } ] } }, "winnerTextsByMode": { "exact": "Under", "f32_forward": "Under", "f32_reverse": "Under", "f32_tree64": "Under", "f16_forward": "Under" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Under" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "open-breaking-news-oneword-d", "text": "Choose one word only.\nBREAKING:", "stableTopK": [ { "rank": 1, "tokenId": 18626, "text": "News", "logit": 5.582005500793457, "prob": 0.4405887722969055 }, { "rank": 2, "tokenId": 38370, "text": "Ur", "logit": 5.4590864181518555, "prob": 0.38962817192077637 }, { "rank": 3, "tokenId": 2805, "text": "Un", "logit": 2.9165430068969727, "prob": 0.030650515109300613 }, { "rank": 4, "tokenId": 104071, "text": "Breaking", "logit": 2.7766916751861572, "prob": 0.026650238782167435 }, { "rank": 5, "tokenId": 1018, "text": "**", "logit": 2.6789050102233887, "prob": 0.024167565628886223 }, { "rank": 6, "tokenId": 6445, "text": "Now", "logit": 2.2609879970550537, "prob": 0.015912331640720367 }, { "rank": 7, "tokenId": 24852, "text": "Choose", "logit": 1.702654242515564, "prob": 0.009104426018893719 }, { "rank": 8, "tokenId": 106797, "text": "Developing", "logit": 1.574323296546936, "prob": 0.008007909171283245 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 18626, "winnerText": "News", "winnerScore": 5.647792063010325, "winnerGap": 0.12722390191971655, "top": [ { "rank": 1, "tokenId": 18626, "text": "News", "score": 5.647792063010325 }, { "rank": 2, "tokenId": 38370, "text": "Ur", "score": 5.520568161090608 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": 2.925778614138326 }, { "rank": 4, "tokenId": 104071, "text": "Breaking", "score": 2.784658842791101 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 2.6860599450468357 }, { "rank": 6, "tokenId": 6445, "text": "Now", "score": 2.265284194517342 }, { "rank": 7, "tokenId": 24852, "text": "Choose", "score": 1.7044916733884783 }, { "rank": 8, "tokenId": 106797, "text": "Developing", "score": 1.57577283418253 } ] }, "f32_forward": { "winnerTokenId": 18626, "winnerText": "News", "winnerScore": 5.647792339324951, "winnerGap": 0.1272273063659668, "top": [ { "rank": 1, "tokenId": 18626, "text": "News", "score": 5.647792339324951 }, { "rank": 2, "tokenId": 38370, "text": "Ur", "score": 5.520565032958984 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": 2.925783395767212 }, { "rank": 4, "tokenId": 104071, "text": "Breaking", "score": 2.7846593856811523 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 2.6860554218292236 }, { "rank": 6, "tokenId": 6445, "text": "Now", "score": 2.265286922454834 }, { "rank": 7, "tokenId": 24852, "text": "Choose", "score": 1.7044895887374878 }, { "rank": 8, "tokenId": 106797, "text": "Developing", "score": 1.5757746696472168 } ] }, "f32_reverse": { "winnerTokenId": 18626, "winnerText": "News", "winnerScore": 5.647793769836426, "winnerGap": 0.1272263526916504, "top": [ { "rank": 1, "tokenId": 18626, "text": "News", "score": 5.647793769836426 }, { "rank": 2, "tokenId": 38370, "text": "Ur", "score": 5.520567417144775 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": 2.925779342651367 }, { "rank": 4, "tokenId": 104071, "text": "Breaking", "score": 2.7846579551696777 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 2.6860673427581787 }, { "rank": 6, "tokenId": 6445, "text": "Now", "score": 2.2652831077575684 }, { "rank": 7, "tokenId": 24852, "text": "Choose", "score": 1.7044897079467773 }, { "rank": 8, "tokenId": 106797, "text": "Developing", "score": 1.5757743120193481 } ] }, "f32_tree64": { "winnerTokenId": 18626, "winnerText": "News", "winnerScore": 5.647791862487793, "winnerGap": 0.12722396850585938, "top": [ { "rank": 1, "tokenId": 18626, "text": "News", "score": 5.647791862487793 }, { "rank": 2, "tokenId": 38370, "text": "Ur", "score": 5.520567893981934 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": 2.9257781505584717 }, { "rank": 4, "tokenId": 104071, "text": "Breaking", "score": 2.7846591472625732 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 2.686060667037964 }, { "rank": 6, "tokenId": 6445, "text": "Now", "score": 2.265284299850464 }, { "rank": 7, "tokenId": 24852, "text": "Choose", "score": 1.7044916152954102 }, { "rank": 8, "tokenId": 106797, "text": "Developing", "score": 1.575772762298584 } ] }, "f16_forward": { "winnerTokenId": 18626, "winnerText": "News", "winnerScore": 5.6875, "winnerGap": 1.13671875, "top": [ { "rank": 1, "tokenId": 18626, "text": "News", "score": 5.6875 }, { "rank": 2, "tokenId": 38370, "text": "Ur", "score": 4.55078125 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": 2.931640625 }, { "rank": 4, "tokenId": 104071, "text": "Breaking", "score": 2.80859375 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 2.681640625 }, { "rank": 6, "tokenId": 6445, "text": "Now", "score": 2.2734375 }, { "rank": 7, "tokenId": 24852, "text": "Choose", "score": 1.73046875 }, { "rank": 8, "tokenId": 95207, "text": "NEWS", "score": 1.5986328125 } ] } }, "winnerTextsByMode": { "exact": "News", "f32_forward": "News", "f32_reverse": "News", "f32_tree64": "News", "f16_forward": "News" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "News" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "open-dear-sir-oneword-a", "text": "Answer with exactly one word only.\nDear Sir or Madam,", "stableTopK": [ { "rank": 1, "tokenId": 9259, "text": "Hello", "logit": 5.28516149520874, "prob": 0.5203035473823547 }, { "rank": 2, "tokenId": 6126, "text": "Response", "logit": 4.828464031219482, "prob": 0.32954567670822144 }, { "rank": 3, "tokenId": 40654, "text": "Reply", "logit": 2.317758321762085, "prob": 0.02676270343363285 }, { "rank": 4, "tokenId": 39125, "text": "Sir", "logit": 1.723089575767517, "prob": 0.014766194857656956 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "logit": 1.575853943824768, "prob": 0.012744564563035965 }, { "rank": 6, "tokenId": 118904, "text": "Respond", "logit": 1.4472962617874146, "prob": 0.01120709627866745 }, { "rank": 7, "tokenId": 9366, "text": "Please", "logit": 0.9167835116386414, "prob": 0.006593171041458845 }, { "rank": 8, "tokenId": 14782, "text": "Thank", "logit": 0.8986081480979919, "prob": 0.006474420428276062 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 9259, "winnerText": "Hello", "winnerScore": 5.340880582248708, "winnerGap": 0.47006347520136416, "top": [ { "rank": 1, "tokenId": 9259, "text": "Hello", "score": 5.340880582248708 }, { "rank": 2, "tokenId": 6126, "text": "Response", "score": 4.870817107047344 }, { "rank": 3, "tokenId": 40654, "text": "Reply", "score": 2.3223865138763635 }, { "rank": 4, "tokenId": 39125, "text": "Sir", "score": 1.7249890486116612 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 1.5773052118564195 }, { "rank": 6, "tokenId": 118904, "text": "Respond", "score": 1.4484146383996404 }, { "rank": 7, "tokenId": 9366, "text": "Please", "score": 0.9170686566976158 }, { "rank": 8, "tokenId": 14782, "text": "Thank", "score": 0.898877034879078 } ] }, "f32_forward": { "winnerTokenId": 9259, "winnerText": "Hello", "winnerScore": 5.340881824493408, "winnerGap": 0.4700655937194824, "top": [ { "rank": 1, "tokenId": 9259, "text": "Hello", "score": 5.340881824493408 }, { "rank": 2, "tokenId": 6126, "text": "Response", "score": 4.870816230773926 }, { "rank": 3, "tokenId": 40654, "text": "Reply", "score": 2.322382926940918 }, { "rank": 4, "tokenId": 39125, "text": "Sir", "score": 1.7249925136566162 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 1.577303171157837 }, { "rank": 6, "tokenId": 118904, "text": "Respond", "score": 1.4484171867370605 }, { "rank": 7, "tokenId": 9366, "text": "Please", "score": 0.9170669913291931 }, { "rank": 8, "tokenId": 14782, "text": "Thank", "score": 0.8988765478134155 } ] }, "f32_reverse": { "winnerTokenId": 9259, "winnerText": "Hello", "winnerScore": 5.340882301330566, "winnerGap": 0.470059871673584, "top": [ { "rank": 1, "tokenId": 9259, "text": "Hello", "score": 5.340882301330566 }, { "rank": 2, "tokenId": 6126, "text": "Response", "score": 4.870822429656982 }, { "rank": 3, "tokenId": 40654, "text": "Reply", "score": 2.3223860263824463 }, { "rank": 4, "tokenId": 39125, "text": "Sir", "score": 1.72498619556427 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 1.5773048400878906 }, { "rank": 6, "tokenId": 118904, "text": "Respond", "score": 1.4484124183654785 }, { "rank": 7, "tokenId": 9366, "text": "Please", "score": 0.917068600654602 }, { "rank": 8, "tokenId": 14782, "text": "Thank", "score": 0.8988780975341797 } ] }, "f32_tree64": { "winnerTokenId": 9259, "winnerText": "Hello", "winnerScore": 5.34088134765625, "winnerGap": 0.4700641632080078, "top": [ { "rank": 1, "tokenId": 9259, "text": "Hello", "score": 5.34088134765625 }, { "rank": 2, "tokenId": 6126, "text": "Response", "score": 4.870817184448242 }, { "rank": 3, "tokenId": 40654, "text": "Reply", "score": 2.3223862648010254 }, { "rank": 4, "tokenId": 39125, "text": "Sir", "score": 1.7249889373779297 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 1.5773046016693115 }, { "rank": 6, "tokenId": 118904, "text": "Respond", "score": 1.4484140872955322 }, { "rank": 7, "tokenId": 9366, "text": "Please", "score": 0.9170694351196289 }, { "rank": 8, "tokenId": 14782, "text": "Thank", "score": 0.8988765478134155 } ] }, "f16_forward": { "winnerTokenId": 9259, "winnerText": "Hello", "winnerScore": 5.4453125, "winnerGap": 0.51171875, "top": [ { "rank": 1, "tokenId": 9259, "text": "Hello", "score": 5.4453125 }, { "rank": 2, "tokenId": 6126, "text": "Response", "score": 4.93359375 }, { "rank": 3, "tokenId": 40654, "text": "Reply", "score": 2.32421875 }, { "rank": 4, "tokenId": 39125, "text": "Sir", "score": 1.7822265625 }, { "rank": 5, "tokenId": 118904, "text": "Respond", "score": 1.5478515625 }, { "rank": 6, "tokenId": 41387, "text": "Unknown", "score": 1.525390625 }, { "rank": 7, "tokenId": 1988, "text": "Res", "score": 0.95458984375 }, { "rank": 8, "tokenId": 14782, "text": "Thank", "score": 0.94189453125 } ] } }, "winnerTextsByMode": { "exact": "Hello", "f32_forward": "Hello", "f32_reverse": "Hello", "f32_tree64": "Hello", "f16_forward": "Hello" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Hello" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "open-imagine-oneword-b", "text": "Respond with one word only.\nImagine a world where", "stableTopK": [ { "rank": 1, "tokenId": 125133, "text": "Tomorrow", "logit": 2.8851311206817627, "prob": 0.2505944073200226 }, { "rank": 2, "tokenId": 45542, "text": "Poss", "logit": 2.2577109336853027, "prob": 0.13380928337574005 }, { "rank": 3, "tokenId": 236836, "text": "U", "logit": 2.057777166366577, "prob": 0.10956103354692459 }, { "rank": 4, "tokenId": 104884, "text": "Potential", "logit": 1.484519124031067, "prob": 0.06175801157951355 }, { "rank": 5, "tokenId": 8764, "text": "poss", "logit": 0.9875304698944092, "prob": 0.03757109493017197 }, { "rank": 6, "tokenId": 3689, "text": "What", "logit": 0.9083248972892761, "prob": 0.03471005707979202 }, { "rank": 7, "tokenId": 9233, "text": "Change", "logit": 0.7841058373451233, "prob": 0.030655449256300926 }, { "rank": 8, "tokenId": 45946, "text": "Science", "logit": 0.47548604011535645, "prob": 0.022515200078487396 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 125133, "winnerText": "Tomorrow", "winnerScore": 2.894076278216069, "winnerGap": 0.6320909821930201, "top": [ { "rank": 1, "tokenId": 125133, "text": "Tomorrow", "score": 2.894076278216069 }, { "rank": 2, "tokenId": 45542, "text": "Poss", "score": 2.261985296023049 }, { "rank": 3, "tokenId": 236836, "text": "U", "score": 2.06101205964036 }, { "rank": 4, "tokenId": 104884, "text": "Potential", "score": 1.4857324287724083 }, { "rank": 5, "tokenId": 8764, "text": "poss", "score": 0.9878881049756423 }, { "rank": 6, "tokenId": 3689, "text": "What", "score": 0.908604177414162 }, { "rank": 7, "tokenId": 9233, "text": "Change", "score": 0.7842834653379125 }, { "rank": 8, "tokenId": 45946, "text": "Science", "score": 0.4755247153322779 } ] }, "f32_forward": { "winnerTokenId": 125133, "winnerText": "Tomorrow", "winnerScore": 2.89408540725708, "winnerGap": 0.6320948600769043, "top": [ { "rank": 1, "tokenId": 125133, "text": "Tomorrow", "score": 2.89408540725708 }, { "rank": 2, "tokenId": 45542, "text": "Poss", "score": 2.261990547180176 }, { "rank": 3, "tokenId": 236836, "text": "U", "score": 2.061014413833618 }, { "rank": 4, "tokenId": 104884, "text": "Potential", "score": 1.4857301712036133 }, { "rank": 5, "tokenId": 8764, "text": "poss", "score": 0.987885057926178 }, { "rank": 6, "tokenId": 3689, "text": "What", "score": 0.9086061120033264 }, { "rank": 7, "tokenId": 9233, "text": "Change", "score": 0.7842798233032227 }, { "rank": 8, "tokenId": 45946, "text": "Science", "score": 0.47552579641342163 } ] }, "f32_reverse": { "winnerTokenId": 125133, "winnerText": "Tomorrow", "winnerScore": 2.8940768241882324, "winnerGap": 0.6320919990539551, "top": [ { "rank": 1, "tokenId": 125133, "text": "Tomorrow", "score": 2.8940768241882324 }, { "rank": 2, "tokenId": 45542, "text": "Poss", "score": 2.2619848251342773 }, { "rank": 3, "tokenId": 236836, "text": "U", "score": 2.0610108375549316 }, { "rank": 4, "tokenId": 104884, "text": "Potential", "score": 1.485732078552246 }, { "rank": 5, "tokenId": 8764, "text": "poss", "score": 0.9878861308097839 }, { "rank": 6, "tokenId": 3689, "text": "What", "score": 0.90860515832901 }, { "rank": 7, "tokenId": 9233, "text": "Change", "score": 0.7842833995819092 }, { "rank": 8, "tokenId": 45946, "text": "Science", "score": 0.4755280017852783 } ] }, "f32_tree64": { "winnerTokenId": 125133, "winnerText": "Tomorrow", "winnerScore": 2.8940773010253906, "winnerGap": 0.6320915222167969, "top": [ { "rank": 1, "tokenId": 125133, "text": "Tomorrow", "score": 2.8940773010253906 }, { "rank": 2, "tokenId": 45542, "text": "Poss", "score": 2.2619857788085938 }, { "rank": 3, "tokenId": 236836, "text": "U", "score": 2.0610122680664062 }, { "rank": 4, "tokenId": 104884, "text": "Potential", "score": 1.4857325553894043 }, { "rank": 5, "tokenId": 8764, "text": "poss", "score": 0.9878873825073242 }, { "rank": 6, "tokenId": 3689, "text": "What", "score": 0.9086041450500488 }, { "rank": 7, "tokenId": 9233, "text": "Change", "score": 0.7842841148376465 }, { "rank": 8, "tokenId": 45946, "text": "Science", "score": 0.4755239486694336 } ] }, "f16_forward": { "winnerTokenId": 125133, "winnerText": "Tomorrow", "winnerScore": 2.814453125, "winnerGap": 0.42578125, "top": [ { "rank": 1, "tokenId": 125133, "text": "Tomorrow", "score": 2.814453125 }, { "rank": 2, "tokenId": 45542, "text": "Poss", "score": 2.388671875 }, { "rank": 3, "tokenId": 236836, "text": "U", "score": 2.091796875 }, { "rank": 4, "tokenId": 104884, "text": "Potential", "score": 1.5126953125 }, { "rank": 5, "tokenId": 8764, "text": "poss", "score": 0.99365234375 }, { "rank": 6, "tokenId": 3689, "text": "What", "score": 0.93896484375 }, { "rank": 7, "tokenId": 36177, "text": "Future", "score": 0.77294921875 }, { "rank": 8, "tokenId": 9233, "text": "Change", "score": 0.4638671875 } ] } }, "winnerTextsByMode": { "exact": "Tomorrow", "f32_forward": "Tomorrow", "f32_reverse": "Tomorrow", "f32_tree64": "Tomorrow", "f16_forward": "Tomorrow" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Tomorrow" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "open-in-conclusion-oneword-c", "text": "No explanation. One word only.\nIn conclusion,", "stableTopK": [ { "rank": 1, "tokenId": 20416, "text": "Therefore", "logit": 3.8708856105804443, "prob": 0.3725425899028778 }, { "rank": 2, "tokenId": 3810, "text": "There", "logit": 3.321139335632324, "prob": 0.21499291062355042 }, { "rank": 3, "tokenId": 30792, "text": "Conclusion", "logit": 3.1263906955718994, "prob": 0.17694810032844543 }, { "rank": 4, "tokenId": 34496, "text": "Done", "logit": 2.535768747329712, "prob": 0.09802616387605667 }, { "rank": 5, "tokenId": 7315, "text": "End", "logit": 2.2375686168670654, "prob": 0.07275039702653885 }, { "rank": 6, "tokenId": 21204, "text": "Thus", "logit": 0.7225100994110107, "prob": 0.01599019579589367 }, { "rank": 7, "tokenId": 60903, "text": "Finished", "logit": 0.265186071395874, "prob": 0.01012139767408371 }, { "rank": 8, "tokenId": 91607, "text": "Ultimately", "logit": 0.22740858793258667, "prob": 0.009746168740093708 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 20416, "winnerText": "Therefore", "winnerScore": 3.8925795189252455, "winnerGap": 0.5577727962149481, "top": [ { "rank": 1, "tokenId": 20416, "text": "Therefore", "score": 3.8925795189252455 }, { "rank": 2, "tokenId": 3810, "text": "There", "score": 3.3348067227102973 }, { "rank": 3, "tokenId": 30792, "text": "Conclusion", "score": 3.137784179172769 }, { "rank": 4, "tokenId": 34496, "text": "Done", "score": 2.5418310495876746 }, { "rank": 5, "tokenId": 7315, "text": "End", "score": 2.241733735905896 }, { "rank": 6, "tokenId": 21204, "text": "Thus", "score": 0.7226504139957228 }, { "rank": 7, "tokenId": 60903, "text": "Finished", "score": 0.26519551824741683 }, { "rank": 8, "tokenId": 91607, "text": "Ultimately", "score": 0.2274149170425268 } ] }, "f32_forward": { "winnerTokenId": 20416, "winnerText": "Therefore", "winnerScore": 3.892582893371582, "winnerGap": 0.5577723979949951, "top": [ { "rank": 1, "tokenId": 20416, "text": "Therefore", "score": 3.892582893371582 }, { "rank": 2, "tokenId": 3810, "text": "There", "score": 3.334810495376587 }, { "rank": 3, "tokenId": 30792, "text": "Conclusion", "score": 3.1377787590026855 }, { "rank": 4, "tokenId": 34496, "text": "Done", "score": 2.5418295860290527 }, { "rank": 5, "tokenId": 7315, "text": "End", "score": 2.2417304515838623 }, { "rank": 6, "tokenId": 21204, "text": "Thus", "score": 0.7226442694664001 }, { "rank": 7, "tokenId": 60903, "text": "Finished", "score": 0.2651903033256531 }, { "rank": 8, "tokenId": 91607, "text": "Ultimately", "score": 0.2274157553911209 } ] }, "f32_reverse": { "winnerTokenId": 20416, "winnerText": "Therefore", "winnerScore": 3.8925769329071045, "winnerGap": 0.5577719211578369, "top": [ { "rank": 1, "tokenId": 20416, "text": "Therefore", "score": 3.8925769329071045 }, { "rank": 2, "tokenId": 3810, "text": "There", "score": 3.3348050117492676 }, { "rank": 3, "tokenId": 30792, "text": "Conclusion", "score": 3.1377830505371094 }, { "rank": 4, "tokenId": 34496, "text": "Done", "score": 2.5418314933776855 }, { "rank": 5, "tokenId": 7315, "text": "End", "score": 2.241734027862549 }, { "rank": 6, "tokenId": 21204, "text": "Thus", "score": 0.7226441502571106 }, { "rank": 7, "tokenId": 60903, "text": "Finished", "score": 0.26519277691841125 }, { "rank": 8, "tokenId": 91607, "text": "Ultimately", "score": 0.22741572558879852 } ] }, "f32_tree64": { "winnerTokenId": 20416, "winnerText": "Therefore", "winnerScore": 3.8925797939300537, "winnerGap": 0.557772159576416, "top": [ { "rank": 1, "tokenId": 20416, "text": "Therefore", "score": 3.8925797939300537 }, { "rank": 2, "tokenId": 3810, "text": "There", "score": 3.3348076343536377 }, { "rank": 3, "tokenId": 30792, "text": "Conclusion", "score": 3.137784481048584 }, { "rank": 4, "tokenId": 34496, "text": "Done", "score": 2.5418317317962646 }, { "rank": 5, "tokenId": 7315, "text": "End", "score": 2.2417333126068115 }, { "rank": 6, "tokenId": 21204, "text": "Thus", "score": 0.7226512432098389 }, { "rank": 7, "tokenId": 60903, "text": "Finished", "score": 0.26519644260406494 }, { "rank": 8, "tokenId": 91607, "text": "Ultimately", "score": 0.22741401195526123 } ] }, "f16_forward": { "winnerTokenId": 20416, "winnerText": "Therefore", "winnerScore": 3.89453125, "winnerGap": 0.51171875, "top": [ { "rank": 1, "tokenId": 20416, "text": "Therefore", "score": 3.89453125 }, { "rank": 2, "tokenId": 3810, "text": "There", "score": 3.3828125 }, { "rank": 3, "tokenId": 30792, "text": "Conclusion", "score": 3.1953125 }, { "rank": 4, "tokenId": 34496, "text": "Done", "score": 2.56640625 }, { "rank": 5, "tokenId": 7315, "text": "End", "score": 1.25390625 }, { "rank": 6, "tokenId": 21204, "text": "Thus", "score": 0.83935546875 }, { "rank": 7, "tokenId": 60903, "text": "Finished", "score": 0.328125 }, { "rank": 8, "tokenId": 91607, "text": "Ultimately", "score": 0.250244140625 } ] } }, "winnerTextsByMode": { "exact": "Therefore", "f32_forward": "Therefore", "f32_reverse": "Therefore", "f32_tree64": "Therefore", "f16_forward": "Therefore" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Therefore" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "open-the-problem-is-oneword-d", "text": "Choose one word only.\nThe problem is", "stableTopK": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "logit": 8.288745880126953, "prob": 0.5360616445541382 }, { "rank": 2, "tokenId": 1018, "text": "**", "logit": 7.043375015258789, "prob": 0.15429683029651642 }, { "rank": 3, "tokenId": 2704, "text": "Ind", "logit": 6.669158458709717, "prob": 0.10612967610359192 }, { "rank": 4, "tokenId": 115587, "text": "Undefined", "logit": 5.607959747314453, "prob": 0.03672519326210022 }, { "rank": 5, "tokenId": 2805, "text": "Un", "logit": 5.586851596832275, "prob": 0.03595811873674393 }, { "rank": 6, "tokenId": 236847, "text": "V", "logit": 5.499101161956787, "prob": 0.03293725475668907 }, { "rank": 7, "tokenId": 24852, "text": "Choose", "logit": 4.940821647644043, "prob": 0.018846455961465836 }, { "rank": 8, "tokenId": 21009, "text": "Problem", "logit": 4.661199569702148, "prob": 0.014249228872358799 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 8.509879111814044, "winnerGap": 1.332632924028232, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 8.509879111814044 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 7.177246187785812 }, { "rank": 3, "tokenId": 2704, "text": "Ind", "score": 6.782399352858139 }, { "rank": 4, "tokenId": 115587, "text": "Undefined", "score": 5.674685699288101 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": 5.652815266065536 }, { "rank": 6, "tokenId": 236847, "text": "V", "score": 5.56195987633879 }, { "rank": 7, "tokenId": 24852, "text": "Choose", "score": 4.98623369321637 }, { "rank": 8, "tokenId": 21009, "text": "Problem", "score": 4.699262441396463 } ] }, "f32_forward": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 8.509873390197754, "winnerGap": 1.332629680633545, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 8.509873390197754 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 7.177243709564209 }, { "rank": 3, "tokenId": 2704, "text": "Ind", "score": 6.78240442276001 }, { "rank": 4, "tokenId": 115587, "text": "Undefined", "score": 5.674685478210449 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": 5.652818202972412 }, { "rank": 6, "tokenId": 236847, "text": "V", "score": 5.5619587898254395 }, { "rank": 7, "tokenId": 24852, "text": "Choose", "score": 4.986230373382568 }, { "rank": 8, "tokenId": 21009, "text": "Problem", "score": 4.699265003204346 } ] }, "f32_reverse": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 8.509878158569336, "winnerGap": 1.3326315879821777, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 8.509878158569336 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 7.177246570587158 }, { "rank": 3, "tokenId": 2704, "text": "Ind", "score": 6.782398700714111 }, { "rank": 4, "tokenId": 115587, "text": "Undefined", "score": 5.67468786239624 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": 5.6528143882751465 }, { "rank": 6, "tokenId": 236847, "text": "V", "score": 5.5619587898254395 }, { "rank": 7, "tokenId": 24852, "text": "Choose", "score": 4.986234188079834 }, { "rank": 8, "tokenId": 21009, "text": "Problem", "score": 4.699259281158447 } ] }, "f32_tree64": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 8.509878158569336, "winnerGap": 1.332632064819336, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 8.509878158569336 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 7.17724609375 }, { "rank": 3, "tokenId": 2704, "text": "Ind", "score": 6.7823991775512695 }, { "rank": 4, "tokenId": 115587, "text": "Undefined", "score": 5.674686431884766 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": 5.6528143882751465 }, { "rank": 6, "tokenId": 236847, "text": "V", "score": 5.561960220336914 }, { "rank": 7, "tokenId": 24852, "text": "Choose", "score": 4.986233711242676 }, { "rank": 8, "tokenId": 21009, "text": "Problem", "score": 4.699263572692871 } ] }, "f16_forward": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 8.40625, "winnerGap": 1.10546875, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 8.40625 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 7.30078125 }, { "rank": 3, "tokenId": 24852, "text": "Choose", "score": 5.08203125 }, { "rank": 4, "tokenId": 21009, "text": "Problem", "score": 4.71484375 }, { "rank": 5, "tokenId": 30340, "text": "Und", "score": 4.6484375 }, { "rank": 6, "tokenId": 1525, "text": "amb", "score": 4.0625 }, { "rank": 7, "tokenId": 191239, "text": "Difficult", "score": 3.267578125 }, { "rank": 8, "tokenId": 41387, "text": "Unknown", "score": 3.15625 } ] } }, "winnerTextsByMode": { "exact": "Amb", "f32_forward": "Amb", "f32_reverse": "Amb", "f32_tree64": "Amb", "f16_forward": "Amb" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Amb" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "original-derivative-both-oneword-a", "text": "Answer with exactly one word only.\nOriginal, derivative, or both: a transformative remix is", "stableTopK": [ { "rank": 1, "tokenId": 22186, "text": "Both", "logit": 18.8055419921875, "prob": 0.807121753692627 }, { "rank": 2, "tokenId": 27413, "text": "both", "logit": 17.069639205932617, "prob": 0.1422479748725891 }, { "rank": 3, "tokenId": 163971, "text": "Derivative", "logit": 15.801445960998535, "prob": 0.040019966661930084 }, { "rank": 4, "tokenId": 133678, "text": "derivative", "logit": 14.35598373413086, "prob": 0.009430190548300743 }, { "rank": 5, "tokenId": 30498, "text": "Original", "logit": 12.15250015258789, "prob": 0.0010412612464278936 }, { "rank": 6, "tokenId": 10254, "text": "transform", "logit": 8.969862937927246, "prob": 0.00004318748688092455 }, { "rank": 7, "tokenId": 1800, "text": " both", "logit": 8.563772201538086, "prob": 0.000028773649319191463 }, { "rank": 8, "tokenId": 18385, "text": "Transform", "logit": 7.911823749542236, "prob": 0.000014991922398621682 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 22186, "winnerText": "Both", "winnerScore": 22.086385286357135, "winnerGap": 2.705630670756399, "top": [ { "rank": 1, "tokenId": 22186, "text": "Both", "score": 22.086385286357135 }, { "rank": 2, "tokenId": 27413, "text": "both", "score": 19.380754615600736 }, { "rank": 3, "tokenId": 163971, "text": "Derivative", "score": 17.567636281904434 }, { "rank": 4, "tokenId": 133678, "text": "derivative", "score": 15.63238783257713 }, { "rank": 5, "tokenId": 30498, "text": "Original", "score": 12.891460937897941 }, { "rank": 6, "tokenId": 10254, "text": "transform", "score": 9.252483826856446 }, { "rank": 7, "tokenId": 1800, "text": " both", "score": 8.808465217555508 }, { "rank": 8, "tokenId": 18385, "text": "Transform", "score": 8.10330880297164 } ] }, "f32_forward": { "winnerTokenId": 22186, "winnerText": "Both", "winnerScore": 22.086421966552734, "winnerGap": 2.705677032470703, "top": [ { "rank": 1, "tokenId": 22186, "text": "Both", "score": 22.086421966552734 }, { "rank": 2, "tokenId": 27413, "text": "both", "score": 19.38074493408203 }, { "rank": 3, "tokenId": 163971, "text": "Derivative", "score": 17.567651748657227 }, { "rank": 4, "tokenId": 133678, "text": "derivative", "score": 15.63237476348877 }, { "rank": 5, "tokenId": 30498, "text": "Original", "score": 12.891450881958008 }, { "rank": 6, "tokenId": 10254, "text": "transform", "score": 9.25248908996582 }, { "rank": 7, "tokenId": 1800, "text": " both", "score": 8.808464050292969 }, { "rank": 8, "tokenId": 18385, "text": "Transform", "score": 8.103302001953125 } ] }, "f32_reverse": { "winnerTokenId": 22186, "winnerText": "Both", "winnerScore": 22.086387634277344, "winnerGap": 2.705646514892578, "top": [ { "rank": 1, "tokenId": 22186, "text": "Both", "score": 22.086387634277344 }, { "rank": 2, "tokenId": 27413, "text": "both", "score": 19.380741119384766 }, { "rank": 3, "tokenId": 163971, "text": "Derivative", "score": 17.567638397216797 }, { "rank": 4, "tokenId": 133678, "text": "derivative", "score": 15.632386207580566 }, { "rank": 5, "tokenId": 30498, "text": "Original", "score": 12.89145278930664 }, { "rank": 6, "tokenId": 10254, "text": "transform", "score": 9.252484321594238 }, { "rank": 7, "tokenId": 1800, "text": " both", "score": 8.808465957641602 }, { "rank": 8, "tokenId": 18385, "text": "Transform", "score": 8.10330581665039 } ] }, "f32_tree64": { "winnerTokenId": 22186, "winnerText": "Both", "winnerScore": 22.086383819580078, "winnerGap": 2.7056312561035156, "top": [ { "rank": 1, "tokenId": 22186, "text": "Both", "score": 22.086383819580078 }, { "rank": 2, "tokenId": 27413, "text": "both", "score": 19.380752563476562 }, { "rank": 3, "tokenId": 163971, "text": "Derivative", "score": 17.56763458251953 }, { "rank": 4, "tokenId": 133678, "text": "derivative", "score": 15.632387161254883 }, { "rank": 5, "tokenId": 30498, "text": "Original", "score": 12.891461372375488 }, { "rank": 6, "tokenId": 10254, "text": "transform", "score": 9.252482414245605 }, { "rank": 7, "tokenId": 1800, "text": " both", "score": 8.808465957641602 }, { "rank": 8, "tokenId": 18385, "text": "Transform", "score": 8.103309631347656 } ] }, "f16_forward": { "winnerTokenId": 22186, "winnerText": "Both", "winnerScore": 21.953125, "winnerGap": 2.5625, "top": [ { "rank": 1, "tokenId": 22186, "text": "Both", "score": 21.953125 }, { "rank": 2, "tokenId": 27413, "text": "both", "score": 19.390625 }, { "rank": 3, "tokenId": 163971, "text": "Derivative", "score": 17.328125 }, { "rank": 4, "tokenId": 133678, "text": "derivative", "score": 15.53125 }, { "rank": 5, "tokenId": 30498, "text": "Original", "score": 12.96875 }, { "rank": 6, "tokenId": 1800, "text": " both", "score": 8.765625 }, { "rank": 7, "tokenId": 17368, "text": "original", "score": 7.625 }, { "rank": 8, "tokenId": 1128, "text": "der", "score": 7.04296875 } ] } }, "winnerTextsByMode": { "exact": "Both", "f32_forward": "Both", "f32_reverse": "Both", "f32_tree64": "Both", "f16_forward": "Both" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Both" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "password-good-bad-oneword-b", "text": "Respond with one word only.\nSharing a private SSH key publicly is good or bad. It is", "stableTopK": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "logit": 11.047394752502441, "prob": 0.9999982714653015 }, { "rank": 2, "tokenId": 15242, "text": "bad", "logit": -2.2687692642211914, "prob": 0.0000016476413975397008 }, { "rank": 3, "tokenId": 37701, "text": "Ter", "logit": -6.096305847167969, "prob": 3.58578837733603e-8 }, { "rank": 4, "tokenId": 26546, "text": "Very", "logit": -7.690915584564209, "prob": 7.2787105231952864e-9 }, { "rank": 5, "tokenId": 13741, "text": " Bad", "logit": -7.911609649658203, "prob": 5.8372493505487455e-9 }, { "rank": 6, "tokenId": 1018, "text": "**", "logit": -7.96135139465332, "prob": 5.55399726209771e-9 }, { "rank": 7, "tokenId": 3292, "text": "$\\", "logit": -7.968139171600342, "prob": 5.516425538587555e-9 }, { "rank": 8, "tokenId": 4287, "text": " bad", "logit": -8.178683280944824, "prob": 4.469095316039784e-9 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 11.591787020297991, "winnerGap": 13.864895238975, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 11.591787020297991 }, { "rank": 2, "tokenId": 15242, "text": "bad", "score": -2.2731082186770095 }, { "rank": 3, "tokenId": 37701, "text": "Ter", "score": -6.182361976681882 }, { "rank": 4, "tokenId": 26546, "text": "Very", "score": -7.866375720447943 }, { "rank": 5, "tokenId": 13741, "text": " Bad", "score": -8.10307974683469 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -8.156563560927234 }, { "rank": 7, "tokenId": 3292, "text": "$\\", "score": -8.16386495494719 }, { "rank": 8, "tokenId": 4287, "text": " bad", "score": -8.390856251577269 } ] }, "f32_forward": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 11.591777801513672, "winnerGap": 13.86488652229309, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 11.591777801513672 }, { "rank": 2, "tokenId": 15242, "text": "bad", "score": -2.273108720779419 }, { "rank": 3, "tokenId": 37701, "text": "Ter", "score": -6.18235969543457 }, { "rank": 4, "tokenId": 26546, "text": "Very", "score": -7.866374492645264 }, { "rank": 5, "tokenId": 13741, "text": " Bad", "score": -8.103067398071289 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -8.156561851501465 }, { "rank": 7, "tokenId": 3292, "text": "$\\", "score": -8.163867950439453 }, { "rank": 8, "tokenId": 4287, "text": " bad", "score": -8.39086627960205 } ] }, "f32_reverse": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 11.591778755187988, "winnerGap": 13.864885330200195, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 11.591778755187988 }, { "rank": 2, "tokenId": 15242, "text": "bad", "score": -2.273106575012207 }, { "rank": 3, "tokenId": 37701, "text": "Ter", "score": -6.182374954223633 }, { "rank": 4, "tokenId": 26546, "text": "Very", "score": -7.8663763999938965 }, { "rank": 5, "tokenId": 13741, "text": " Bad", "score": -8.103086471557617 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -8.15656566619873 }, { "rank": 7, "tokenId": 3292, "text": "$\\", "score": -8.16387939453125 }, { "rank": 8, "tokenId": 4287, "text": " bad", "score": -8.390854835510254 } ] }, "f32_tree64": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 11.591787338256836, "winnerGap": 13.86489462852478, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 11.591787338256836 }, { "rank": 2, "tokenId": 15242, "text": "bad", "score": -2.2731072902679443 }, { "rank": 3, "tokenId": 37701, "text": "Ter", "score": -6.182362079620361 }, { "rank": 4, "tokenId": 26546, "text": "Very", "score": -7.866374492645264 }, { "rank": 5, "tokenId": 13741, "text": " Bad", "score": -8.103078842163086 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -8.156562805175781 }, { "rank": 7, "tokenId": 3292, "text": "$\\", "score": -8.163864135742188 }, { "rank": 8, "tokenId": 4287, "text": " bad", "score": -8.390856742858887 } ] }, "f16_forward": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 7.69921875, "winnerGap": 9.970703125, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 7.69921875 }, { "rank": 2, "tokenId": 15242, "text": "bad", "score": -2.271484375 }, { "rank": 3, "tokenId": 13741, "text": " Bad", "score": -4.13671875 }, { "rank": 4, "tokenId": 25429, "text": "Cat", "score": -5.8046875 }, { "rank": 5, "tokenId": 37701, "text": "Ter", "score": -6.18359375 }, { "rank": 6, "tokenId": 236799, "text": "B", "score": -6.71875 }, { "rank": 7, "tokenId": 52980, "text": "Fatal", "score": -7.671875 }, { "rank": 8, "tokenId": 26546, "text": "Very", "score": -7.9296875 } ] } }, "winnerTextsByMode": { "exact": "Bad", "f32_forward": "Bad", "f32_reverse": "Bad", "f32_tree64": "Bad", "f16_forward": "Bad" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Bad" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "password-public-private-choice-oneword-c", "text": "No explanation. One word only.\nChoose exactly one word: public or private.\nA production database password should remain", "stableTopK": [ { "rank": 1, "tokenId": 7128, "text": "private", "logit": 8.841669082641602, "prob": 0.9995368719100952 }, { "rank": 2, "tokenId": 29335, "text": "Private", "logit": 1.0767858028411865, "prob": 0.00042418259545229375 }, { "rank": 3, "tokenId": 2153, "text": "public", "logit": -1.4700219631195068, "prob": 0.000033226784580620006 }, { "rank": 4, "tokenId": 1018, "text": "**", "logit": -4.446532249450684, "prob": 0.0000016935822486630059 }, { "rank": 5, "tokenId": 24760, "text": "secret", "logit": -5.017850875854492, "prob": 9.565017080603866e-7 }, { "rank": 6, "tokenId": 8707, "text": "password", "logit": -6.3057990074157715, "prob": 2.638377907260292e-7 }, { "rank": 7, "tokenId": 91082, "text": "PRIVATE", "logit": -6.5387043952941895, "prob": 2.090198307769242e-7 }, { "rank": 8, "tokenId": 45505, "text": "secure", "logit": -6.800821304321289, "prob": 1.608242570227958e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 7128, "winnerText": "private", "winnerScore": 9.111895156400585, "winnerGap": 8.034646439678376, "top": [ { "rank": 1, "tokenId": 7128, "text": "private", "score": 9.111895156400585 }, { "rank": 2, "tokenId": 29335, "text": "Private", "score": 1.0772487167222096 }, { "rank": 3, "tokenId": 2153, "text": "public", "score": -1.4712031522786617 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": -4.479529666943421 }, { "rank": 5, "tokenId": 24760, "text": "secret", "score": -5.0654508770656435 }, { "rank": 6, "tokenId": 8707, "text": "password", "score": -6.401204563520459 }, { "rank": 7, "tokenId": 91082, "text": "PRIVATE", "score": -6.6452948764463855 }, { "rank": 8, "tokenId": 45505, "text": "secure", "score": -6.921049919388725 } ] }, "f32_forward": { "winnerTokenId": 7128, "winnerText": "private", "winnerScore": 9.111909866333008, "winnerGap": 8.03466272354126, "top": [ { "rank": 1, "tokenId": 7128, "text": "private", "score": 9.111909866333008 }, { "rank": 2, "tokenId": 29335, "text": "Private", "score": 1.077247142791748 }, { "rank": 3, "tokenId": 2153, "text": "public", "score": -1.471197247505188 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": -4.479528903961182 }, { "rank": 5, "tokenId": 24760, "text": "secret", "score": -5.065452575683594 }, { "rank": 6, "tokenId": 8707, "text": "password", "score": -6.401208877563477 }, { "rank": 7, "tokenId": 91082, "text": "PRIVATE", "score": -6.64530086517334 }, { "rank": 8, "tokenId": 45505, "text": "secure", "score": -6.9210405349731445 } ] }, "f32_reverse": { "winnerTokenId": 7128, "winnerText": "private", "winnerScore": 9.111892700195312, "winnerGap": 8.034647107124329, "top": [ { "rank": 1, "tokenId": 7128, "text": "private", "score": 9.111892700195312 }, { "rank": 2, "tokenId": 29335, "text": "Private", "score": 1.0772455930709839 }, { "rank": 3, "tokenId": 2153, "text": "public", "score": -1.4712013006210327 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": -4.479525566101074 }, { "rank": 5, "tokenId": 24760, "text": "secret", "score": -5.065446376800537 }, { "rank": 6, "tokenId": 8707, "text": "password", "score": -6.4011921882629395 }, { "rank": 7, "tokenId": 91082, "text": "PRIVATE", "score": -6.645290374755859 }, { "rank": 8, "tokenId": 45505, "text": "secure", "score": -6.921046733856201 } ] }, "f32_tree64": { "winnerTokenId": 7128, "winnerText": "private", "winnerScore": 9.111895561218262, "winnerGap": 8.034648180007935, "top": [ { "rank": 1, "tokenId": 7128, "text": "private", "score": 9.111895561218262 }, { "rank": 2, "tokenId": 29335, "text": "Private", "score": 1.0772473812103271 }, { "rank": 3, "tokenId": 2153, "text": "public", "score": -1.4712023735046387 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": -4.47952938079834 }, { "rank": 5, "tokenId": 24760, "text": "secret", "score": -5.0654520988464355 }, { "rank": 6, "tokenId": 8707, "text": "password", "score": -6.401205062866211 }, { "rank": 7, "tokenId": 91082, "text": "PRIVATE", "score": -6.645294189453125 }, { "rank": 8, "tokenId": 45505, "text": "secure", "score": -6.921050071716309 } ] }, "f16_forward": { "winnerTokenId": 7128, "winnerText": "private", "winnerScore": 5.0859375, "winnerGap": 4.01171875, "top": [ { "rank": 1, "tokenId": 7128, "text": "private", "score": 5.0859375 }, { "rank": 2, "tokenId": 29335, "text": "Private", "score": 1.07421875 }, { "rank": 3, "tokenId": 2153, "text": "public", "score": -1.513671875 }, { "rank": 4, "tokenId": 238454, "text": "私", "score": -3.833984375 }, { "rank": 5, "tokenId": 52572, "text": " निजी", "score": -3.912109375 }, { "rank": 6, "tokenId": 129510, "text": "separate", "score": -4.28125 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -4.515625 }, { "rank": 8, "tokenId": 26584, "text": "protected", "score": -4.75390625 } ] } }, "winnerTextsByMode": { "exact": "private", "f32_forward": "private", "f32_reverse": "private", "f32_tree64": "private", "f16_forward": "private" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "private" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "password-yes-no-oneword-d", "text": "Choose one word only.\nShould you share your password with a stranger, yes or no? The answer is", "stableTopK": [ { "rank": 1, "tokenId": 3771, "text": "No", "logit": 20.788326263427734, "prob": 0.9999368786811829 }, { "rank": 2, "tokenId": 1904, "text": "no", "logit": 10.91840648651123, "prob": 0.00005170361328055151 }, { "rank": 3, "tokenId": 1018, "text": "**", "logit": 9.39828872680664, "prob": 0.000011306862688797992 }, { "rank": 4, "tokenId": 7018, "text": "NO", "logit": 4.047826290130615, "prob": 5.3661874943600196e-8 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "logit": 3.4261791706085205, "prob": 2.881959737521811e-8 }, { "rank": 6, "tokenId": 100, "text": "", "logit": 2.368560552597046, "prob": 1.0008522899340733e-8 }, { "rank": 7, "tokenId": 106, "text": "", "logit": 0.1091608926653862, "prob": 1.0450214027457605e-9 }, { "rank": 8, "tokenId": 198, "text": "", "logit": -0.436315655708313, "prob": 6.056583568003759e-10 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 25.607921176354225, "winnerGap": 14.16508799039741, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 25.607921176354225 }, { "rank": 2, "tokenId": 1904, "text": "no", "score": 11.442833185956815 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 9.725219955909168 }, { "rank": 4, "tokenId": 7018, "text": "NO", "score": 4.072664570236562 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "score": 3.441194554378299 }, { "rank": 6, "tokenId": 100, "text": "", "score": 2.373499032210532 }, { "rank": 7, "tokenId": 106, "text": "", "score": 0.10916408106104036 }, { "rank": 8, "tokenId": 198, "text": "", "score": -0.43634482536396035 } ] }, "f32_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 25.607912063598633, "winnerGap": 14.165080070495605, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 25.607912063598633 }, { "rank": 2, "tokenId": 1904, "text": "no", "score": 11.442831993103027 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 9.725221633911133 }, { "rank": 4, "tokenId": 7018, "text": "NO", "score": 4.072666168212891 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "score": 3.4411861896514893 }, { "rank": 6, "tokenId": 100, "text": "", "score": 2.3734915256500244 }, { "rank": 7, "tokenId": 106, "text": "", "score": 0.10916309803724289 }, { "rank": 8, "tokenId": 198, "text": "", "score": -0.43634116649627686 } ] }, "f32_reverse": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 25.607908248901367, "winnerGap": 14.165077209472656, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 25.607908248901367 }, { "rank": 2, "tokenId": 1904, "text": "no", "score": 11.442831039428711 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 9.725218772888184 }, { "rank": 4, "tokenId": 7018, "text": "NO", "score": 4.072662830352783 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "score": 3.441192388534546 }, { "rank": 6, "tokenId": 100, "text": "", "score": 2.373497724533081 }, { "rank": 7, "tokenId": 106, "text": "", "score": 0.10916324704885483 }, { "rank": 8, "tokenId": 198, "text": "", "score": -0.4363439679145813 } ] }, "f32_tree64": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 25.607919692993164, "winnerGap": 14.165085792541504, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 25.607919692993164 }, { "rank": 2, "tokenId": 1904, "text": "no", "score": 11.44283390045166 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 9.7252197265625 }, { "rank": 4, "tokenId": 7018, "text": "NO", "score": 4.072664260864258 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "score": 3.441193103790283 }, { "rank": 6, "tokenId": 100, "text": "", "score": 2.3734984397888184 }, { "rank": 7, "tokenId": 106, "text": "", "score": 0.10916352272033691 }, { "rank": 8, "tokenId": 198, "text": "", "score": -0.4363443851470947 } ] }, "f16_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 25.453125, "winnerGap": 13.8984375, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 25.453125 }, { "rank": 2, "tokenId": 1904, "text": "no", "score": 11.5546875 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 9.78125 }, { "rank": 4, "tokenId": 7018, "text": "NO", "score": 4.12890625 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "score": 3.560546875 }, { "rank": 6, "tokenId": 106, "text": "", "score": 0.1343994140625 }, { "rank": 7, "tokenId": 198, "text": "", "score": -0.685546875 }, { "rank": 8, "tokenId": 2318, "text": "\\[", "score": -0.98681640625 } ] } }, "winnerTextsByMode": { "exact": "No", "f32_forward": "No", "f32_reverse": "No", "f32_tree64": "No", "f16_forward": "No" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "No" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "password-yes-no-choice-oneword-a", "text": "Answer with exactly one word only.\nChoose exactly one word: yes or no.\nShould you share your password with a stranger?", "stableTopK": [ { "rank": 1, "tokenId": 3771, "text": "No", "logit": 16.498043060302734, "prob": 0.9905850887298584 }, { "rank": 2, "tokenId": 1904, "text": "no", "logit": 11.842042922973633, "prob": 0.009414919652044773 }, { "rank": 3, "tokenId": 7018, "text": "NO", "logit": -4.129220962524414, "prob": 1.0903974390075177e-9 }, { "rank": 4, "tokenId": 951, "text": " no", "logit": -5.926704406738281, "prob": 1.806956528849213e-10 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "logit": -7.703030586242676, "prob": 3.058433367675306e-11 }, { "rank": 6, "tokenId": 236751, "text": "s", "logit": -7.760335922241211, "prob": 2.888096359288106e-11 }, { "rank": 7, "tokenId": 32631, "text": "Never", "logit": -7.93923807144165, "prob": 2.414990603327727e-11 }, { "rank": 8, "tokenId": 236797, "text": "N", "logit": -7.9953389167785645, "prob": 2.283237661437898e-11 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 18.548639460492097, "winnerGap": 6.026749841900397, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 18.548639460492097 }, { "rank": 2, "tokenId": 1904, "text": "no", "score": 12.5218896185917 }, { "rank": 3, "tokenId": 7018, "text": "NO", "score": -4.155595695245935 }, { "rank": 4, "tokenId": 951, "text": " no", "score": -6.0056591731725675 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "score": -7.879344619954452 }, { "rank": 6, "tokenId": 236751, "text": "s", "score": -7.940725519166527 }, { "rank": 7, "tokenId": 32631, "text": "Never", "score": -8.132778940228889 }, { "rank": 8, "tokenId": 236797, "text": "N", "score": -8.193138593813543 } ] }, "f32_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 18.548641204833984, "winnerGap": 6.0267534255981445, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 18.548641204833984 }, { "rank": 2, "tokenId": 1904, "text": "no", "score": 12.52188777923584 }, { "rank": 3, "tokenId": 7018, "text": "NO", "score": -4.15559720993042 }, { "rank": 4, "tokenId": 951, "text": " no", "score": -6.00567102432251 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "score": -7.8793487548828125 }, { "rank": 6, "tokenId": 236751, "text": "s", "score": -7.940726280212402 }, { "rank": 7, "tokenId": 32631, "text": "Never", "score": -8.132777214050293 }, { "rank": 8, "tokenId": 236797, "text": "N", "score": -8.19313907623291 } ] }, "f32_reverse": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 18.548648834228516, "winnerGap": 6.026756286621094, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 18.548648834228516 }, { "rank": 2, "tokenId": 1904, "text": "no", "score": 12.521892547607422 }, { "rank": 3, "tokenId": 7018, "text": "NO", "score": -4.155592918395996 }, { "rank": 4, "tokenId": 951, "text": " no", "score": -6.005657196044922 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "score": -7.879337310791016 }, { "rank": 6, "tokenId": 236751, "text": "s", "score": -7.940728664398193 }, { "rank": 7, "tokenId": 32631, "text": "Never", "score": -8.132779121398926 }, { "rank": 8, "tokenId": 236797, "text": "N", "score": -8.19314193725586 } ] }, "f32_tree64": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 18.548641204833984, "winnerGap": 6.026751518249512, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 18.548641204833984 }, { "rank": 2, "tokenId": 1904, "text": "no", "score": 12.521889686584473 }, { "rank": 3, "tokenId": 7018, "text": "NO", "score": -4.1555962562561035 }, { "rank": 4, "tokenId": 951, "text": " no", "score": -6.005660533905029 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "score": -7.879344463348389 }, { "rank": 6, "tokenId": 236751, "text": "s", "score": -7.940725326538086 }, { "rank": 7, "tokenId": 32631, "text": "Never", "score": -8.132781028747559 }, { "rank": 8, "tokenId": 236797, "text": "N", "score": -8.193140029907227 } ] }, "f16_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 18.65625, "winnerGap": 14.078125, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 18.65625 }, { "rank": 2, "tokenId": 1904, "text": "no", "score": 4.578125 }, { "rank": 3, "tokenId": 236779, "text": "_", "score": -3.896484375 }, { "rank": 4, "tokenId": 236751, "text": "s", "score": -4 }, { "rank": 5, "tokenId": 7018, "text": "NO", "score": -4.14453125 }, { "rank": 6, "tokenId": 2301, "text": " No", "score": -5.01171875 }, { "rank": 7, "tokenId": 951, "text": " no", "score": -6.05078125 }, { "rank": 8, "tokenId": 156815, "text": "absolutely", "score": -6.8515625 } ] } }, "winnerTextsByMode": { "exact": "No", "f32_forward": "No", "f32_reverse": "No", "f32_tree64": "No", "f16_forward": "No" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "No" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "patch-approve-deny-oneword-b", "text": "Respond with one word only.\nInstalling a critical security patch on a public server should be approved or denied. It should be", "stableTopK": [ { "rank": 1, "tokenId": 79572, "text": "Mand", "logit": 11.499519348144531, "prob": 0.5292932987213135 }, { "rank": 2, "tokenId": 75938, "text": "Approved", "logit": 10.58264446258545, "prob": 0.21159368753433228 }, { "rank": 3, "tokenId": 48930, "text": "Autom", "logit": 10.116068840026855, "prob": 0.13270016014575958 }, { "rank": 4, "tokenId": 23520, "text": "Required", "logit": 9.43838119506836, "prob": 0.0673837959766388 }, { "rank": 5, "tokenId": 78850, "text": "Automatic", "logit": 8.760660171508789, "prob": 0.03421566262841225 }, { "rank": 6, "tokenId": 143311, "text": "Authorized", "logit": 7.808368682861328, "prob": 0.013202313333749771 }, { "rank": 7, "tokenId": 38370, "text": "Ur", "logit": 6.141161918640137, "prob": 0.002492248546332121 }, { "rank": 8, "tokenId": 229224, "text": "Necessary", "logit": 5.694927215576172, "prob": 0.001595122623257339 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 79572, "winnerText": "Mand", "winnerScore": 12.118279325912233, "winnerGap": 1.0606805331133753, "top": [ { "rank": 1, "tokenId": 79572, "text": "Mand", "score": 12.118279325912233 }, { "rank": 2, "tokenId": 75938, "text": "Approved", "score": 11.057598792798858 }, { "rank": 3, "tokenId": 48930, "text": "Autom", "score": 10.52797851621346 }, { "rank": 4, "tokenId": 23520, "text": "Required", "score": 9.769698521863148 }, { "rank": 5, "tokenId": 78850, "text": "Automatic", "score": 9.023257066760088 }, { "rank": 6, "tokenId": 143311, "text": "Authorized", "score": 7.992230974471035 }, { "rank": 7, "tokenId": 38370, "text": "Ur", "score": 6.229165488505284 }, { "rank": 8, "tokenId": 229224, "text": "Necessary", "score": 5.764853909751391 } ] }, "f32_forward": { "winnerTokenId": 79572, "winnerText": "Mand", "winnerScore": 12.118290901184082, "winnerGap": 1.060704231262207, "top": [ { "rank": 1, "tokenId": 79572, "text": "Mand", "score": 12.118290901184082 }, { "rank": 2, "tokenId": 75938, "text": "Approved", "score": 11.057586669921875 }, { "rank": 3, "tokenId": 48930, "text": "Autom", "score": 10.527982711791992 }, { "rank": 4, "tokenId": 23520, "text": "Required", "score": 9.769694328308105 }, { "rank": 5, "tokenId": 78850, "text": "Automatic", "score": 9.023259162902832 }, { "rank": 6, "tokenId": 143311, "text": "Authorized", "score": 7.992234230041504 }, { "rank": 7, "tokenId": 38370, "text": "Ur", "score": 6.229157447814941 }, { "rank": 8, "tokenId": 229224, "text": "Necessary", "score": 5.764853477478027 } ] }, "f32_reverse": { "winnerTokenId": 79572, "winnerText": "Mand", "winnerScore": 12.118278503417969, "winnerGap": 1.0606727600097656, "top": [ { "rank": 1, "tokenId": 79572, "text": "Mand", "score": 12.118278503417969 }, { "rank": 2, "tokenId": 75938, "text": "Approved", "score": 11.057605743408203 }, { "rank": 3, "tokenId": 48930, "text": "Autom", "score": 10.527976036071777 }, { "rank": 4, "tokenId": 23520, "text": "Required", "score": 9.769700050354004 }, { "rank": 5, "tokenId": 78850, "text": "Automatic", "score": 9.023262023925781 }, { "rank": 6, "tokenId": 143311, "text": "Authorized", "score": 7.99222469329834 }, { "rank": 7, "tokenId": 38370, "text": "Ur", "score": 6.229165554046631 }, { "rank": 8, "tokenId": 229224, "text": "Necessary", "score": 5.7648539543151855 } ] }, "f32_tree64": { "winnerTokenId": 79572, "winnerText": "Mand", "winnerScore": 12.118279457092285, "winnerGap": 1.0606813430786133, "top": [ { "rank": 1, "tokenId": 79572, "text": "Mand", "score": 12.118279457092285 }, { "rank": 2, "tokenId": 75938, "text": "Approved", "score": 11.057598114013672 }, { "rank": 3, "tokenId": 48930, "text": "Autom", "score": 10.527978897094727 }, { "rank": 4, "tokenId": 23520, "text": "Required", "score": 9.769698143005371 }, { "rank": 5, "tokenId": 78850, "text": "Automatic", "score": 9.023256301879883 }, { "rank": 6, "tokenId": 143311, "text": "Authorized", "score": 7.992230415344238 }, { "rank": 7, "tokenId": 38370, "text": "Ur", "score": 6.229165077209473 }, { "rank": 8, "tokenId": 229224, "text": "Necessary", "score": 5.7648539543151855 } ] }, "f16_forward": { "winnerTokenId": 79572, "winnerText": "Mand", "winnerScore": 12.0625, "winnerGap": 1.4609375, "top": [ { "rank": 1, "tokenId": 79572, "text": "Mand", "score": 12.0625 }, { "rank": 2, "tokenId": 48930, "text": "Autom", "score": 10.6015625 }, { "rank": 3, "tokenId": 78850, "text": "Automatic", "score": 9.0234375 }, { "rank": 4, "tokenId": 143311, "text": "Authorized", "score": 7.97265625 }, { "rank": 5, "tokenId": 75938, "text": "Approved", "score": 7.0390625 }, { "rank": 6, "tokenId": 38370, "text": "Ur", "score": 6.3203125 }, { "rank": 7, "tokenId": 229224, "text": "Necessary", "score": 5.8359375 }, { "rank": 8, "tokenId": 23520, "text": "Required", "score": 5.82421875 } ] } }, "winnerTextsByMode": { "exact": "Mand", "f32_forward": "Mand", "f32_reverse": "Mand", "f32_tree64": "Mand", "f16_forward": "Mand" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Mand" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "patch-approve-deny-choice-oneword-c", "text": "No explanation. One word only.\nChoose exactly one word: approve or deny.\nInstalling a critical security patch on a public server should be", "stableTopK": [ { "rank": 1, "tokenId": 2046, "text": "App", "logit": 13.559440612792969, "prob": 0.8928471803665161 }, { "rank": 2, "tokenId": 105961, "text": "approve", "logit": 11.439257621765137, "prob": 0.10715029388666153 }, { "rank": 3, "tokenId": 34767, "text": " approve", "logit": -0.3191012144088745, "prob": 8.383068461625953e-7 }, { "rank": 4, "tokenId": 1110, "text": "app", "logit": -0.4968942105770111, "prob": 7.017598022684979e-7 }, { "rank": 5, "tokenId": 22472, "text": "Den", "logit": -1.6158701181411743, "prob": 2.2920465880815755e-7 }, { "rank": 6, "tokenId": 14131, "text": "install", "logit": -2.067453622817993, "prob": 1.4591608987757354e-7 }, { "rank": 7, "tokenId": 139876, "text": "Approx", "logit": -2.477993965148926, "prob": 9.67849445032698e-8 }, { "rank": 8, "tokenId": 1018, "text": "**", "logit": -2.5473756790161133, "prob": 9.029749037381407e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 2046, "winnerText": "App", "winnerScore": 14.615626152913293, "winnerGap": 2.5679266032616415, "top": [ { "rank": 1, "tokenId": 2046, "text": "App", "score": 14.615626152913293 }, { "rank": 2, "tokenId": 105961, "text": "approve", "score": 12.047699549651652 }, { "rank": 3, "tokenId": 34767, "text": " approve", "score": -0.3191156055891913 }, { "rank": 4, "tokenId": 1110, "text": "app", "score": -0.4969348420197619 }, { "rank": 5, "tokenId": 22472, "text": "Den", "score": -1.6174380190986213 }, { "rank": 6, "tokenId": 14131, "text": "install", "score": -2.070741464681987 }, { "rank": 7, "tokenId": 139876, "text": "Approx", "score": -2.4836512383640468 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": -2.553526817311237 } ] }, "f32_forward": { "winnerTokenId": 2046, "winnerText": "App", "winnerScore": 14.615623474121094, "winnerGap": 2.567930221557617, "top": [ { "rank": 1, "tokenId": 2046, "text": "App", "score": 14.615623474121094 }, { "rank": 2, "tokenId": 105961, "text": "approve", "score": 12.047693252563477 }, { "rank": 3, "tokenId": 34767, "text": " approve", "score": -0.31911808252334595 }, { "rank": 4, "tokenId": 1110, "text": "app", "score": -0.4969344735145569 }, { "rank": 5, "tokenId": 22472, "text": "Den", "score": -1.6174321174621582 }, { "rank": 6, "tokenId": 14131, "text": "install", "score": -2.070742607116699 }, { "rank": 7, "tokenId": 139876, "text": "Approx", "score": -2.483651638031006 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": -2.553527355194092 } ] }, "f32_reverse": { "winnerTokenId": 2046, "winnerText": "App", "winnerScore": 14.615628242492676, "winnerGap": 2.5679311752319336, "top": [ { "rank": 1, "tokenId": 2046, "text": "App", "score": 14.615628242492676 }, { "rank": 2, "tokenId": 105961, "text": "approve", "score": 12.047697067260742 }, { "rank": 3, "tokenId": 34767, "text": " approve", "score": -0.3191213607788086 }, { "rank": 4, "tokenId": 1110, "text": "app", "score": -0.4969329237937927 }, { "rank": 5, "tokenId": 22472, "text": "Den", "score": -1.617436408996582 }, { "rank": 6, "tokenId": 14131, "text": "install", "score": -2.0707383155822754 }, { "rank": 7, "tokenId": 139876, "text": "Approx", "score": -2.483642816543579 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": -2.55352520942688 } ] }, "f32_tree64": { "winnerTokenId": 2046, "winnerText": "App", "winnerScore": 14.615623474121094, "winnerGap": 2.567925453186035, "top": [ { "rank": 1, "tokenId": 2046, "text": "App", "score": 14.615623474121094 }, { "rank": 2, "tokenId": 105961, "text": "approve", "score": 12.047698020935059 }, { "rank": 3, "tokenId": 34767, "text": " approve", "score": -0.31911444664001465 }, { "rank": 4, "tokenId": 1110, "text": "app", "score": -0.4969353675842285 }, { "rank": 5, "tokenId": 22472, "text": "Den", "score": -1.6174378395080566 }, { "rank": 6, "tokenId": 14131, "text": "install", "score": -2.0707414150238037 }, { "rank": 7, "tokenId": 139876, "text": "Approx", "score": -2.4836506843566895 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": -2.5535264015197754 } ] }, "f16_forward": { "winnerTokenId": 2046, "winnerText": "App", "winnerScore": 14.5703125, "winnerGap": 2.5859375, "top": [ { "rank": 1, "tokenId": 2046, "text": "App", "score": 14.5703125 }, { "rank": 2, "tokenId": 105961, "text": "approve", "score": 11.984375 }, { "rank": 3, "tokenId": 34767, "text": " approve", "score": -0.333984375 }, { "rank": 4, "tokenId": 1110, "text": "app", "score": -0.51171875 }, { "rank": 5, "tokenId": 22472, "text": "Den", "score": -1.5498046875 }, { "rank": 6, "tokenId": 14131, "text": "install", "score": -2.05859375 }, { "rank": 7, "tokenId": 139876, "text": "Approx", "score": -2.498046875 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": -2.55859375 } ] } }, "winnerTextsByMode": { "exact": "App", "f32_forward": "App", "f32_reverse": "App", "f32_tree64": "App", "f16_forward": "App" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "App" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "patches-good-bad-oneword-d", "text": "Choose one word only.\nIgnoring critical security patches for months is good or bad. It is", "stableTopK": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "logit": 20.96201515197754, "prob": 0.9878689646720886 }, { "rank": 2, "tokenId": 1018, "text": "**", "logit": 16.016845703125, "prob": 0.007031363435089588 }, { "rank": 3, "tokenId": 15242, "text": "bad", "logit": 15.69433879852295, "prob": 0.005093033891171217 }, { "rank": 4, "tokenId": 67185, "text": "BAD", "logit": 8.32062816619873, "prob": 0.0000031960585147317033 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "logit": 6.842527866363525, "prob": 7.289268069143873e-7 }, { "rank": 6, "tokenId": 4287, "text": " bad", "logit": 6.089447498321533, "prob": 3.4326163245168573e-7 }, { "rank": 7, "tokenId": 13741, "text": " Bad", "logit": 5.866444110870361, "prob": 2.746478173776268e-7 }, { "rank": 8, "tokenId": 79151, "text": "Harm", "logit": 5.691359519958496, "prob": 2.3053553377394564e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 25.944672663056796, "winnerGap": 8.077353212486951, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 25.944672663056796 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 17.867319450569845 }, { "rank": 3, "tokenId": 15242, "text": "bad", "score": 17.419787448123834 }, { "rank": 4, "tokenId": 67185, "text": "BAD", "score": 8.544410922839077 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "score": 6.96503130301409 }, { "rank": 6, "tokenId": 4287, "text": " bad", "score": 6.175203935336853 }, { "rank": 7, "tokenId": 13741, "text": " Bad", "score": 5.9429829064375 }, { "rank": 8, "tokenId": 79151, "text": "Harm", "score": 5.761152595512954 } ] }, "f32_forward": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 25.944671630859375, "winnerGap": 8.077360153198242, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 25.944671630859375 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 17.867311477661133 }, { "rank": 3, "tokenId": 15242, "text": "bad", "score": 17.4197940826416 }, { "rank": 4, "tokenId": 67185, "text": "BAD", "score": 8.544416427612305 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "score": 6.965041637420654 }, { "rank": 6, "tokenId": 4287, "text": " bad", "score": 6.175203323364258 }, { "rank": 7, "tokenId": 13741, "text": " Bad", "score": 5.942979335784912 }, { "rank": 8, "tokenId": 79151, "text": "Harm", "score": 5.761161804199219 } ] }, "f32_reverse": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 25.944677352905273, "winnerGap": 8.07735824584961, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 25.944677352905273 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 17.867319107055664 }, { "rank": 3, "tokenId": 15242, "text": "bad", "score": 17.41977882385254 }, { "rank": 4, "tokenId": 67185, "text": "BAD", "score": 8.54440975189209 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "score": 6.965033531188965 }, { "rank": 6, "tokenId": 4287, "text": " bad", "score": 6.175204277038574 }, { "rank": 7, "tokenId": 13741, "text": " Bad", "score": 5.942975044250488 }, { "rank": 8, "tokenId": 79151, "text": "Harm", "score": 5.76115083694458 } ] }, "f32_tree64": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 25.944673538208008, "winnerGap": 8.077356338500977, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 25.944673538208008 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 17.86731719970703 }, { "rank": 3, "tokenId": 15242, "text": "bad", "score": 17.419788360595703 }, { "rank": 4, "tokenId": 67185, "text": "BAD", "score": 8.544410705566406 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "score": 6.965031623840332 }, { "rank": 6, "tokenId": 4287, "text": " bad", "score": 6.175204277038574 }, { "rank": 7, "tokenId": 13741, "text": " Bad", "score": 5.942983627319336 }, { "rank": 8, "tokenId": 79151, "text": "Harm", "score": 5.761152744293213 } ] }, "f16_forward": { "winnerTokenId": 20260, "winnerText": "Bad", "winnerScore": 25.734375, "winnerGap": 7.90625, "top": [ { "rank": 1, "tokenId": 20260, "text": "Bad", "score": 25.734375 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 17.828125 }, { "rank": 3, "tokenId": 15242, "text": "bad", "score": 17.546875 }, { "rank": 4, "tokenId": 67185, "text": "BAD", "score": 8.5703125 }, { "rank": 5, "tokenId": 4287, "text": " bad", "score": 6.19921875 }, { "rank": 6, "tokenId": 13741, "text": " Bad", "score": 5.9765625 }, { "rank": 7, "tokenId": 37701, "text": "Ter", "score": 5.7109375 }, { "rank": 8, "tokenId": 79151, "text": "Harm", "score": 5.6484375 } ] } }, "winnerTextsByMode": { "exact": "Bad", "f32_forward": "Bad", "f32_reverse": "Bad", "f32_tree64": "Bad", "f16_forward": "Bad" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Bad" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "payroll-public-private-oneword-a", "text": "Answer with exactly one word only.\nPublic or private: a payroll spreadsheet should remain", "stableTopK": [ { "rank": 1, "tokenId": 29335, "text": "Private", "logit": 4.6262736320495605, "prob": 0.9271388649940491 }, { "rank": 2, "tokenId": 7128, "text": "private", "logit": 2.077012777328491, "prob": 0.0724460780620575 }, { "rank": 3, "tokenId": 15928, "text": "Conf", "logit": -3.345924139022827, "prob": 0.000319788814522326 }, { "rank": 4, "tokenId": 10331, "text": "conf", "logit": -4.93750524520874, "prob": 0.00006511010724352673 }, { "rank": 5, "tokenId": 91082, "text": "PRIVATE", "logit": -8.053372383117676, "prob": 0.000002886984930228209 }, { "rank": 6, "tokenId": 26647, "text": "Internal", "logit": -8.065826416015625, "prob": 0.000002851253157132305 }, { "rank": 7, "tokenId": 30730, "text": "Secret", "logit": -8.304543495178223, "prob": 0.0000022457545583165484 }, { "rank": 8, "tokenId": 19118, "text": "Public", "logit": -8.410247802734375, "prob": 0.0000020204843167448416 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 29335, "winnerText": "Private", "winnerScore": 4.663480223666184, "winnerGap": 2.5831371631048654, "top": [ { "rank": 1, "tokenId": 29335, "text": "Private", "score": 4.663480223666184 }, { "rank": 2, "tokenId": 7128, "text": "private", "score": 2.0803430605613187 }, { "rank": 3, "tokenId": 15928, "text": "Conf", "score": -3.359901673583522 }, { "rank": 4, "tokenId": 10331, "text": "conf", "score": -4.982827094238004 }, { "rank": 5, "tokenId": 91082, "text": "PRIVATE", "score": -8.25563608498588 }, { "rank": 6, "tokenId": 26647, "text": "Internal", "score": -8.269067707757607 }, { "rank": 7, "tokenId": 30730, "text": "Secret", "score": -8.526989714886547 }, { "rank": 8, "tokenId": 19118, "text": "Public", "score": -8.6415817609561 } ] }, "f32_forward": { "winnerTokenId": 29335, "winnerText": "Private", "winnerScore": 4.66348934173584, "winnerGap": 2.5831475257873535, "top": [ { "rank": 1, "tokenId": 29335, "text": "Private", "score": 4.66348934173584 }, { "rank": 2, "tokenId": 7128, "text": "private", "score": 2.0803418159484863 }, { "rank": 3, "tokenId": 15928, "text": "Conf", "score": -3.3599019050598145 }, { "rank": 4, "tokenId": 10331, "text": "conf", "score": -4.982827663421631 }, { "rank": 5, "tokenId": 91082, "text": "PRIVATE", "score": -8.255630493164062 }, { "rank": 6, "tokenId": 26647, "text": "Internal", "score": -8.269073486328125 }, { "rank": 7, "tokenId": 30730, "text": "Secret", "score": -8.526992797851562 }, { "rank": 8, "tokenId": 19118, "text": "Public", "score": -8.641581535339355 } ] }, "f32_reverse": { "winnerTokenId": 29335, "winnerText": "Private", "winnerScore": 4.663483142852783, "winnerGap": 2.5831427574157715, "top": [ { "rank": 1, "tokenId": 29335, "text": "Private", "score": 4.663483142852783 }, { "rank": 2, "tokenId": 7128, "text": "private", "score": 2.0803403854370117 }, { "rank": 3, "tokenId": 15928, "text": "Conf", "score": -3.3599014282226562 }, { "rank": 4, "tokenId": 10331, "text": "conf", "score": -4.9828290939331055 }, { "rank": 5, "tokenId": 91082, "text": "PRIVATE", "score": -8.255636215209961 }, { "rank": 6, "tokenId": 26647, "text": "Internal", "score": -8.269063949584961 }, { "rank": 7, "tokenId": 30730, "text": "Secret", "score": -8.526986122131348 }, { "rank": 8, "tokenId": 19118, "text": "Public", "score": -8.641568183898926 } ] }, "f32_tree64": { "winnerTokenId": 29335, "winnerText": "Private", "winnerScore": 4.663479328155518, "winnerGap": 2.583136558532715, "top": [ { "rank": 1, "tokenId": 29335, "text": "Private", "score": 4.663479328155518 }, { "rank": 2, "tokenId": 7128, "text": "private", "score": 2.0803427696228027 }, { "rank": 3, "tokenId": 15928, "text": "Conf", "score": -3.3599014282226562 }, { "rank": 4, "tokenId": 10331, "text": "conf", "score": -4.982827186584473 }, { "rank": 5, "tokenId": 91082, "text": "PRIVATE", "score": -8.255634307861328 }, { "rank": 6, "tokenId": 26647, "text": "Internal", "score": -8.269067764282227 }, { "rank": 7, "tokenId": 30730, "text": "Secret", "score": -8.526988983154297 }, { "rank": 8, "tokenId": 19118, "text": "Public", "score": -8.641582489013672 } ] }, "f16_forward": { "winnerTokenId": 29335, "winnerText": "Private", "winnerScore": 4.6328125, "winnerGap": 2.55078125, "top": [ { "rank": 1, "tokenId": 29335, "text": "Private", "score": 4.6328125 }, { "rank": 2, "tokenId": 7128, "text": "private", "score": 2.08203125 }, { "rank": 3, "tokenId": 91082, "text": "PRIVATE", "score": -4.3125 }, { "rank": 4, "tokenId": 19118, "text": "Public", "score": -4.6171875 }, { "rank": 5, "tokenId": 10331, "text": "conf", "score": -4.921875 }, { "rank": 6, "tokenId": 52581, "text": "Privacy", "score": -5.16796875 }, { "rank": 7, "tokenId": 2147, "text": " private", "score": -5.640625 }, { "rank": 8, "tokenId": 238454, "text": "私", "score": -5.9296875 } ] } }, "winnerTextsByMode": { "exact": "Private", "f32_forward": "Private", "f32_reverse": "Private", "f32_tree64": "Private", "f16_forward": "Private" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Private" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "philosophy-rigor-ambiguity-answer-oneword-b", "text": "Respond with one word only.\nAnswer with exactly one word: true or false. Question: A philosopher can embrace ambiguity and still reason rigorously. Answer:", "stableTopK": [ { "rank": 1, "tokenId": 3397, "text": "true", "logit": 14.762823104858398, "prob": 0.9998562335968018 }, { "rank": 2, "tokenId": 4339, "text": "True", "logit": 5.914029598236084, "prob": 0.00014353419828694314 }, { "rank": 3, "tokenId": 4530, "text": "false", "logit": -1.329883098602295, "prob": 1.0255670446213117e-7 }, { "rank": 4, "tokenId": 57985, "text": "truth", "logit": -2.0158071517944336, "prob": 5.1650083321419515e-8 }, { "rank": 5, "tokenId": 1847, "text": " true", "logit": -3.2758309841156006, "prob": 1.4650405333327399e-8 }, { "rank": 6, "tokenId": 236745, "text": "t", "logit": -4.275380611419678, "prob": 5.392010837823591e-9 }, { "rank": 7, "tokenId": 106, "text": "", "logit": -4.645901203155518, "prob": 3.7225085236514133e-9 }, { "rank": 8, "tokenId": 1, "text": "", "logit": -4.734137058258057, "prob": 3.4081235611438387e-9 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 16.164598657088113, "winnerGap": 10.172121650938692, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 16.164598657088113 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 5.9924770061494215 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": -1.3307580636031027 }, { "rank": 4, "tokenId": 57985, "text": "truth", "score": -2.0188451072671967 }, { "rank": 5, "tokenId": 1847, "text": " true", "score": -3.2889403329874654 }, { "rank": 6, "tokenId": 236745, "text": "t", "score": -4.304680975803459 }, { "rank": 7, "tokenId": 106, "text": "", "score": -4.683584461539871 }, { "rank": 8, "tokenId": 1, "text": "", "score": -4.774030017789911 } ] }, "f32_forward": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 16.164608001708984, "winnerGap": 10.172138214111328, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 16.164608001708984 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 5.992469787597656 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": -1.3307569026947021 }, { "rank": 4, "tokenId": 57985, "text": "truth", "score": -2.0188543796539307 }, { "rank": 5, "tokenId": 1847, "text": " true", "score": -3.2889411449432373 }, { "rank": 6, "tokenId": 236745, "text": "t", "score": -4.304680824279785 }, { "rank": 7, "tokenId": 106, "text": "", "score": -4.6835856437683105 }, { "rank": 8, "tokenId": 1, "text": "", "score": -4.7740302085876465 } ] }, "f32_reverse": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 16.16459846496582, "winnerGap": 10.172123432159424, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 16.16459846496582 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 5.9924750328063965 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": -1.3307561874389648 }, { "rank": 4, "tokenId": 57985, "text": "truth", "score": -2.01885724067688 }, { "rank": 5, "tokenId": 1847, "text": " true", "score": -3.2889418601989746 }, { "rank": 6, "tokenId": 236745, "text": "t", "score": -4.304673671722412 }, { "rank": 7, "tokenId": 106, "text": "", "score": -4.68358850479126 }, { "rank": 8, "tokenId": 1, "text": "", "score": -4.774031639099121 } ] }, "f32_tree64": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 16.164600372314453, "winnerGap": 10.172122955322266, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 16.164600372314453 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 5.9924774169921875 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": -1.330757975578308 }, { "rank": 4, "tokenId": 57985, "text": "truth", "score": -2.0188446044921875 }, { "rank": 5, "tokenId": 1847, "text": " true", "score": -3.288938283920288 }, { "rank": 6, "tokenId": 236745, "text": "t", "score": -4.304680824279785 }, { "rank": 7, "tokenId": 106, "text": "", "score": -4.683586120605469 }, { "rank": 8, "tokenId": 1, "text": "", "score": -4.7740302085876465 } ] }, "f16_forward": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 16.21875, "winnerGap": 10.24609375, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 16.21875 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 5.97265625 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": -1.34765625 }, { "rank": 4, "tokenId": 57985, "text": "truth", "score": -1.9638671875 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": -2.802734375 }, { "rank": 6, "tokenId": 1847, "text": " true", "score": -3.32421875 }, { "rank": 7, "tokenId": 3292, "text": "$\\", "score": -3.85546875 }, { "rank": 8, "tokenId": 10750, "text": "tree", "score": -3.95703125 } ] } }, "winnerTextsByMode": { "exact": "true", "f32_forward": "true", "f32_reverse": "true", "f32_tree64": "true", "f16_forward": "true" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "true" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "philosophy-vague-useful-answer-oneword-c", "text": "No explanation. One word only.\nAnswer with exactly one word: true or false. Question: A concept can be vague and still useful. Answer:", "stableTopK": [ { "rank": 1, "tokenId": 3397, "text": "true", "logit": 15.683016777038574, "prob": 0.6462236642837524 }, { "rank": 2, "tokenId": 4339, "text": "True", "logit": 15.080534934997559, "prob": 0.3537759482860565 }, { "rank": 3, "tokenId": 1847, "text": " true", "logit": -0.020848628133535385, "prob": 9.77869163421019e-8 }, { "rank": 4, "tokenId": 9277, "text": "False", "logit": -0.027483217418193817, "prob": 9.714028692542342e-8 }, { "rank": 5, "tokenId": 4530, "text": "false", "logit": -0.28346553444862366, "prob": 7.520170441921437e-8 }, { "rank": 6, "tokenId": 57985, "text": "truth", "logit": -1.5284277200698853, "prob": 2.1654466664244865e-8 }, { "rank": 7, "tokenId": 6288, "text": " True", "logit": -1.8657246828079224, "prob": 1.5454723722996278e-8 }, { "rank": 8, "tokenId": 22369, "text": "TRUE", "logit": -1.9320622682571411, "prob": 1.4462760766775773e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 17.40420258063044, "winnerGap": 0.8174480358165326, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 17.40420258063044 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 16.586754544813907 }, { "rank": 3, "tokenId": 1847, "text": " true", "score": -0.020846632011060073 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": -0.027480761943406984 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": -0.2834689356099336 }, { "rank": 6, "tokenId": 57985, "text": "truth", "score": -1.529758054954204 }, { "rank": 7, "tokenId": 6288, "text": " True", "score": -1.8681326708928978 }, { "rank": 8, "tokenId": 22369, "text": "TRUE", "score": -1.9347431546002856 } ] }, "f32_forward": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 17.404197692871094, "winnerGap": 0.81744384765625, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 17.404197692871094 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 16.586753845214844 }, { "rank": 3, "tokenId": 1847, "text": " true", "score": -0.020848777145147324 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": -0.027483027428388596 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": -0.28346842527389526 }, { "rank": 6, "tokenId": 57985, "text": "truth", "score": -1.5297571420669556 }, { "rank": 7, "tokenId": 6288, "text": " True", "score": -1.868131399154663 }, { "rank": 8, "tokenId": 22369, "text": "TRUE", "score": -1.9347405433654785 } ] }, "f32_reverse": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 17.404199600219727, "winnerGap": 0.8174591064453125, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 17.404199600219727 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 16.586740493774414 }, { "rank": 3, "tokenId": 1847, "text": " true", "score": -0.02085031569004059 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": -0.027483433485031128 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": -0.28346890211105347 }, { "rank": 6, "tokenId": 57985, "text": "truth", "score": -1.5297614336013794 }, { "rank": 7, "tokenId": 6288, "text": " True", "score": -1.8681343793869019 }, { "rank": 8, "tokenId": 22369, "text": "TRUE", "score": -1.9347447156906128 } ] }, "f32_tree64": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 17.40420150756836, "winnerGap": 0.8174476623535156, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 17.40420150756836 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 16.586753845214844 }, { "rank": 3, "tokenId": 1847, "text": " true", "score": -0.02084636688232422 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": -0.027480602264404297 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": -0.2834686040878296 }, { "rank": 6, "tokenId": 57985, "text": "truth", "score": -1.5297561883926392 }, { "rank": 7, "tokenId": 6288, "text": " True", "score": -1.8681344985961914 }, { "rank": 8, "tokenId": 22369, "text": "TRUE", "score": -1.934743046760559 } ] }, "f16_forward": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 17.609375, "winnerGap": 1, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 17.609375 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 16.609375 }, { "rank": 3, "tokenId": 1847, "text": " true", "score": 0.96923828125 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": -0.0212554931640625 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": -0.2568359375 }, { "rank": 6, "tokenId": 6288, "text": " True", "score": -0.88623046875 }, { "rank": 7, "tokenId": 57985, "text": "truth", "score": -1.5068359375 }, { "rank": 8, "tokenId": 22369, "text": "TRUE", "score": -1.90234375 } ] } }, "winnerTextsByMode": { "exact": "true", "f32_forward": "true", "f32_reverse": "true", "f32_tree64": "true", "f16_forward": "true" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "true" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "phishing-allow-block-oneword-d", "text": "Choose one word only.\nA phishing email should be allowed or blocked. It should be", "stableTopK": [ { "rank": 1, "tokenId": 210872, "text": "Blocked", "logit": 24.024219512939453, "prob": 0.9304724335670471 }, { "rank": 2, "tokenId": 1018, "text": "**", "logit": 21.0067081451416, "prob": 0.0455213338136673 }, { "rank": 3, "tokenId": 98542, "text": "blocked", "logit": 20.361167907714844, "prob": 0.0238704401999712 }, { "rank": 4, "tokenId": 10809, "text": "Block", "logit": 15.163952827453613, "prob": 0.00013205005961935967 }, { "rank": 5, "tokenId": 31336, "text": "Should", "logit": 10.10552978515625, "prob": 8.392539143642352e-7 }, { "rank": 6, "tokenId": 70242, "text": "Allowed", "logit": 9.736249923706055, "prob": 5.801190923193644e-7 }, { "rank": 7, "tokenId": 29994, "text": "Blo", "logit": 9.120244026184082, "prob": 3.133207826522266e-7 }, { "rank": 8, "tokenId": 5088, "text": "block", "logit": 9.016596794128418, "prob": 2.8247225714039814e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 210872, "winnerText": "Blocked", "winnerScore": 33.025766103879405, "winnerGap": 6.993596275379787, "top": [ { "rank": 1, "tokenId": 210872, "text": "Blocked", "score": 33.025766103879405 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 26.03216982849962 }, { "rank": 3, "tokenId": 98542, "text": "blocked", "score": 24.801298913606516 }, { "rank": 4, "tokenId": 10809, "text": "Block", "score": 16.698590375572252 }, { "rank": 5, "tokenId": 31336, "text": "Should", "score": 10.516080870572601 }, { "rank": 6, "tokenId": 70242, "text": "Allowed", "score": 10.101457783241749 }, { "rank": 7, "tokenId": 29994, "text": "Blo", "score": 9.417907314774869 }, { "rank": 8, "tokenId": 5088, "text": "block", "score": 9.303825535818262 } ] }, "f32_forward": { "winnerTokenId": 210872, "winnerText": "Blocked", "winnerScore": 33.025794982910156, "winnerGap": 6.993616104125977, "top": [ { "rank": 1, "tokenId": 210872, "text": "Blocked", "score": 33.025794982910156 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 26.03217887878418 }, { "rank": 3, "tokenId": 98542, "text": "blocked", "score": 24.80130958557129 }, { "rank": 4, "tokenId": 10809, "text": "Block", "score": 16.698619842529297 }, { "rank": 5, "tokenId": 31336, "text": "Should", "score": 10.516082763671875 }, { "rank": 6, "tokenId": 70242, "text": "Allowed", "score": 10.101460456848145 }, { "rank": 7, "tokenId": 29994, "text": "Blo", "score": 9.417901039123535 }, { "rank": 8, "tokenId": 5088, "text": "block", "score": 9.303828239440918 } ] }, "f32_reverse": { "winnerTokenId": 210872, "winnerText": "Blocked", "winnerScore": 33.02576446533203, "winnerGap": 6.993587493896484, "top": [ { "rank": 1, "tokenId": 210872, "text": "Blocked", "score": 33.02576446533203 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 26.032176971435547 }, { "rank": 3, "tokenId": 98542, "text": "blocked", "score": 24.801307678222656 }, { "rank": 4, "tokenId": 10809, "text": "Block", "score": 16.69859504699707 }, { "rank": 5, "tokenId": 31336, "text": "Should", "score": 10.516084671020508 }, { "rank": 6, "tokenId": 70242, "text": "Allowed", "score": 10.101448059082031 }, { "rank": 7, "tokenId": 29994, "text": "Blo", "score": 9.417905807495117 }, { "rank": 8, "tokenId": 5088, "text": "block", "score": 9.303827285766602 } ] }, "f32_tree64": { "winnerTokenId": 210872, "winnerText": "Blocked", "winnerScore": 33.02576446533203, "winnerGap": 6.993595123291016, "top": [ { "rank": 1, "tokenId": 210872, "text": "Blocked", "score": 33.02576446533203 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 26.032169342041016 }, { "rank": 3, "tokenId": 98542, "text": "blocked", "score": 24.80129623413086 }, { "rank": 4, "tokenId": 10809, "text": "Block", "score": 16.698593139648438 }, { "rank": 5, "tokenId": 31336, "text": "Should", "score": 10.516080856323242 }, { "rank": 6, "tokenId": 70242, "text": "Allowed", "score": 10.101458549499512 }, { "rank": 7, "tokenId": 29994, "text": "Blo", "score": 9.417906761169434 }, { "rank": 8, "tokenId": 5088, "text": "block", "score": 9.303828239440918 } ] }, "f16_forward": { "winnerTokenId": 210872, "winnerText": "Blocked", "winnerScore": 33.34375, "winnerGap": 7.640625, "top": [ { "rank": 1, "tokenId": 210872, "text": "Blocked", "score": 33.34375 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 25.703125 }, { "rank": 3, "tokenId": 98542, "text": "blocked", "score": 25.03125 }, { "rank": 4, "tokenId": 10809, "text": "Block", "score": 16.90625 }, { "rank": 5, "tokenId": 29994, "text": "Blo", "score": 9.5859375 }, { "rank": 6, "tokenId": 120367, "text": "Blocking", "score": 9.484375 }, { "rank": 7, "tokenId": 5088, "text": "block", "score": 9.40625 }, { "rank": 8, "tokenId": 8017, "text": "BL", "score": 9.03125 } ] } }, "winnerTextsByMode": { "exact": "Blocked", "f32_forward": "Blocked", "f32_reverse": "Blocked", "f32_tree64": "Blocked", "f16_forward": "Blocked" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Blocked" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "phishing-allow-block-choice-oneword-a", "text": "Answer with exactly one word only.\nChoose exactly one word: allow or block.\nA phishing email should", "stableTopK": [ { "rank": 1, "tokenId": 5088, "text": "block", "logit": 7.749195575714111, "prob": 0.999801754951477 }, { "rank": 2, "tokenId": 10809, "text": "Block", "logit": -0.7898135781288147, "prob": 0.0001956452615559101 }, { "rank": 3, "tokenId": 8021, "text": "allow", "logit": -5.459392070770264, "prob": 0.0000018344131831327104 }, { "rank": 4, "tokenId": 124174, "text": "ブロック", "logit": -7.854128360748291, "prob": 1.6729248386582185e-7 }, { "rank": 5, "tokenId": 61836, "text": "BLOCK", "logit": -7.889890193939209, "prob": 1.6141549963322177e-7 }, { "rank": 6, "tokenId": 70961, "text": " ब्लॉक", "logit": -8.567679405212402, "prob": 8.195682710265828e-8 }, { "rank": 7, "tokenId": 27122, "text": "Allow", "logit": -9.567538261413574, "prob": 3.015448868382009e-8 }, { "rank": 8, "tokenId": 91844, "text": "blocking", "logit": -9.677699089050293, "prob": 2.7009074088368834e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 5088, "winnerText": "block", "winnerScore": 7.928787723083097, "winnerGap": 8.71878593017668, "top": [ { "rank": 1, "tokenId": 5088, "text": "block", "score": 7.928787723083097 }, { "rank": 2, "tokenId": 10809, "text": "Block", "score": -0.789998207093582 }, { "rank": 3, "tokenId": 8021, "text": "allow", "score": -5.520884614358778 }, { "rank": 4, "tokenId": 124174, "text": "ブロック", "score": -8.041338455021428 }, { "rank": 5, "tokenId": 61836, "text": "BLOCK", "score": -8.079747645525877 }, { "rank": 6, "tokenId": 70961, "text": " ब्लॉक", "score": -8.812718498339251 }, { "rank": 7, "tokenId": 27122, "text": "Allow", "score": -9.913258693194756 }, { "rank": 8, "tokenId": 91844, "text": "blocking", "score": -10.036053667512913 } ] }, "f32_forward": { "winnerTokenId": 5088, "winnerText": "block", "winnerScore": 7.928785800933838, "winnerGap": 8.718786537647247, "top": [ { "rank": 1, "tokenId": 5088, "text": "block", "score": 7.928785800933838 }, { "rank": 2, "tokenId": 10809, "text": "Block", "score": -0.7900007367134094 }, { "rank": 3, "tokenId": 8021, "text": "allow", "score": -5.5208821296691895 }, { "rank": 4, "tokenId": 124174, "text": "ブロック", "score": -8.041336059570312 }, { "rank": 5, "tokenId": 61836, "text": "BLOCK", "score": -8.079732894897461 }, { "rank": 6, "tokenId": 70961, "text": " ब्लॉक", "score": -8.812710762023926 }, { "rank": 7, "tokenId": 27122, "text": "Allow", "score": -9.913262367248535 }, { "rank": 8, "tokenId": 91844, "text": "blocking", "score": -10.036052703857422 } ] }, "f32_reverse": { "winnerTokenId": 5088, "winnerText": "block", "winnerScore": 7.928792953491211, "winnerGap": 8.718793094158173, "top": [ { "rank": 1, "tokenId": 5088, "text": "block", "score": 7.928792953491211 }, { "rank": 2, "tokenId": 10809, "text": "Block", "score": -0.7900001406669617 }, { "rank": 3, "tokenId": 8021, "text": "allow", "score": -5.520888805389404 }, { "rank": 4, "tokenId": 124174, "text": "ブロック", "score": -8.041356086730957 }, { "rank": 5, "tokenId": 61836, "text": "BLOCK", "score": -8.079739570617676 }, { "rank": 6, "tokenId": 70961, "text": " ब्लॉक", "score": -8.812727928161621 }, { "rank": 7, "tokenId": 27122, "text": "Allow", "score": -9.913259506225586 }, { "rank": 8, "tokenId": 91844, "text": "blocking", "score": -10.036050796508789 } ] }, "f32_tree64": { "winnerTokenId": 5088, "winnerText": "block", "winnerScore": 7.928788661956787, "winnerGap": 8.718786597251892, "top": [ { "rank": 1, "tokenId": 5088, "text": "block", "score": 7.928788661956787 }, { "rank": 2, "tokenId": 10809, "text": "Block", "score": -0.789997935295105 }, { "rank": 3, "tokenId": 8021, "text": "allow", "score": -5.520884990692139 }, { "rank": 4, "tokenId": 124174, "text": "ブロック", "score": -8.041340827941895 }, { "rank": 5, "tokenId": 61836, "text": "BLOCK", "score": -8.079748153686523 }, { "rank": 6, "tokenId": 70961, "text": " ब्लॉक", "score": -8.812719345092773 }, { "rank": 7, "tokenId": 27122, "text": "Allow", "score": -9.913259506225586 }, { "rank": 8, "tokenId": 91844, "text": "blocking", "score": -10.036052703857422 } ] }, "f16_forward": { "winnerTokenId": 5088, "winnerText": "block", "winnerScore": 7.8515625, "winnerGap": 8.58203125, "top": [ { "rank": 1, "tokenId": 5088, "text": "block", "score": 7.8515625 }, { "rank": 2, "tokenId": 10809, "text": "Block", "score": -0.73046875 }, { "rank": 3, "tokenId": 124174, "text": "ブロック", "score": -4.046875 }, { "rank": 4, "tokenId": 61836, "text": "BLOCK", "score": -4.109375 }, { "rank": 5, "tokenId": 8021, "text": "allow", "score": -4.47265625 }, { "rank": 6, "tokenId": 152064, "text": "blok", "score": -6.70703125 }, { "rank": 7, "tokenId": 187671, "text": "analyze", "score": -7.6796875 }, { "rank": 8, "tokenId": 61993, "text": "reject", "score": -7.94140625 } ] } }, "winnerTextsByMode": { "exact": "block", "f32_forward": "block", "f32_reverse": "block", "f32_tree64": "block", "f16_forward": "block" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "block" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "poem-open-oneword-b", "text": "Respond with one word only.\nRoses are red,", "stableTopK": [ { "rank": 1, "tokenId": 1192, "text": "red", "logit": -9.747687339782715, "prob": 0.523150622844696 }, { "rank": 2, "tokenId": 9264, "text": "Red", "logit": -10.440605163574219, "prob": 0.26163530349731445 }, { "rank": 3, "tokenId": 81990, "text": "Flowers", "logit": -11.763616561889648, "prob": 0.06968187540769577 }, { "rank": 4, "tokenId": 26462, "text": "Love", "logit": -12.003881454467773, "prob": 0.05479918420314789 }, { "rank": 5, "tokenId": 21867, "text": "love", "logit": -12.122856140136719, "prob": 0.048652373254299164 }, { "rank": 6, "tokenId": 16927, "text": "blo", "logit": -12.737939834594727, "prob": 0.02630126103758812 }, { "rank": 7, "tokenId": 236847, "text": "V", "logit": -14.88318157196045, "prob": 0.0030782928224653006 }, { "rank": 8, "tokenId": 102378, "text": "Tul", "logit": -14.929760932922363, "prob": 0.002938196063041687 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 1192, "winnerText": "red", "winnerScore": -10.114242139341584, "winnerGap": 0.7814432067351991, "top": [ { "rank": 1, "tokenId": 1192, "text": "red", "score": -10.114242139341584 }, { "rank": 2, "tokenId": 9264, "text": "Red", "score": -10.895685346076784 }, { "rank": 3, "tokenId": 81990, "text": "Flowers", "score": -12.429123425446534 }, { "rank": 4, "tokenId": 26462, "text": "Love", "score": -12.71408589011412 }, { "rank": 5, "tokenId": 21867, "text": "love", "score": -12.856004259451534 }, { "rank": 6, "tokenId": 16927, "text": "blo", "score": -13.59865577972932 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": -16.323831719288208 }, { "rank": 8, "tokenId": 102378, "text": "Tul", "score": -16.38567842749749 } ] }, "f32_forward": { "winnerTokenId": 1192, "winnerText": "red", "winnerScore": -10.114229202270508, "winnerGap": 0.7814607620239258, "top": [ { "rank": 1, "tokenId": 1192, "text": "red", "score": -10.114229202270508 }, { "rank": 2, "tokenId": 9264, "text": "Red", "score": -10.895689964294434 }, { "rank": 3, "tokenId": 81990, "text": "Flowers", "score": -12.429121971130371 }, { "rank": 4, "tokenId": 26462, "text": "Love", "score": -12.714088439941406 }, { "rank": 5, "tokenId": 21867, "text": "love", "score": -12.855999946594238 }, { "rank": 6, "tokenId": 16927, "text": "blo", "score": -13.598648071289062 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": -16.32383155822754 }, { "rank": 8, "tokenId": 102378, "text": "Tul", "score": -16.385683059692383 } ] }, "f32_reverse": { "winnerTokenId": 1192, "winnerText": "red", "winnerScore": -10.114246368408203, "winnerGap": 0.7814226150512695, "top": [ { "rank": 1, "tokenId": 1192, "text": "red", "score": -10.114246368408203 }, { "rank": 2, "tokenId": 9264, "text": "Red", "score": -10.895668983459473 }, { "rank": 3, "tokenId": 81990, "text": "Flowers", "score": -12.429099082946777 }, { "rank": 4, "tokenId": 26462, "text": "Love", "score": -12.714075088500977 }, { "rank": 5, "tokenId": 21867, "text": "love", "score": -12.85600471496582 }, { "rank": 6, "tokenId": 16927, "text": "blo", "score": -13.598650932312012 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": -16.323837280273438 }, { "rank": 8, "tokenId": 102378, "text": "Tul", "score": -16.38562774658203 } ] }, "f32_tree64": { "winnerTokenId": 1192, "winnerText": "red", "winnerScore": -10.114240646362305, "winnerGap": 0.7814445495605469, "top": [ { "rank": 1, "tokenId": 1192, "text": "red", "score": -10.114240646362305 }, { "rank": 2, "tokenId": 9264, "text": "Red", "score": -10.895685195922852 }, { "rank": 3, "tokenId": 81990, "text": "Flowers", "score": -12.42912483215332 }, { "rank": 4, "tokenId": 26462, "text": "Love", "score": -12.714082717895508 }, { "rank": 5, "tokenId": 21867, "text": "love", "score": -12.856003761291504 }, { "rank": 6, "tokenId": 16927, "text": "blo", "score": -13.598652839660645 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": -16.323833465576172 }, { "rank": 8, "tokenId": 102378, "text": "Tul", "score": -16.38567543029785 } ] }, "f16_forward": { "winnerTokenId": 1192, "winnerText": "red", "winnerScore": -10.1484375, "winnerGap": 0.78125, "top": [ { "rank": 1, "tokenId": 1192, "text": "red", "score": -10.1484375 }, { "rank": 2, "tokenId": 9264, "text": "Red", "score": -10.9296875 }, { "rank": 3, "tokenId": 81990, "text": "Flowers", "score": -12.359375 }, { "rank": 4, "tokenId": 26462, "text": "Love", "score": -12.65625 }, { "rank": 5, "tokenId": 21867, "text": "love", "score": -12.8515625 }, { "rank": 6, "tokenId": 16927, "text": "blo", "score": -13.2421875 }, { "rank": 7, "tokenId": 12059, "text": "Long", "score": -15.5078125 }, { "rank": 8, "tokenId": 236847, "text": "V", "score": -16.109375 } ] } }, "winnerTextsByMode": { "exact": "red", "f32_forward": "red", "f32_reverse": "red", "f32_tree64": "red", "f16_forward": "red" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "red" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "pol-abortion-access-oneword-c", "text": "No explanation. One word only.\nRestricting abortion access protects or harms women. It", "stableTopK": [ { "rank": 1, "tokenId": 22945, "text": "Har", "logit": 6.894969940185547, "prob": 0.9897807240486145 }, { "rank": 2, "tokenId": 79151, "text": "Harm", "logit": 2.096104383468628, "prob": 0.00815489050000906 }, { "rank": 3, "tokenId": 25151, "text": "Prote", "logit": 0.180434912443161, "prob": 0.001200752449221909 }, { "rank": 4, "tokenId": 6234, "text": "har", "logit": -1.7861442565917969, "prob": 0.00016802696336526424 }, { "rank": 5, "tokenId": 90374, "text": "Hur", "logit": -1.9934440851211548, "prob": 0.0001365682837786153 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "logit": -2.8610572814941406, "prob": 0.00005735221566283144 }, { "rank": 7, "tokenId": 69720, "text": "Benefits", "logit": -3.441016435623169, "prob": 0.0000321127226925455 }, { "rank": 8, "tokenId": 236814, "text": "H", "logit": -3.97269606590271, "prob": 0.00001886998870759271 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 22945, "winnerText": "Har", "winnerScore": 7.020374207354722, "winnerGap": 4.92085027332312, "top": [ { "rank": 1, "tokenId": 22945, "text": "Har", "score": 7.020374207354722 }, { "rank": 2, "tokenId": 79151, "text": "Harm", "score": 2.099523934031602 }, { "rank": 3, "tokenId": 25151, "text": "Prote", "score": 0.18043575765197417 }, { "rank": 4, "tokenId": 6234, "text": "har", "score": -1.7882589248975016 }, { "rank": 5, "tokenId": 90374, "text": "Hur", "score": -1.9963863134222986 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": -2.8697795527186107 }, { "rank": 7, "tokenId": 69720, "text": "Benefits", "score": -3.4562252254099644 }, { "rank": 8, "tokenId": 236814, "text": "H", "score": -3.996164894093976 } ] }, "f32_forward": { "winnerTokenId": 22945, "winnerText": "Har", "winnerScore": 7.020377159118652, "winnerGap": 4.920858860015869, "top": [ { "rank": 1, "tokenId": 22945, "text": "Har", "score": 7.020377159118652 }, { "rank": 2, "tokenId": 79151, "text": "Harm", "score": 2.099518299102783 }, { "rank": 3, "tokenId": 25151, "text": "Prote", "score": 0.18044281005859375 }, { "rank": 4, "tokenId": 6234, "text": "har", "score": -1.7882602214813232 }, { "rank": 5, "tokenId": 90374, "text": "Hur", "score": -1.9963873624801636 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": -2.869776964187622 }, { "rank": 7, "tokenId": 69720, "text": "Benefits", "score": -3.4562301635742188 }, { "rank": 8, "tokenId": 236814, "text": "H", "score": -3.996164560317993 } ] }, "f32_reverse": { "winnerTokenId": 22945, "winnerText": "Har", "winnerScore": 7.02037239074707, "winnerGap": 4.920846462249756, "top": [ { "rank": 1, "tokenId": 22945, "text": "Har", "score": 7.02037239074707 }, { "rank": 2, "tokenId": 79151, "text": "Harm", "score": 2.0995259284973145 }, { "rank": 3, "tokenId": 25151, "text": "Prote", "score": 0.18043896555900574 }, { "rank": 4, "tokenId": 6234, "text": "har", "score": -1.788259744644165 }, { "rank": 5, "tokenId": 90374, "text": "Hur", "score": -1.9963808059692383 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": -2.869774103164673 }, { "rank": 7, "tokenId": 69720, "text": "Benefits", "score": -3.456216335296631 }, { "rank": 8, "tokenId": 236814, "text": "H", "score": -3.9961740970611572 } ] }, "f32_tree64": { "winnerTokenId": 22945, "winnerText": "Har", "winnerScore": 7.020374298095703, "winnerGap": 4.9208502769470215, "top": [ { "rank": 1, "tokenId": 22945, "text": "Har", "score": 7.020374298095703 }, { "rank": 2, "tokenId": 79151, "text": "Harm", "score": 2.0995240211486816 }, { "rank": 3, "tokenId": 25151, "text": "Prote", "score": 0.1804344654083252 }, { "rank": 4, "tokenId": 6234, "text": "har", "score": -1.788259506225586 }, { "rank": 5, "tokenId": 90374, "text": "Hur", "score": -1.996385097503662 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": -2.8697807788848877 }, { "rank": 7, "tokenId": 69720, "text": "Benefits", "score": -3.4562244415283203 }, { "rank": 8, "tokenId": 236814, "text": "H", "score": -3.996164321899414 } ] }, "f16_forward": { "winnerTokenId": 22945, "winnerText": "Har", "winnerScore": 6.984375, "winnerGap": 4.900390625, "top": [ { "rank": 1, "tokenId": 22945, "text": "Har", "score": 6.984375 }, { "rank": 2, "tokenId": 79151, "text": "Harm", "score": 2.083984375 }, { "rank": 3, "tokenId": 25151, "text": "Prote", "score": 0.1798095703125 }, { "rank": 4, "tokenId": 6234, "text": "har", "score": -0.783203125 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": -1.7724609375 }, { "rank": 6, "tokenId": 183182, "text": "Opens", "score": -2.37890625 }, { "rank": 7, "tokenId": 4969, "text": " Har", "score": -2.400390625 }, { "rank": 8, "tokenId": 149915, "text": "Reve", "score": -2.48046875 } ] } }, "winnerTextsByMode": { "exact": "Har", "f32_forward": "Har", "f32_reverse": "Har", "f32_tree64": "Har", "f16_forward": "Har" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Har" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "pol-billionaire-space-oneword-d", "text": "Choose one word only.\nBillionaires funding space exploration is visionary or wasteful. It is", "stableTopK": [ { "rank": 1, "tokenId": 66121, "text": "Vision", "logit": 18.904582977294922, "prob": 0.9581258296966553 }, { "rank": 2, "tokenId": 13031, "text": "vision", "logit": 15.765483856201172, "prob": 0.041507743299007416 }, { "rank": 3, "tokenId": 1018, "text": "**", "logit": 10.736700057983398, "prob": 0.00027174156275577843 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "logit": 7.600135326385498, "prob": 0.0000118022098831716 }, { "rank": 5, "tokenId": 95408, "text": " visionary", "logit": 7.350698947906494, "prob": 0.000009196752216666937 }, { "rank": 6, "tokenId": 6942, "text": "Vis", "logit": 7.028644561767578, "prob": 0.0000066645070546655916 }, { "rank": 7, "tokenId": 1584, "text": "Cont", "logit": 6.7034382820129395, "prob": 0.000004814295152755221 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "logit": 6.359713554382324, "prob": 0.000003413932972762268 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 66121, "winnerText": "Vision", "winnerScore": 22.25008848645436, "winnerGap": 4.73217949274229, "top": [ { "rank": 1, "tokenId": 66121, "text": "Vision", "score": 22.25008848645436 }, { "rank": 2, "tokenId": 13031, "text": "vision", "score": 17.51790899371207 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 11.233918593831733 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 7.769291744271165 }, { "rank": 5, "tokenId": 95408, "text": " visionary", "score": 7.5033402435617695 }, { "rank": 6, "tokenId": 6942, "text": "Vis", "score": 7.161654674718115 }, { "rank": 7, "tokenId": 1584, "text": "Cont", "score": 6.8184698234422605 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 6.4576423001948555 } ] }, "f32_forward": { "winnerTokenId": 66121, "winnerText": "Vision", "winnerScore": 22.250059127807617, "winnerGap": 4.732141494750977, "top": [ { "rank": 1, "tokenId": 66121, "text": "Vision", "score": 22.250059127807617 }, { "rank": 2, "tokenId": 13031, "text": "vision", "score": 17.51791763305664 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 11.233914375305176 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 7.769290447235107 }, { "rank": 5, "tokenId": 95408, "text": " visionary", "score": 7.503333568572998 }, { "rank": 6, "tokenId": 6942, "text": "Vis", "score": 7.161640644073486 }, { "rank": 7, "tokenId": 1584, "text": "Cont", "score": 6.818459987640381 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 6.457648277282715 } ] }, "f32_reverse": { "winnerTokenId": 66121, "winnerText": "Vision", "winnerScore": 22.250093460083008, "winnerGap": 4.732187271118164, "top": [ { "rank": 1, "tokenId": 66121, "text": "Vision", "score": 22.250093460083008 }, { "rank": 2, "tokenId": 13031, "text": "vision", "score": 17.517906188964844 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 11.233932495117188 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 7.769294738769531 }, { "rank": 5, "tokenId": 95408, "text": " visionary", "score": 7.503337860107422 }, { "rank": 6, "tokenId": 6942, "text": "Vis", "score": 7.161656856536865 }, { "rank": 7, "tokenId": 1584, "text": "Cont", "score": 6.818470001220703 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 6.457642555236816 } ] }, "f32_tree64": { "winnerTokenId": 66121, "winnerText": "Vision", "winnerScore": 22.25008773803711, "winnerGap": 4.732177734375, "top": [ { "rank": 1, "tokenId": 66121, "text": "Vision", "score": 22.25008773803711 }, { "rank": 2, "tokenId": 13031, "text": "vision", "score": 17.51791000366211 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 11.233917236328125 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 7.769291877746582 }, { "rank": 5, "tokenId": 95408, "text": " visionary", "score": 7.503337383270264 }, { "rank": 6, "tokenId": 6942, "text": "Vis", "score": 7.161653995513916 }, { "rank": 7, "tokenId": 1584, "text": "Cont", "score": 6.818470001220703 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 6.457642555236816 } ] }, "f16_forward": { "winnerTokenId": 66121, "winnerText": "Vision", "winnerScore": 22.15625, "winnerGap": 4.578125, "top": [ { "rank": 1, "tokenId": 66121, "text": "Vision", "score": 22.15625 }, { "rank": 2, "tokenId": 13031, "text": "vision", "score": 17.578125 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 11.2578125 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 7.77734375 }, { "rank": 5, "tokenId": 95408, "text": " visionary", "score": 7.625 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "score": 6.84375 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "score": 6.44140625 }, { "rank": 8, "tokenId": 109055, "text": "VISION", "score": 6.26953125 } ] } }, "winnerTextsByMode": { "exact": "Vision", "f32_forward": "Vision", "f32_reverse": "Vision", "f32_tree64": "Vision", "f16_forward": "Vision" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Vision" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "pol-censorship-hate-oneword-a", "text": "Answer with exactly one word only.\nBanning hate speech protects or restricts freedom. It", "stableTopK": [ { "rank": 1, "tokenId": 25151, "text": "Prote", "logit": 5.800872325897217, "prob": 0.9960501790046692 }, { "rank": 2, "tokenId": 14151, "text": "Rest", "logit": -0.22176720201969147, "prob": 0.0024136933498084545 }, { "rank": 3, "tokenId": 87417, "text": "protect", "logit": -1.0306501388549805, "prob": 0.0010749511420726776 }, { "rank": 4, "tokenId": 110362, "text": "Protect", "logit": -2.9117043018341064, "prob": 0.00016385408525820822 }, { "rank": 5, "tokenId": 22186, "text": "Both", "logit": -3.6605613231658936, "prob": 0.00007748771167825907 }, { "rank": 6, "tokenId": 4506, "text": "rest", "logit": -4.446539402008057, "prob": 0.00003530911635607481 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "logit": -4.800811767578125, "prob": 0.000024775836209300905 }, { "rank": 8, "tokenId": 18463, "text": "Perm", "logit": -5.373316287994385, "prob": 0.000013976317859487608 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 25151, "winnerText": "Prote", "winnerScore": 5.874837700937206, "winnerGap": 6.096609833211765, "top": [ { "rank": 1, "tokenId": 25151, "text": "Prote", "score": 5.874837700937206 }, { "rank": 2, "tokenId": 14151, "text": "Rest", "score": -0.22177213227455894 }, { "rank": 3, "tokenId": 87417, "text": "protect", "score": -1.0310545083925575 }, { "rank": 4, "tokenId": 110362, "text": "Protect", "score": -2.920897777323752 }, { "rank": 5, "tokenId": 22186, "text": "Both", "score": -3.678888684148923 }, { "rank": 6, "tokenId": 4506, "text": "rest", "score": -4.479537745853936 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": -4.842437088688101 }, { "rank": 8, "tokenId": 18463, "text": "Perm", "score": -5.431909243784569 } ] }, "f32_forward": { "winnerTokenId": 25151, "winnerText": "Prote", "winnerScore": 5.874841213226318, "winnerGap": 6.096615895628929, "top": [ { "rank": 1, "tokenId": 25151, "text": "Prote", "score": 5.874841213226318 }, { "rank": 2, "tokenId": 14151, "text": "Rest", "score": -0.22177468240261078 }, { "rank": 3, "tokenId": 87417, "text": "protect", "score": -1.0310579538345337 }, { "rank": 4, "tokenId": 110362, "text": "Protect", "score": -2.9208984375 }, { "rank": 5, "tokenId": 22186, "text": "Both", "score": -3.6788837909698486 }, { "rank": 6, "tokenId": 4506, "text": "rest", "score": -4.479541778564453 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": -4.842440128326416 }, { "rank": 8, "tokenId": 18463, "text": "Perm", "score": -5.431901931762695 } ] }, "f32_reverse": { "winnerTokenId": 25151, "winnerText": "Prote", "winnerScore": 5.874836444854736, "winnerGap": 6.096611529588699, "top": [ { "rank": 1, "tokenId": 25151, "text": "Prote", "score": 5.874836444854736 }, { "rank": 2, "tokenId": 14151, "text": "Rest", "score": -0.221775084733963 }, { "rank": 3, "tokenId": 87417, "text": "protect", "score": -1.0310524702072144 }, { "rank": 4, "tokenId": 110362, "text": "Protect", "score": -2.920893669128418 }, { "rank": 5, "tokenId": 22186, "text": "Both", "score": -3.6788907051086426 }, { "rank": 6, "tokenId": 4506, "text": "rest", "score": -4.479537010192871 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": -4.842428684234619 }, { "rank": 8, "tokenId": 18463, "text": "Perm", "score": -5.431907653808594 } ] }, "f32_tree64": { "winnerTokenId": 25151, "winnerText": "Prote", "winnerScore": 5.874837875366211, "winnerGap": 6.096609354019165, "top": [ { "rank": 1, "tokenId": 25151, "text": "Prote", "score": 5.874837875366211 }, { "rank": 2, "tokenId": 14151, "text": "Rest", "score": -0.2217714786529541 }, { "rank": 3, "tokenId": 87417, "text": "protect", "score": -1.0310535430908203 }, { "rank": 4, "tokenId": 110362, "text": "Protect", "score": -2.920898199081421 }, { "rank": 5, "tokenId": 22186, "text": "Both", "score": -3.6788878440856934 }, { "rank": 6, "tokenId": 4506, "text": "rest", "score": -4.479537010192871 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": -4.842437744140625 }, { "rank": 8, "tokenId": 18463, "text": "Perm", "score": -5.431910037994385 } ] }, "f16_forward": { "winnerTokenId": 25151, "winnerText": "Prote", "winnerScore": 0.90869140625, "winnerGap": 1.1419677734375, "top": [ { "rank": 1, "tokenId": 25151, "text": "Prote", "score": 0.90869140625 }, { "rank": 2, "tokenId": 14151, "text": "Rest", "score": -0.2332763671875 }, { "rank": 3, "tokenId": 87417, "text": "protect", "score": -1.02734375 }, { "rank": 4, "tokenId": 110362, "text": "Protect", "score": -2.9140625 }, { "rank": 5, "tokenId": 4506, "text": "rest", "score": -3.515625 }, { "rank": 6, "tokenId": 22186, "text": "Both", "score": -3.685546875 }, { "rank": 7, "tokenId": 27413, "text": "both", "score": -3.9453125 }, { "rank": 8, "tokenId": 106729, "text": "Freedom", "score": -3.955078125 } ] } }, "winnerTextsByMode": { "exact": "Prote", "f32_forward": "Prote", "f32_reverse": "Prote", "f32_tree64": "Prote", "f16_forward": "Prote" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Prote" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "pol-climate-mandate-oneword-b", "text": "Respond with one word only.\nMandatory emission caps help or hurt the economy. They", "stableTopK": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "logit": 9.481443405151367, "prob": 0.7099173069000244 }, { "rank": 2, "tokenId": 105367, "text": "Mixed", "logit": 8.000961303710938, "prob": 0.16152605414390564 }, { "rank": 3, "tokenId": 236847, "text": "V", "logit": 6.354050636291504, "prob": 0.031117046251893044 }, { "rank": 4, "tokenId": 22186, "text": "Both", "logit": 6.249353408813477, "prob": 0.028023920953273773 }, { "rank": 5, "tokenId": 48333, "text": "Complex", "logit": 5.425909996032715, "prob": 0.012300195172429085 }, { "rank": 6, "tokenId": 28368, "text": "Help", "logit": 5.191773891448975, "prob": 0.009732579812407494 }, { "rank": 7, "tokenId": 19204, "text": "Variable", "logit": 4.851049423217773, "prob": 0.0069223446771502495 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "logit": 4.769594669342041, "prob": 0.006380840204656124 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 9.817519021646373, "winnerGap": 1.618329301412583, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 9.817519021646373 }, { "rank": 2, "tokenId": 105367, "text": "Mixed", "score": 8.19918972023379 }, { "rank": 3, "tokenId": 236847, "text": "V", "score": 6.451707830921954 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 6.34217736880219 }, { "rank": 5, "tokenId": 48333, "text": "Complex", "score": 5.486265413349351 }, { "rank": 6, "tokenId": 28368, "text": "Help", "score": 5.244559949550123 }, { "rank": 7, "tokenId": 19204, "text": "Variable", "score": 4.894004182827786 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 4.810402544774423 } ] }, "f32_forward": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 9.817516326904297, "winnerGap": 1.6183185577392578, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 9.817516326904297 }, { "rank": 2, "tokenId": 105367, "text": "Mixed", "score": 8.199197769165039 }, { "rank": 3, "tokenId": 236847, "text": "V", "score": 6.451714038848877 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 6.342174053192139 }, { "rank": 5, "tokenId": 48333, "text": "Complex", "score": 5.486274719238281 }, { "rank": 6, "tokenId": 28368, "text": "Help", "score": 5.244560241699219 }, { "rank": 7, "tokenId": 19204, "text": "Variable", "score": 4.8940110206604 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 4.810403347015381 } ] }, "f32_reverse": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 9.817514419555664, "winnerGap": 1.6183204650878906, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 9.817514419555664 }, { "rank": 2, "tokenId": 105367, "text": "Mixed", "score": 8.199193954467773 }, { "rank": 3, "tokenId": 236847, "text": "V", "score": 6.451709747314453 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 6.342174530029297 }, { "rank": 5, "tokenId": 48333, "text": "Complex", "score": 5.486266613006592 }, { "rank": 6, "tokenId": 28368, "text": "Help", "score": 5.244555473327637 }, { "rank": 7, "tokenId": 19204, "text": "Variable", "score": 4.894010066986084 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 4.810400009155273 } ] }, "f32_tree64": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 9.817519187927246, "winnerGap": 1.6183290481567383, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 9.817519187927246 }, { "rank": 2, "tokenId": 105367, "text": "Mixed", "score": 8.199190139770508 }, { "rank": 3, "tokenId": 236847, "text": "V", "score": 6.451706886291504 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 6.3421783447265625 }, { "rank": 5, "tokenId": 48333, "text": "Complex", "score": 5.486266136169434 }, { "rank": 6, "tokenId": 28368, "text": "Help", "score": 5.244559288024902 }, { "rank": 7, "tokenId": 19204, "text": "Variable", "score": 4.894002914428711 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 4.8104023933410645 } ] }, "f16_forward": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 9.9140625, "winnerGap": 1.6875, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 9.9140625 }, { "rank": 2, "tokenId": 105367, "text": "Mixed", "score": 8.2265625 }, { "rank": 3, "tokenId": 236847, "text": "V", "score": 6.515625 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 6.33984375 }, { "rank": 5, "tokenId": 48333, "text": "Complex", "score": 5.55859375 }, { "rank": 6, "tokenId": 28368, "text": "Help", "score": 5.265625 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "score": 4.84375 }, { "rank": 8, "tokenId": 20809, "text": "Neutral", "score": 4.6953125 } ] } }, "winnerTextsByMode": { "exact": "Depends", "f32_forward": "Depends", "f32_reverse": "Depends", "f32_tree64": "Depends", "f16_forward": "Depends" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Depends" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "pol-death-penalty-oneword-c", "text": "No explanation. One word only.\nThe death penalty deters crime — true, false, or inconclusive?", "stableTopK": [ { "rank": 1, "tokenId": 902, "text": "In", "logit": 8.388161659240723, "prob": 0.9994384050369263 }, { "rank": 2, "tokenId": 9277, "text": "False", "logit": 0.7155339121818542, "prob": 0.00046513191773556173 }, { "rank": 3, "tokenId": 1505, "text": "Con", "logit": -2.39465069770813, "prob": 0.00002074149961117655 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "logit": -3.2645347118377686, "prob": 0.000008690691174706444 }, { "rank": 5, "tokenId": 4339, "text": "True", "logit": -3.76161789894104, "prob": 0.000005286568011797499 }, { "rank": 6, "tokenId": 3771, "text": "No", "logit": -3.915438175201416, "prob": 0.000004532841558102518 }, { "rank": 7, "tokenId": 48333, "text": "Complex", "logit": -4.009742259979248, "prob": 0.00000412491272072657 }, { "rank": 8, "tokenId": 14977, "text": "Question", "logit": -4.35827112197876, "prob": 0.0000029110562991263578 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 902, "winnerText": "In", "winnerScore": 8.617620188405212, "winnerGap": 7.901947807447868, "top": [ { "rank": 1, "tokenId": 902, "text": "In", "score": 8.617620188405212 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 0.7156723809573435 }, { "rank": 3, "tokenId": 1505, "text": "Con", "score": -2.3997575946701097 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": -3.277513400731232 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -3.781519142130847 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": -3.9378963171517967 }, { "rank": 7, "tokenId": 48333, "text": "Complex", "score": -4.033878157617643 }, { "rank": 8, "tokenId": 14977, "text": "Question", "score": -4.389323350147727 } ] }, "f32_forward": { "winnerTokenId": 902, "winnerText": "In", "winnerScore": 8.617619514465332, "winnerGap": 7.901942551136017, "top": [ { "rank": 1, "tokenId": 902, "text": "In", "score": 8.617619514465332 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 0.7156769633293152 }, { "rank": 3, "tokenId": 1505, "text": "Con", "score": -2.3997576236724854 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": -3.2775158882141113 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -3.7815186977386475 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": -3.9378981590270996 }, { "rank": 7, "tokenId": 48333, "text": "Complex", "score": -4.033879280090332 }, { "rank": 8, "tokenId": 14977, "text": "Question", "score": -4.389326095581055 } ] }, "f32_reverse": { "winnerTokenId": 902, "winnerText": "In", "winnerScore": 8.617620468139648, "winnerGap": 7.90194433927536, "top": [ { "rank": 1, "tokenId": 902, "text": "In", "score": 8.617620468139648 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 0.7156761288642883 }, { "rank": 3, "tokenId": 1505, "text": "Con", "score": -2.3997535705566406 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": -3.277517318725586 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -3.7815210819244385 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": -3.937899351119995 }, { "rank": 7, "tokenId": 48333, "text": "Complex", "score": -4.033864498138428 }, { "rank": 8, "tokenId": 14977, "text": "Question", "score": -4.38932991027832 } ] }, "f32_tree64": { "winnerTokenId": 902, "winnerText": "In", "winnerScore": 8.617620468139648, "winnerGap": 7.901948809623718, "top": [ { "rank": 1, "tokenId": 902, "text": "In", "score": 8.617620468139648 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 0.7156716585159302 }, { "rank": 3, "tokenId": 1505, "text": "Con", "score": -2.3997578620910645 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": -3.2775135040283203 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -3.7815191745758057 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": -3.9378957748413086 }, { "rank": 7, "tokenId": 48333, "text": "Complex", "score": -4.033878326416016 }, { "rank": 8, "tokenId": 14977, "text": "Question", "score": -4.389324188232422 } ] }, "f16_forward": { "winnerTokenId": 902, "winnerText": "In", "winnerScore": 8.625, "winnerGap": 7.98388671875, "top": [ { "rank": 1, "tokenId": 902, "text": "In", "score": 8.625 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 0.64111328125 }, { "rank": 3, "tokenId": 1505, "text": "Con", "score": -2.404296875 }, { "rank": 4, "tokenId": 153153, "text": " inconclusive", "score": -2.498046875 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": -3.28125 }, { "rank": 6, "tokenId": 4339, "text": "True", "score": -3.734375 }, { "rank": 7, "tokenId": 3771, "text": "No", "score": -3.955078125 }, { "rank": 8, "tokenId": 48333, "text": "Complex", "score": -4.0546875 } ] } }, "winnerTextsByMode": { "exact": "In", "f32_forward": "In", "f32_reverse": "In", "f32_tree64": "In", "f16_forward": "In" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "In" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "pol-drug-decrim-oneword-d", "text": "Choose one word only.\nDecriminalizing all drugs would help or hurt society. It would", "stableTopK": [ { "rank": 1, "tokenId": 28368, "text": "Help", "logit": 11.982474327087402, "prob": 0.6537466645240784 }, { "rank": 2, "tokenId": 113214, "text": "depends", "logit": 10.643902778625488, "prob": 0.17142541706562042 }, { "rank": 3, "tokenId": 17002, "text": "help", "logit": 9.476640701293945, "prob": 0.05335065722465515 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "logit": 9.468265533447266, "prob": 0.05290570110082626 }, { "rank": 5, "tokenId": 46052, "text": "could", "logit": 8.516032218933105, "prob": 0.020415160804986954 }, { "rank": 6, "tokenId": 30092, "text": "Could", "logit": 8.35599422454834, "prob": 0.0173959918320179 }, { "rank": 7, "tokenId": 178899, "text": "Benefit", "logit": 6.8227338790893555, "prob": 0.0037545913364738226 }, { "rank": 8, "tokenId": 1018, "text": "**", "logit": 6.673704147338867, "prob": 0.0032347438391298056 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 28368, "winnerText": "Help", "winnerScore": 12.688610646353595, "winnerGap": 1.5609880119444117, "top": [ { "rank": 1, "tokenId": 28368, "text": "Help", "score": 12.688610646353595 }, { "rank": 2, "tokenId": 113214, "text": "depends", "score": 11.127622634409184 }, { "rank": 3, "tokenId": 17002, "text": "help", "score": 9.812183174516166 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": 9.802881061226987 }, { "rank": 5, "tokenId": 46052, "text": "could", "score": 8.756513775524347 }, { "rank": 6, "tokenId": 30092, "text": "Could", "score": 8.582737369464674 }, { "rank": 7, "tokenId": 178899, "text": "Benefit", "score": 6.94415342630608 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": 6.787179407753463 } ] }, "f32_forward": { "winnerTokenId": 28368, "winnerText": "Help", "winnerScore": 12.688611030578613, "winnerGap": 1.5609922409057617, "top": [ { "rank": 1, "tokenId": 28368, "text": "Help", "score": 12.688611030578613 }, { "rank": 2, "tokenId": 113214, "text": "depends", "score": 11.127618789672852 }, { "rank": 3, "tokenId": 17002, "text": "help", "score": 9.812189102172852 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": 9.802872657775879 }, { "rank": 5, "tokenId": 46052, "text": "could", "score": 8.756514549255371 }, { "rank": 6, "tokenId": 30092, "text": "Could", "score": 8.582734107971191 }, { "rank": 7, "tokenId": 178899, "text": "Benefit", "score": 6.944154739379883 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": 6.787174224853516 } ] }, "f32_reverse": { "winnerTokenId": 28368, "winnerText": "Help", "winnerScore": 12.688619613647461, "winnerGap": 1.560999870300293, "top": [ { "rank": 1, "tokenId": 28368, "text": "Help", "score": 12.688619613647461 }, { "rank": 2, "tokenId": 113214, "text": "depends", "score": 11.127619743347168 }, { "rank": 3, "tokenId": 17002, "text": "help", "score": 9.812191009521484 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": 9.802885055541992 }, { "rank": 5, "tokenId": 46052, "text": "could", "score": 8.756518363952637 }, { "rank": 6, "tokenId": 30092, "text": "Could", "score": 8.582740783691406 }, { "rank": 7, "tokenId": 178899, "text": "Benefit", "score": 6.944147109985352 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": 6.787185192108154 } ] }, "f32_tree64": { "winnerTokenId": 28368, "winnerText": "Help", "winnerScore": 12.688611030578613, "winnerGap": 1.560988426208496, "top": [ { "rank": 1, "tokenId": 28368, "text": "Help", "score": 12.688611030578613 }, { "rank": 2, "tokenId": 113214, "text": "depends", "score": 11.127622604370117 }, { "rank": 3, "tokenId": 17002, "text": "help", "score": 9.81218433380127 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": 9.80288314819336 }, { "rank": 5, "tokenId": 46052, "text": "could", "score": 8.756513595581055 }, { "rank": 6, "tokenId": 30092, "text": "Could", "score": 8.58273696899414 }, { "rank": 7, "tokenId": 178899, "text": "Benefit", "score": 6.944153308868408 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": 6.787179946899414 } ] }, "f16_forward": { "winnerTokenId": 28368, "winnerText": "Help", "winnerScore": 11.703125, "winnerGap": 1.8671875, "top": [ { "rank": 1, "tokenId": 28368, "text": "Help", "score": 11.703125 }, { "rank": 2, "tokenId": 17002, "text": "help", "score": 9.8359375 }, { "rank": 3, "tokenId": 179306, "text": "Depends", "score": 9.796875 }, { "rank": 4, "tokenId": 46052, "text": "could", "score": 8.734375 }, { "rank": 5, "tokenId": 30092, "text": "Could", "score": 8.640625 }, { "rank": 6, "tokenId": 113214, "text": "depends", "score": 7.12890625 }, { "rank": 7, "tokenId": 178899, "text": "Benefit", "score": 6.9921875 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": 6.83203125 } ] } }, "winnerTextsByMode": { "exact": "Help", "f32_forward": "Help", "f32_reverse": "Help", "f32_tree64": "Help", "f16_forward": "Help" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Help" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "pol-electoral-college-oneword-a", "text": "Answer with exactly one word only.\nThe electoral college is democratic or undemocratic. It is", "stableTopK": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "logit": 10.970091819763184, "prob": 0.5576547980308533 }, { "rank": 2, "tokenId": 30340, "text": "Und", "logit": 9.834888458251953, "prob": 0.17920614778995514 }, { "rank": 3, "tokenId": 105367, "text": "Mixed", "logit": 8.876933097839355, "prob": 0.06875719875097275 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "logit": 8.654661178588867, "prob": 0.055053722113370895 }, { "rank": 5, "tokenId": 131022, "text": "Hybrid", "logit": 8.592805862426758, "prob": 0.05175154283642769 }, { "rank": 6, "tokenId": 95889, "text": "Neither", "logit": 8.03950309753418, "prob": 0.029759589582681656 }, { "rank": 7, "tokenId": 151808, "text": "Democratic", "logit": 7.664699554443359, "prob": 0.020457465201616287 }, { "rank": 8, "tokenId": 28498, "text": "Dem", "logit": 6.957534313201904, "prob": 0.010086345486342907 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 11.502456560299052, "winnerGap": 1.290617205075712, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 11.502456560299052 }, { "rank": 2, "tokenId": 30340, "text": "Und", "score": 10.21183935522334 }, { "rank": 3, "tokenId": 105367, "text": "Mixed", "score": 9.150529812064946 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 8.907507787682334 }, { "rank": 5, "tokenId": 131022, "text": "Hybrid", "score": 8.84008094617403 }, { "rank": 6, "tokenId": 95889, "text": "Neither", "score": 8.240696051719596 }, { "rank": 7, "tokenId": 151808, "text": "Democratic", "score": 7.838319375888602 }, { "rank": 8, "tokenId": 28498, "text": "Dem", "score": 7.086455744988088 } ] }, "f32_forward": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 11.50247573852539, "winnerGap": 1.2906322479248047, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 11.50247573852539 }, { "rank": 2, "tokenId": 30340, "text": "Und", "score": 10.211843490600586 }, { "rank": 3, "tokenId": 105367, "text": "Mixed", "score": 9.150533676147461 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 8.907509803771973 }, { "rank": 5, "tokenId": 131022, "text": "Hybrid", "score": 8.840078353881836 }, { "rank": 6, "tokenId": 95889, "text": "Neither", "score": 8.240690231323242 }, { "rank": 7, "tokenId": 151808, "text": "Democratic", "score": 7.838320255279541 }, { "rank": 8, "tokenId": 28498, "text": "Dem", "score": 7.086453437805176 } ] }, "f32_reverse": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 11.502457618713379, "winnerGap": 1.290618896484375, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 11.502457618713379 }, { "rank": 2, "tokenId": 30340, "text": "Und", "score": 10.211838722229004 }, { "rank": 3, "tokenId": 105367, "text": "Mixed", "score": 9.150524139404297 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 8.907508850097656 }, { "rank": 5, "tokenId": 131022, "text": "Hybrid", "score": 8.84007740020752 }, { "rank": 6, "tokenId": 95889, "text": "Neither", "score": 8.240691184997559 }, { "rank": 7, "tokenId": 151808, "text": "Democratic", "score": 7.838316917419434 }, { "rank": 8, "tokenId": 28498, "text": "Dem", "score": 7.08644962310791 } ] }, "f32_tree64": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 11.502455711364746, "winnerGap": 1.2906160354614258, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 11.502455711364746 }, { "rank": 2, "tokenId": 30340, "text": "Und", "score": 10.21183967590332 }, { "rank": 3, "tokenId": 105367, "text": "Mixed", "score": 9.150529861450195 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 8.90750789642334 }, { "rank": 5, "tokenId": 131022, "text": "Hybrid", "score": 8.840080261230469 }, { "rank": 6, "tokenId": 95889, "text": "Neither", "score": 8.240696907043457 }, { "rank": 7, "tokenId": 151808, "text": "Democratic", "score": 7.838318824768066 }, { "rank": 8, "tokenId": 28498, "text": "Dem", "score": 7.086455345153809 } ] }, "f16_forward": { "winnerTokenId": 30340, "winnerText": "Und", "winnerScore": 10.2734375, "winnerGap": 1.15625, "top": [ { "rank": 1, "tokenId": 30340, "text": "Und", "score": 10.2734375 }, { "rank": 2, "tokenId": 105367, "text": "Mixed", "score": 9.1171875 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": 9.0078125 }, { "rank": 4, "tokenId": 151808, "text": "Democratic", "score": 7.96484375 }, { "rank": 5, "tokenId": 28498, "text": "Dem", "score": 7.25 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "score": 6.82421875 }, { "rank": 7, "tokenId": 48333, "text": "Complex", "score": 6.69140625 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 6.625 } ] } }, "winnerTextsByMode": { "exact": "Amb", "f32_forward": "Amb", "f32_reverse": "Amb", "f32_tree64": "Amb", "f16_forward": "Und" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Amb", "Und" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 55144, 47705 ], "decodedText": "Ambiguous", "steps": [ { "step": 0, "tokenId": 55144, "text": "Amb" }, { "step": 1, "tokenId": 47705, "text": "iguous" } ] }, "f16_forward": { "tokenIds": [ 30340, 581, 48790 ], "decodedText": "Undemocratic", "steps": [ { "step": 0, "tokenId": 30340, "text": "Und" }, { "step": 1, "tokenId": 581, "text": "em" }, { "step": 2, "tokenId": 48790, "text": "ocratic" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 2, "differingStepCount": 3, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "pol-guns-regulate-oneword-b", "text": "Respond with one word only.\nStricter gun control laws would reduce violence — agree or disagree?", "stableTopK": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "logit": 5.205803394317627, "prob": 0.9833501577377319 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "logit": 0.7313008904457092, "prob": 0.011206150986254215 }, { "rank": 3, "tokenId": 30340, "text": "Und", "logit": -0.9362594485282898, "prob": 0.0021146778017282486 }, { "rank": 4, "tokenId": 115863, "text": "Analyze", "logit": -1.979064702987671, "prob": 0.0007453489233739674 }, { "rank": 5, "tokenId": 48333, "text": "Complex", "logit": -2.035980463027954, "prob": 0.0007041114731691778 }, { "rank": 6, "tokenId": 43868, "text": "Nu", "logit": -2.3701226711273193, "prob": 0.0005041100084781647 }, { "rank": 7, "tokenId": 20809, "text": "Neutral", "logit": -3.136566400527954, "prob": 0.00023424105893354863 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "logit": -3.2069404125213623, "prob": 0.00021832324273418635 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 208903, "winnerText": "Agree", "winnerScore": 5.259021361960819, "winnerGap": 4.527579409144053, "top": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "score": 5.259021361960819 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 0.7314419528167662 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": -0.9365620424601282 }, { "rank": 4, "tokenId": 115863, "text": "Analyze", "score": -1.9819407320138538 }, { "rank": 5, "tokenId": 48333, "text": "Complex", "score": -2.039116045204935 }, { "rank": 6, "tokenId": 43868, "text": "Nu", "score": -2.375072065700355 }, { "rank": 7, "tokenId": 20809, "text": "Neutral", "score": -3.148067637106813 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "score": -3.2192400395284544 } ] }, "f32_forward": { "winnerTokenId": 208903, "winnerText": "Agree", "winnerScore": 5.25902795791626, "winnerGap": 4.52758401632309, "top": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "score": 5.25902795791626 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 0.7314439415931702 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": -0.9365621209144592 }, { "rank": 4, "tokenId": 115863, "text": "Analyze", "score": -1.9819387197494507 }, { "rank": 5, "tokenId": 48333, "text": "Complex", "score": -2.039116621017456 }, { "rank": 6, "tokenId": 43868, "text": "Nu", "score": -2.3750736713409424 }, { "rank": 7, "tokenId": 20809, "text": "Neutral", "score": -3.1480605602264404 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "score": -3.219238042831421 } ] }, "f32_reverse": { "winnerTokenId": 208903, "winnerText": "Agree", "winnerScore": 5.259023666381836, "winnerGap": 4.5275790095329285, "top": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "score": 5.259023666381836 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 0.7314446568489075 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": -0.9365625381469727 }, { "rank": 4, "tokenId": 115863, "text": "Analyze", "score": -1.9819427728652954 }, { "rank": 5, "tokenId": 48333, "text": "Complex", "score": -2.039118766784668 }, { "rank": 6, "tokenId": 43868, "text": "Nu", "score": -2.375073194503784 }, { "rank": 7, "tokenId": 20809, "text": "Neutral", "score": -3.1480555534362793 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "score": -3.219242572784424 } ] }, "f32_tree64": { "winnerTokenId": 208903, "winnerText": "Agree", "winnerScore": 5.259021282196045, "winnerGap": 4.527580499649048, "top": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "score": 5.259021282196045 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 0.7314407825469971 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": -0.936562180519104 }, { "rank": 4, "tokenId": 115863, "text": "Analyze", "score": -1.981939673423767 }, { "rank": 5, "tokenId": 48333, "text": "Complex", "score": -2.0391154289245605 }, { "rank": 6, "tokenId": 43868, "text": "Nu", "score": -2.3750722408294678 }, { "rank": 7, "tokenId": 20809, "text": "Neutral", "score": -3.1480679512023926 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "score": -3.2192399501800537 } ] }, "f16_forward": { "winnerTokenId": 208903, "winnerText": "Agree", "winnerScore": 5.25390625, "winnerGap": 4.5439453125, "top": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "score": 5.25390625 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 0.7099609375 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": -0.91845703125 }, { "rank": 4, "tokenId": 233950, "text": "Disagree", "score": -1.8515625 }, { "rank": 5, "tokenId": 115863, "text": "Analyze", "score": -1.984375 }, { "rank": 6, "tokenId": 48333, "text": "Complex", "score": -1.986328125 }, { "rank": 7, "tokenId": 20809, "text": "Neutral", "score": -2.095703125 }, { "rank": 8, "tokenId": 43868, "text": "Nu", "score": -2.37109375 } ] } }, "winnerTextsByMode": { "exact": "Agree", "f32_forward": "Agree", "f32_reverse": "Agree", "f32_tree64": "Agree", "f16_forward": "Agree" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Agree" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "pol-immigration-open-oneword-c", "text": "No explanation. One word only.\nA country with fully open borders would be stronger or weaker. It would be", "stableTopK": [ { "rank": 1, "tokenId": 2805, "text": "Un", "logit": 10.981008529663086, "prob": 0.5528779029846191 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "logit": 9.62384033203125, "prob": 0.14230448007583618 }, { "rank": 3, "tokenId": 51757, "text": "Strong", "logit": 9.534478187561035, "prob": 0.13013948500156403 }, { "rank": 4, "tokenId": 55144, "text": "Amb", "logit": 8.742533683776855, "prob": 0.05894839018583298 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "logit": 8.662287712097168, "prob": 0.05440283939242363 }, { "rank": 6, "tokenId": 19204, "text": "Variable", "logit": 7.257406711578369, "prob": 0.013350252993404865 }, { "rank": 7, "tokenId": 236847, "text": "V", "logit": 7.038510322570801, "prob": 0.010725659318268299 }, { "rank": 8, "tokenId": 13811, "text": "Vol", "logit": 6.8833794593811035, "prob": 0.009184414520859718 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 11.515059637843319, "winnerGap": 1.5390787544325821, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 11.515059637843319 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 9.975980883410736 }, { "rank": 3, "tokenId": 51757, "text": "Strong", "score": 9.876474237047653 }, { "rank": 4, "tokenId": 55144, "text": "Amb", "score": 9.003447539028333 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 8.915830951856503 }, { "rank": 6, "tokenId": 19204, "text": "Variable", "score": 7.404172061145799 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": 7.172097120084313 }, { "rank": 8, "tokenId": 13811, "text": "Vol", "score": 7.008134255207544 } ] }, "f32_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 11.515056610107422, "winnerGap": 1.5390892028808594, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 11.515056610107422 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 9.975967407226562 }, { "rank": 3, "tokenId": 51757, "text": "Strong", "score": 9.876480102539062 }, { "rank": 4, "tokenId": 55144, "text": "Amb", "score": 9.003459930419922 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 8.915833473205566 }, { "rank": 6, "tokenId": 19204, "text": "Variable", "score": 7.40416955947876 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": 7.172092914581299 }, { "rank": 8, "tokenId": 13811, "text": "Vol", "score": 7.008139133453369 } ] }, "f32_reverse": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 11.515056610107422, "winnerGap": 1.539076805114746, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 11.515056610107422 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 9.975979804992676 }, { "rank": 3, "tokenId": 51757, "text": "Strong", "score": 9.87647819519043 }, { "rank": 4, "tokenId": 55144, "text": "Amb", "score": 9.00344181060791 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 8.915825843811035 }, { "rank": 6, "tokenId": 19204, "text": "Variable", "score": 7.404170989990234 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": 7.172093868255615 }, { "rank": 8, "tokenId": 13811, "text": "Vol", "score": 7.0081353187561035 } ] }, "f32_tree64": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 11.515060424804688, "winnerGap": 1.5390796661376953, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 11.515060424804688 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 9.975980758666992 }, { "rank": 3, "tokenId": 51757, "text": "Strong", "score": 9.876474380493164 }, { "rank": 4, "tokenId": 55144, "text": "Amb", "score": 9.003448486328125 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 8.915830612182617 }, { "rank": 6, "tokenId": 19204, "text": "Variable", "score": 7.404171943664551 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": 7.1720967292785645 }, { "rank": 8, "tokenId": 13811, "text": "Vol", "score": 7.008134841918945 } ] }, "f16_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 11.6328125, "winnerGap": 1.625, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 11.6328125 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 10.0078125 }, { "rank": 3, "tokenId": 51757, "text": "Strong", "score": 9.953125 }, { "rank": 4, "tokenId": 41387, "text": "Unknown", "score": 8.9453125 }, { "rank": 5, "tokenId": 19204, "text": "Variable", "score": 7.40234375 }, { "rank": 6, "tokenId": 13811, "text": "Vol", "score": 7.08984375 }, { "rank": 7, "tokenId": 123725, "text": "Strength", "score": 6.16796875 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": 5.37109375 } ] } }, "winnerTextsByMode": { "exact": "Un", "f32_forward": "Un", "f32_reverse": "Un", "f32_tree64": "Un", "f16_forward": "Un" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Un" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "pol-nationalism-oneword-d", "text": "Choose one word only.\nNationalism is healthy or dangerous. It is", "stableTopK": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "logit": 7.727067947387695, "prob": 0.5011601448059082 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "logit": 7.1222991943359375, "prob": 0.27373403310775757 }, { "rank": 3, "tokenId": 3637, "text": "Context", "logit": 5.9908127784729, "prob": 0.08829385787248611 }, { "rank": 4, "tokenId": 1018, "text": "**", "logit": 5.055270195007324, "prob": 0.03464411571621895 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "logit": 4.828028678894043, "prob": 0.027601949870586395 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "logit": 4.581209182739258, "prob": 0.021564899012446404 }, { "rank": 7, "tokenId": 236536, "text": "Dangerous", "logit": 3.9423043727874756, "prob": 0.01138346828520298 }, { "rank": 8, "tokenId": 1584, "text": "Cont", "logit": 3.881270408630371, "prob": 0.010709467343986034 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 7.90508686807474, "winnerGap": 0.6442600312512106, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 7.90508686807474 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 7.260826836823529 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 6.072407577765482 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 5.103950356937261 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 4.870367305764452 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 4.617325807578055 }, { "rank": 7, "tokenId": 236536, "text": "Dangerous", "score": 3.9652395869346435 }, { "rank": 8, "tokenId": 1584, "text": "Cont", "score": 3.9031449422427613 } ] }, "f32_forward": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 7.905084133148193, "winnerGap": 0.6442747116088867, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 7.905084133148193 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 7.260809421539307 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 6.072408199310303 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 5.103952884674072 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 4.87036657333374 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 4.617335796356201 }, { "rank": 7, "tokenId": 236536, "text": "Dangerous", "score": 3.9652342796325684 }, { "rank": 8, "tokenId": 1584, "text": "Cont", "score": 3.9031450748443604 } ] }, "f32_reverse": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 7.905088424682617, "winnerGap": 0.644261360168457, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 7.905088424682617 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 7.26082706451416 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 6.0724053382873535 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 5.103946208953857 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 4.8703765869140625 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 4.617318630218506 }, { "rank": 7, "tokenId": 236536, "text": "Dangerous", "score": 3.9652421474456787 }, { "rank": 8, "tokenId": 1584, "text": "Cont", "score": 3.9031429290771484 } ] }, "f32_tree64": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 7.905086994171143, "winnerGap": 0.6442599296569824, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 7.905086994171143 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 7.26082706451416 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 6.0724077224731445 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 5.103951454162598 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 4.870368480682373 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 4.6173272132873535 }, { "rank": 7, "tokenId": 236536, "text": "Dangerous", "score": 3.9652392864227295 }, { "rank": 8, "tokenId": 1584, "text": "Cont", "score": 3.903144598007202 } ] }, "f16_forward": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 7.828125, "winnerGap": 0.609375, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 7.828125 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 7.21875 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 5.078125 }, { "rank": 4, "tokenId": 3637, "text": "Context", "score": 5.06640625 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 4.9296875 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 4.5859375 }, { "rank": 7, "tokenId": 1584, "text": "Cont", "score": 4.046875 }, { "rank": 8, "tokenId": 1525, "text": "amb", "score": 3.236328125 } ] } }, "winnerTextsByMode": { "exact": "Amb", "f32_forward": "Amb", "f32_reverse": "Amb", "f32_tree64": "Amb", "f16_forward": "Amb" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Amb" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "pol-reparations-oneword-a", "text": "Answer with exactly one word only.\nReparations for slavery are owed or impractical. They are", "stableTopK": [ { "rank": 1, "tokenId": 236806, "text": "O", "logit": 16.860118865966797, "prob": 0.7816417813301086 }, { "rank": 2, "tokenId": 197598, "text": "Obl", "logit": 14.309174537658691, "prob": 0.06097428500652313 }, { "rank": 3, "tokenId": 229224, "text": "Necessary", "logit": 14.263924598693848, "prob": 0.0582766979932785 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "logit": 13.94417953491211, "prob": 0.0423283576965332 }, { "rank": 5, "tokenId": 2805, "text": "Un", "logit": 12.665090560913086, "prob": 0.011779588647186756 }, { "rank": 6, "tokenId": 23520, "text": "Required", "logit": 12.355366706848145, "prob": 0.008642089553177357 }, { "rank": 7, "tokenId": 203760, "text": "Ongoing", "logit": 11.793274879455566, "prob": 0.0049261245876550674 }, { "rank": 8, "tokenId": 30340, "text": "Und", "logit": 11.780943870544434, "prob": 0.004865753464400768 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 236806, "winnerText": "O", "winnerScore": 19.072728771187258, "winnerGap": 3.50099714663358, "top": [ { "rank": 1, "tokenId": 236806, "text": "O", "score": 19.072728771187258 }, { "rank": 2, "tokenId": 197598, "text": "Obl", "score": 15.571731624553678 }, { "rank": 3, "tokenId": 229224, "text": "Necessary", "score": 15.51321458181092 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 15.102735005630905 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": 13.509885714653361 }, { "rank": 6, "tokenId": 23520, "text": "Required", "score": 13.134952599741458 }, { "rank": 7, "tokenId": 203760, "text": "Ongoing", "score": 12.464166460681465 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": 12.449583693503435 } ] }, "f32_forward": { "winnerTokenId": 236806, "winnerText": "O", "winnerScore": 19.072736740112305, "winnerGap": 3.5010175704956055, "top": [ { "rank": 1, "tokenId": 236806, "text": "O", "score": 19.072736740112305 }, { "rank": 2, "tokenId": 197598, "text": "Obl", "score": 15.5717191696167 }, { "rank": 3, "tokenId": 229224, "text": "Necessary", "score": 15.513228416442871 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 15.102740287780762 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": 13.509883880615234 }, { "rank": 6, "tokenId": 23520, "text": "Required", "score": 13.134952545166016 }, { "rank": 7, "tokenId": 203760, "text": "Ongoing", "score": 12.46416187286377 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": 12.449583053588867 } ] }, "f32_reverse": { "winnerTokenId": 236806, "winnerText": "O", "winnerScore": 19.072711944580078, "winnerGap": 3.500973701477051, "top": [ { "rank": 1, "tokenId": 236806, "text": "O", "score": 19.072711944580078 }, { "rank": 2, "tokenId": 197598, "text": "Obl", "score": 15.571738243103027 }, { "rank": 3, "tokenId": 229224, "text": "Necessary", "score": 15.513219833374023 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 15.10273551940918 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": 13.509888648986816 }, { "rank": 6, "tokenId": 23520, "text": "Required", "score": 13.134950637817383 }, { "rank": 7, "tokenId": 203760, "text": "Ongoing", "score": 12.464164733886719 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": 12.449584007263184 } ] }, "f32_tree64": { "winnerTokenId": 236806, "winnerText": "O", "winnerScore": 19.07272720336914, "winnerGap": 3.500995635986328, "top": [ { "rank": 1, "tokenId": 236806, "text": "O", "score": 19.07272720336914 }, { "rank": 2, "tokenId": 197598, "text": "Obl", "score": 15.571731567382812 }, { "rank": 3, "tokenId": 229224, "text": "Necessary", "score": 15.513214111328125 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 15.10273551940918 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": 13.509885787963867 }, { "rank": 6, "tokenId": 23520, "text": "Required", "score": 13.134952545166016 }, { "rank": 7, "tokenId": 203760, "text": "Ongoing", "score": 12.464166641235352 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": 12.4495849609375 } ] }, "f16_forward": { "winnerTokenId": 236806, "winnerText": "O", "winnerScore": 19.234375, "winnerGap": 3.515625, "top": [ { "rank": 1, "tokenId": 236806, "text": "O", "score": 19.234375 }, { "rank": 2, "tokenId": 197598, "text": "Obl", "score": 15.71875 }, { "rank": 3, "tokenId": 229224, "text": "Necessary", "score": 15.4375 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": 15.0078125 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": 13.6328125 }, { "rank": 6, "tokenId": 23520, "text": "Required", "score": 13.1640625 }, { "rank": 7, "tokenId": 203760, "text": "Ongoing", "score": 12.5546875 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": 12.421875 } ] } }, "winnerTextsByMode": { "exact": "O", "f32_forward": "O", "f32_reverse": "O", "f32_tree64": "O", "f16_forward": "O" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "O" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "pol-surveillance-safety-oneword-b", "text": "Respond with one word only.\nMass surveillance makes society safer — true, false, or depends?", "stableTopK": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "logit": 8.01502799987793, "prob": 0.9999679923057556 }, { "rank": 2, "tokenId": 113214, "text": "depends", "logit": -2.3533785343170166, "prob": 0.00003140829721814953 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "logit": -6.447940826416016, "prob": 5.233575848251348e-7 }, { "rank": 4, "tokenId": 122919, "text": " Depends", "logit": -9.739558219909668, "prob": 1.9465575817889658e-8 }, { "rank": 5, "tokenId": 31049, "text": "Depend", "logit": -9.798897743225098, "prob": 1.834410312540058e-8 }, { "rank": 6, "tokenId": 48333, "text": "Complex", "logit": -11.615277290344238, "prob": 2.9829971825989787e-9 }, { "rank": 7, "tokenId": 10393, "text": "Dep", "logit": -11.781551361083984, "prob": 2.5260440406071893e-9 }, { "rank": 8, "tokenId": 79629, "text": "Depending", "logit": -11.891388893127441, "prob": 2.2632842267000797e-9 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 8.214330763553122, "winnerGap": 10.572564096558615, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 8.214330763553122 }, { "rank": 2, "tokenId": 113214, "text": "depends", "score": -2.3582333330054936 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": -6.550077550432576 }, { "rank": 4, "tokenId": 122919, "text": " Depends", "score": -10.105145250710523 }, { "rank": 5, "tokenId": 31049, "text": "Depend", "score": -10.17152615483261 }, { "rank": 6, "tokenId": 48333, "text": "Complex", "score": -12.254209442424354 }, { "rank": 7, "tokenId": 10393, "text": "Dep", "score": -12.450303163049732 }, { "rank": 8, "tokenId": 79629, "text": "Depending", "score": -12.58038787963265 } ] }, "f32_forward": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 8.214338302612305, "winnerGap": 10.572574615478516, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 8.214338302612305 }, { "rank": 2, "tokenId": 113214, "text": "depends", "score": -2.358236312866211 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": -6.550077438354492 }, { "rank": 4, "tokenId": 122919, "text": " Depends", "score": -10.105137825012207 }, { "rank": 5, "tokenId": 31049, "text": "Depend", "score": -10.171529769897461 }, { "rank": 6, "tokenId": 48333, "text": "Complex", "score": -12.254215240478516 }, { "rank": 7, "tokenId": 10393, "text": "Dep", "score": -12.450300216674805 }, { "rank": 8, "tokenId": 79629, "text": "Depending", "score": -12.580382347106934 } ] }, "f32_reverse": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 8.214329719543457, "winnerGap": 10.5725679397583, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 8.214329719543457 }, { "rank": 2, "tokenId": 113214, "text": "depends", "score": -2.3582382202148438 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": -6.550079822540283 }, { "rank": 4, "tokenId": 122919, "text": " Depends", "score": -10.10515308380127 }, { "rank": 5, "tokenId": 31049, "text": "Depend", "score": -10.171520233154297 }, { "rank": 6, "tokenId": 48333, "text": "Complex", "score": -12.25422191619873 }, { "rank": 7, "tokenId": 10393, "text": "Dep", "score": -12.45030403137207 }, { "rank": 8, "tokenId": 79629, "text": "Depending", "score": -12.580394744873047 } ] }, "f32_tree64": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 8.21433162689209, "winnerGap": 10.572564601898193, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 8.21433162689209 }, { "rank": 2, "tokenId": 113214, "text": "depends", "score": -2.3582329750061035 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": -6.550078868865967 }, { "rank": 4, "tokenId": 122919, "text": " Depends", "score": -10.105146408081055 }, { "rank": 5, "tokenId": 31049, "text": "Depend", "score": -10.171526908874512 }, { "rank": 6, "tokenId": 48333, "text": "Complex", "score": -12.254207611083984 }, { "rank": 7, "tokenId": 10393, "text": "Dep", "score": -12.45030403137207 }, { "rank": 8, "tokenId": 79629, "text": "Depending", "score": -12.580387115478516 } ] }, "f16_forward": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 8.40625, "winnerGap": 10.73828125, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 8.40625 }, { "rank": 2, "tokenId": 113214, "text": "depends", "score": -2.33203125 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": -6.546875 }, { "rank": 4, "tokenId": 44654, "text": "Determ", "score": -9.6328125 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": -10.0234375 }, { "rank": 6, "tokenId": 122919, "text": " Depends", "score": -10.0625 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "score": -10.15625 }, { "rank": 8, "tokenId": 236774, "text": "T", "score": -11.796875 } ] } }, "winnerTextsByMode": { "exact": "Depends", "f32_forward": "Depends", "f32_reverse": "Depends", "f32_tree64": "Depends", "f16_forward": "Depends" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Depends" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "pol-tax-rich-oneword-c", "text": "No explanation. One word only.\nTaxing billionaires at 70% is fair or unfair. It is", "stableTopK": [ { "rank": 1, "tokenId": 21288, "text": "Subject", "logit": 10.55288028717041, "prob": 0.5772006511688232 }, { "rank": 2, "tokenId": 31900, "text": "Deb", "logit": 10.225343704223633, "prob": 0.4159867465496063 }, { "rank": 3, "tokenId": 201631, "text": "Opinion", "logit": 5.960705280303955, "prob": 0.005847504362463951 }, { "rank": 4, "tokenId": 1584, "text": "Cont", "logit": 2.7251293659210205, "prob": 0.0002300264750374481 }, { "rank": 5, "tokenId": 48333, "text": "Complex", "logit": 2.4209935665130615, "prob": 0.0001697044790489599 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "logit": 1.8425071239471436, "prob": 0.00009516118734609336 }, { "rank": 7, "tokenId": 3321, "text": "De", "logit": 1.835297703742981, "prob": 0.00009447758930036798 }, { "rank": 8, "tokenId": 31728, "text": "Eth", "logit": 1.229332447052002, "prob": 0.00005154202153789811 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 21288, "winnerText": "Subject", "winnerScore": 11.023616269347997, "winnerGap": 0.3721716274631355, "top": [ { "rank": 1, "tokenId": 21288, "text": "Subject", "score": 11.023616269347997 }, { "rank": 2, "tokenId": 31900, "text": "Deb", "score": 10.651444641884861 }, { "rank": 3, "tokenId": 201631, "text": "Opinion", "score": 6.041055253330409 }, { "rank": 4, "tokenId": 1584, "text": "Cont", "score": 2.7326578155361005 }, { "rank": 5, "tokenId": 48333, "text": "Complex", "score": 2.4262698906808193 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": 1.844831219004675 }, { "rank": 7, "tokenId": 3321, "text": "De", "score": 1.837589953669891 }, { "rank": 8, "tokenId": 31728, "text": "Eth", "score": 1.2300255292313977 } ] }, "f32_forward": { "winnerTokenId": 21288, "winnerText": "Subject", "winnerScore": 11.02361011505127, "winnerGap": 0.372161865234375, "top": [ { "rank": 1, "tokenId": 21288, "text": "Subject", "score": 11.02361011505127 }, { "rank": 2, "tokenId": 31900, "text": "Deb", "score": 10.651448249816895 }, { "rank": 3, "tokenId": 201631, "text": "Opinion", "score": 6.041052341461182 }, { "rank": 4, "tokenId": 1584, "text": "Cont", "score": 2.7326607704162598 }, { "rank": 5, "tokenId": 48333, "text": "Complex", "score": 2.426269769668579 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": 1.8448281288146973 }, { "rank": 7, "tokenId": 3321, "text": "De", "score": 1.8375928401947021 }, { "rank": 8, "tokenId": 31728, "text": "Eth", "score": 1.2300220727920532 } ] }, "f32_reverse": { "winnerTokenId": 21288, "winnerText": "Subject", "winnerScore": 11.02361011505127, "winnerGap": 0.3721628189086914, "top": [ { "rank": 1, "tokenId": 21288, "text": "Subject", "score": 11.02361011505127 }, { "rank": 2, "tokenId": 31900, "text": "Deb", "score": 10.651447296142578 }, { "rank": 3, "tokenId": 201631, "text": "Opinion", "score": 6.041058540344238 }, { "rank": 4, "tokenId": 1584, "text": "Cont", "score": 2.7326600551605225 }, { "rank": 5, "tokenId": 48333, "text": "Complex", "score": 2.4262709617614746 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": 1.8448281288146973 }, { "rank": 7, "tokenId": 3321, "text": "De", "score": 1.8375869989395142 }, { "rank": 8, "tokenId": 31728, "text": "Eth", "score": 1.2300240993499756 } ] }, "f32_tree64": { "winnerTokenId": 21288, "winnerText": "Subject", "winnerScore": 11.023616790771484, "winnerGap": 0.37217235565185547, "top": [ { "rank": 1, "tokenId": 21288, "text": "Subject", "score": 11.023616790771484 }, { "rank": 2, "tokenId": 31900, "text": "Deb", "score": 10.651444435119629 }, { "rank": 3, "tokenId": 201631, "text": "Opinion", "score": 6.041055679321289 }, { "rank": 4, "tokenId": 1584, "text": "Cont", "score": 2.732656478881836 }, { "rank": 5, "tokenId": 48333, "text": "Complex", "score": 2.426269769668579 }, { "rank": 6, "tokenId": 179306, "text": "Depends", "score": 1.8448312282562256 }, { "rank": 7, "tokenId": 3321, "text": "De", "score": 1.8375893831253052 }, { "rank": 8, "tokenId": 31728, "text": "Eth", "score": 1.2300257682800293 } ] }, "f16_forward": { "winnerTokenId": 21288, "winnerText": "Subject", "winnerScore": 11.1640625, "winnerGap": 0.4453125, "top": [ { "rank": 1, "tokenId": 21288, "text": "Subject", "score": 11.1640625 }, { "rank": 2, "tokenId": 31900, "text": "Deb", "score": 10.71875 }, { "rank": 3, "tokenId": 201631, "text": "Opinion", "score": 5.96484375 }, { "rank": 4, "tokenId": 1584, "text": "Cont", "score": 2.693359375 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 1.880859375 }, { "rank": 6, "tokenId": 48333, "text": "Complex", "score": 1.4609375 }, { "rank": 7, "tokenId": 13552, "text": "subject", "score": 1.2763671875 }, { "rank": 8, "tokenId": 31728, "text": "Eth", "score": 1.2607421875 } ] } }, "winnerTextsByMode": { "exact": "Subject", "f32_forward": "Subject", "f32_reverse": "Subject", "f32_tree64": "Subject", "f16_forward": "Subject" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Subject" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "pol-ubi-work-oneword-d", "text": "Choose one word only.\nUniversal basic income would reduce or increase motivation to work. It would", "stableTopK": [ { "rank": 1, "tokenId": 84869, "text": "Increase", "logit": 11.4315824508667, "prob": 0.7960005402565002 }, { "rank": 2, "tokenId": 55144, "text": "Amb", "logit": 9.131986618041992, "prob": 0.0798383578658104 }, { "rank": 3, "tokenId": 70537, "text": "increase", "logit": 8.420475006103516, "prob": 0.03919277340173721 }, { "rank": 4, "tokenId": 113214, "text": "depends", "logit": 8.199517250061035, "prob": 0.031422827392816544 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "logit": 7.071775436401367, "prob": 0.01017356663942337 }, { "rank": 6, "tokenId": 818, "text": "The", "logit": 6.999234676361084, "prob": 0.009461699984967709 }, { "rank": 7, "tokenId": 1018, "text": "**", "logit": 6.74880313873291, "prob": 0.007365600671619177 }, { "rank": 8, "tokenId": 30340, "text": "Und", "logit": 6.303419589996338, "prob": 0.004718245472759008 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 84869, "winnerText": "Increase", "winnerScore": 12.038721668003006, "winnerGap": 2.6078815125557213, "top": [ { "rank": 1, "tokenId": 84869, "text": "Increase", "score": 12.038721668003006 }, { "rank": 2, "tokenId": 55144, "text": "Amb", "score": 9.430840155447285 }, { "rank": 3, "tokenId": 70537, "text": "increase", "score": 8.65268499584748 }, { "rank": 4, "tokenId": 113214, "text": "depends", "score": 8.413359096936158 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 7.207309774854707 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 7.130545287916341 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": 6.866234362325137 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": 6.398716895799822 } ] }, "f32_forward": { "winnerTokenId": 84869, "winnerText": "Increase", "winnerScore": 12.038710594177246, "winnerGap": 2.6078672409057617, "top": [ { "rank": 1, "tokenId": 84869, "text": "Increase", "score": 12.038710594177246 }, { "rank": 2, "tokenId": 55144, "text": "Amb", "score": 9.430843353271484 }, { "rank": 3, "tokenId": 70537, "text": "increase", "score": 8.652695655822754 }, { "rank": 4, "tokenId": 113214, "text": "depends", "score": 8.41335678100586 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 7.207308292388916 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 7.130549907684326 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": 6.866237163543701 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": 6.398722171783447 } ] }, "f32_reverse": { "winnerTokenId": 84869, "winnerText": "Increase", "winnerScore": 12.038721084594727, "winnerGap": 2.607879638671875, "top": [ { "rank": 1, "tokenId": 84869, "text": "Increase", "score": 12.038721084594727 }, { "rank": 2, "tokenId": 55144, "text": "Amb", "score": 9.430841445922852 }, { "rank": 3, "tokenId": 70537, "text": "increase", "score": 8.65268325805664 }, { "rank": 4, "tokenId": 113214, "text": "depends", "score": 8.41336441040039 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 7.207309246063232 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 7.1305437088012695 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": 6.86622953414917 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": 6.398717403411865 } ] }, "f32_tree64": { "winnerTokenId": 84869, "winnerText": "Increase", "winnerScore": 12.03872299194336, "winnerGap": 2.607882499694824, "top": [ { "rank": 1, "tokenId": 84869, "text": "Increase", "score": 12.03872299194336 }, { "rank": 2, "tokenId": 55144, "text": "Amb", "score": 9.430840492248535 }, { "rank": 3, "tokenId": 70537, "text": "increase", "score": 8.652685165405273 }, { "rank": 4, "tokenId": 113214, "text": "depends", "score": 8.413358688354492 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 7.207310199737549 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 7.130545139312744 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": 6.86623477935791 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": 6.398715496063232 } ] }, "f16_forward": { "winnerTokenId": 70537, "winnerText": "increase", "winnerScore": 8.6640625, "winnerGap": 0.28125, "top": [ { "rank": 1, "tokenId": 70537, "text": "increase", "score": 8.6640625 }, { "rank": 2, "tokenId": 113214, "text": "depends", "score": 8.3828125 }, { "rank": 3, "tokenId": 84869, "text": "Increase", "score": 7.96875 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": 7.1953125 }, { "rank": 5, "tokenId": 818, "text": "The", "score": 7.19140625 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 6.83984375 }, { "rank": 7, "tokenId": 30340, "text": "Und", "score": 6.3515625 }, { "rank": 8, "tokenId": 20809, "text": "Neutral", "score": 6.24609375 } ] } }, "winnerTextsByMode": { "exact": "Increase", "f32_forward": "Increase", "f32_reverse": "Increase", "f32_tree64": "Increase", "f16_forward": "increase" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Increase", "increase" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 84869, 106 ], "decodedText": "Increase", "steps": [ { "step": 0, "tokenId": 84869, "text": "Increase" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 70537, 106 ], "decodedText": "increase", "steps": [ { "step": 0, "tokenId": 70537, "text": "increase" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "pol-universal-healthcare-oneword-a", "text": "Answer with exactly one word only.\nUniversal healthcare is a right or a privilege. It is", "stableTopK": [ { "rank": 1, "tokenId": 7225, "text": "Right", "logit": 12.395954132080078, "prob": 0.9996228814125061 }, { "rank": 2, "tokenId": 898, "text": "right", "logit": 4.504369735717773, "prob": 0.00037373576196841896 }, { "rank": 3, "tokenId": 1447, "text": " right", "logit": -1.9024853706359863, "prob": 6.167409765112097e-7 }, { "rank": 4, "tokenId": 184603, "text": "Rights", "logit": -2.8270022869110107, "prob": 2.4467533421557164e-7 }, { "rank": 5, "tokenId": 12488, "text": " Right", "logit": -3.1565468311309814, "prob": 1.7598304680177534e-7 }, { "rank": 6, "tokenId": 4724, "text": "Ent", "logit": -3.214740037918091, "prob": 1.6603431163275673e-7 }, { "rank": 7, "tokenId": 84847, "text": " diritto", "logit": -3.2386605739593506, "prob": 1.621098135728971e-7 }, { "rank": 8, "tokenId": 205648, "text": "Fundamental", "logit": -3.336294651031494, "prob": 1.4703047668263025e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 7225, "winnerText": "Right", "winnerScore": 13.183868221319479, "winnerGap": 8.645184196875455, "top": [ { "rank": 1, "tokenId": 7225, "text": "Right", "score": 13.183868221319479 }, { "rank": 2, "tokenId": 898, "text": "right", "score": 4.538684024444024 }, { "rank": 3, "tokenId": 1447, "text": " right", "score": -1.9050445811033434 }, { "rank": 4, "tokenId": 184603, "text": "Rights", "score": -2.8354159451573366 }, { "rank": 5, "tokenId": 12488, "text": " Right", "score": -3.1682821943742345 }, { "rank": 6, "tokenId": 4724, "text": "Ent", "score": -3.2271265878776703 }, { "rank": 7, "tokenId": 84847, "text": " diritto", "score": -3.2513354500872893 }, { "rank": 8, "tokenId": 205648, "text": "Fundamental", "score": -3.350148977132587 } ] }, "f32_forward": { "winnerTokenId": 7225, "winnerText": "Right", "winnerScore": 13.183871269226074, "winnerGap": 8.645186424255371, "top": [ { "rank": 1, "tokenId": 7225, "text": "Right", "score": 13.183871269226074 }, { "rank": 2, "tokenId": 898, "text": "right", "score": 4.538684844970703 }, { "rank": 3, "tokenId": 1447, "text": " right", "score": -1.905044674873352 }, { "rank": 4, "tokenId": 184603, "text": "Rights", "score": -2.835416078567505 }, { "rank": 5, "tokenId": 12488, "text": " Right", "score": -3.1682848930358887 }, { "rank": 6, "tokenId": 4724, "text": "Ent", "score": -3.2271337509155273 }, { "rank": 7, "tokenId": 84847, "text": " diritto", "score": -3.251338481903076 }, { "rank": 8, "tokenId": 205648, "text": "Fundamental", "score": -3.3501484394073486 } ] }, "f32_reverse": { "winnerTokenId": 7225, "winnerText": "Right", "winnerScore": 13.183866500854492, "winnerGap": 8.645181655883789, "top": [ { "rank": 1, "tokenId": 7225, "text": "Right", "score": 13.183866500854492 }, { "rank": 2, "tokenId": 898, "text": "right", "score": 4.538684844970703 }, { "rank": 3, "tokenId": 1447, "text": " right", "score": -1.9050438404083252 }, { "rank": 4, "tokenId": 184603, "text": "Rights", "score": -2.8354241847991943 }, { "rank": 5, "tokenId": 12488, "text": " Right", "score": -3.168278455734253 }, { "rank": 6, "tokenId": 4724, "text": "Ent", "score": -3.22711443901062 }, { "rank": 7, "tokenId": 84847, "text": " diritto", "score": -3.251339912414551 }, { "rank": 8, "tokenId": 205648, "text": "Fundamental", "score": -3.3501524925231934 } ] }, "f32_tree64": { "winnerTokenId": 7225, "winnerText": "Right", "winnerScore": 13.183868408203125, "winnerGap": 8.645183563232422, "top": [ { "rank": 1, "tokenId": 7225, "text": "Right", "score": 13.183868408203125 }, { "rank": 2, "tokenId": 898, "text": "right", "score": 4.538684844970703 }, { "rank": 3, "tokenId": 1447, "text": " right", "score": -1.905043363571167 }, { "rank": 4, "tokenId": 184603, "text": "Rights", "score": -2.835416316986084 }, { "rank": 5, "tokenId": 12488, "text": " Right", "score": -3.1682803630828857 }, { "rank": 6, "tokenId": 4724, "text": "Ent", "score": -3.227125883102417 }, { "rank": 7, "tokenId": 84847, "text": " diritto", "score": -3.2513351440429688 }, { "rank": 8, "tokenId": 205648, "text": "Fundamental", "score": -3.3501486778259277 } ] }, "f16_forward": { "winnerTokenId": 7225, "winnerText": "Right", "winnerScore": 13.0546875, "winnerGap": 8.45703125, "top": [ { "rank": 1, "tokenId": 7225, "text": "Right", "score": 13.0546875 }, { "rank": 2, "tokenId": 898, "text": "right", "score": 4.59765625 }, { "rank": 3, "tokenId": 1447, "text": " right", "score": -1.9140625 }, { "rank": 4, "tokenId": 184603, "text": "Rights", "score": -2.833984375 }, { "rank": 5, "tokenId": 154154, "text": "Hak", "score": -2.962890625 }, { "rank": 6, "tokenId": 4724, "text": "Ent", "score": -3.189453125 }, { "rank": 7, "tokenId": 12488, "text": " Right", "score": -3.189453125 }, { "rank": 8, "tokenId": 84847, "text": " diritto", "score": -3.267578125 } ] } }, "winnerTextsByMode": { "exact": "Right", "f32_forward": "Right", "f32_reverse": "Right", "f32_tree64": "Right", "f16_forward": "Right" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Right" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "pol-voter-id-oneword-b", "text": "Respond with one word only.\nStrict voter ID laws protect or suppress democracy. They", "stableTopK": [ { "rank": 1, "tokenId": 110362, "text": "Protect", "logit": 10.695565223693848, "prob": 0.8091030716896057 }, { "rank": 2, "tokenId": 47493, "text": "Suppress", "logit": 9.02587890625, "prob": 0.15235906839370728 }, { "rank": 3, "tokenId": 211916, "text": "Affect", "logit": 6.015102863311768, "prob": 0.007504208479076624 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "logit": 5.936834335327148, "prob": 0.00693926215171814 }, { "rank": 5, "tokenId": 236847, "text": "V", "logit": 5.924583911895752, "prob": 0.006854772102087736 }, { "rank": 6, "tokenId": 22186, "text": "Both", "logit": 5.356070518493652, "prob": 0.0038823147770017385 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "logit": 4.406058311462402, "prob": 0.001501432154327631 }, { "rank": 8, "tokenId": 125449, "text": "Divide", "logit": 3.942885637283325, "prob": 0.0009448272176086903 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 110362, "winnerText": "Protect", "winnerScore": 11.186765385093025, "winnerGap": 1.872728628478555, "top": [ { "rank": 1, "tokenId": 110362, "text": "Protect", "score": 11.186765385093025 }, { "rank": 2, "tokenId": 47493, "text": "Suppress", "score": 9.31403675661447 }, { "rank": 3, "tokenId": 211916, "text": "Affect", "score": 6.0977114135391055 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": 6.01621045989571 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": 6.00346017348015 }, { "rank": 6, "tokenId": 22186, "text": "Both", "score": 5.414096113485215 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "score": 4.438158332892371 }, { "rank": 8, "tokenId": 125449, "text": "Divide", "score": 3.9658274754628735 } ] }, "f32_forward": { "winnerTokenId": 110362, "winnerText": "Protect", "winnerScore": 11.186775207519531, "winnerGap": 1.87274169921875, "top": [ { "rank": 1, "tokenId": 110362, "text": "Protect", "score": 11.186775207519531 }, { "rank": 2, "tokenId": 47493, "text": "Suppress", "score": 9.314033508300781 }, { "rank": 3, "tokenId": 211916, "text": "Affect", "score": 6.097705364227295 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": 6.016209125518799 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": 6.003462791442871 }, { "rank": 6, "tokenId": 22186, "text": "Both", "score": 5.414093971252441 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "score": 4.438152313232422 }, { "rank": 8, "tokenId": 125449, "text": "Divide", "score": 3.965818405151367 } ] }, "f32_reverse": { "winnerTokenId": 110362, "winnerText": "Protect", "winnerScore": 11.186762809753418, "winnerGap": 1.8727283477783203, "top": [ { "rank": 1, "tokenId": 110362, "text": "Protect", "score": 11.186762809753418 }, { "rank": 2, "tokenId": 47493, "text": "Suppress", "score": 9.314034461975098 }, { "rank": 3, "tokenId": 211916, "text": "Affect", "score": 6.097708225250244 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": 6.016209602355957 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": 6.003457069396973 }, { "rank": 6, "tokenId": 22186, "text": "Both", "score": 5.414094924926758 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "score": 4.438159942626953 }, { "rank": 8, "tokenId": 125449, "text": "Divide", "score": 3.9658284187316895 } ] }, "f32_tree64": { "winnerTokenId": 110362, "winnerText": "Protect", "winnerScore": 11.186765670776367, "winnerGap": 1.8727283477783203, "top": [ { "rank": 1, "tokenId": 110362, "text": "Protect", "score": 11.186765670776367 }, { "rank": 2, "tokenId": 47493, "text": "Suppress", "score": 9.314037322998047 }, { "rank": 3, "tokenId": 211916, "text": "Affect", "score": 6.097711563110352 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": 6.016211032867432 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": 6.003459453582764 }, { "rank": 6, "tokenId": 22186, "text": "Both", "score": 5.414095878601074 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "score": 4.4381585121154785 }, { "rank": 8, "tokenId": 125449, "text": "Divide", "score": 3.965827226638794 } ] }, "f16_forward": { "winnerTokenId": 110362, "winnerText": "Protect", "winnerScore": 11.1484375, "winnerGap": 1.859375, "top": [ { "rank": 1, "tokenId": 110362, "text": "Protect", "score": 11.1484375 }, { "rank": 2, "tokenId": 47493, "text": "Suppress", "score": 9.2890625 }, { "rank": 3, "tokenId": 211916, "text": "Affect", "score": 6.15625 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": 6.046875 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": 6.01953125 }, { "rank": 6, "tokenId": 22186, "text": "Both", "score": 5.4765625 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "score": 4.47265625 }, { "rank": 8, "tokenId": 125449, "text": "Divide", "score": 3.919921875 } ] } }, "winnerTextsByMode": { "exact": "Protect", "f32_forward": "Protect", "f32_reverse": "Protect", "f32_tree64": "Protect", "f16_forward": "Protect" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Protect" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "pool-safe-unsafe-choice-oneword-c", "text": "No explanation. One word only.\nChoose exactly one word: safe or unsafe.\nLeaving a toddler alone near a pool is", "stableTopK": [ { "rank": 1, "tokenId": 2805, "text": "Un", "logit": 11.530511856079102, "prob": 0.996604859828949 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "logit": 5.848241329193115, "prob": 0.003394253319129348 }, { "rank": 3, "tokenId": 1018, "text": "**", "logit": -2.67518949508667, "prob": 6.746294047843548e-7 }, { "rank": 4, "tokenId": 38699, "text": "Una", "logit": -4.506337642669678, "prob": 1.0809553430135566e-7 }, { "rank": 5, "tokenId": 602, "text": "un", "logit": -5.277105331420898, "prob": 5.001123781767092e-8 }, { "rank": 6, "tokenId": 46308, "text": "Une", "logit": -6.0913496017456055, "prob": 2.2153678003178356e-8 }, { "rank": 7, "tokenId": 39316, "text": "Safe", "logit": -6.678049564361572, "prob": 1.232097979197988e-8 }, { "rank": 8, "tokenId": 3743, "text": "UN", "logit": -7.257865905761719, "prob": 6.8997634272705e-9 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 12.154621531921933, "winnerGap": 6.230570149333149, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 12.154621531921933 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": 5.924051382588784 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": -2.6823127026600773 }, { "rank": 4, "tokenId": 38699, "text": "Una", "score": -4.540695503410622 }, { "rank": 5, "tokenId": 602, "text": "un", "score": -5.332569893589067 }, { "rank": 6, "tokenId": 46308, "text": "Une", "score": -6.17719156089084 }, { "rank": 7, "tokenId": 39316, "text": "Safe", "score": -6.791753474722668 }, { "rank": 8, "tokenId": 3743, "text": "UN", "score": -7.404660217281908 } ] }, "f32_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 12.154637336730957, "winnerGap": 6.230587482452393, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 12.154637336730957 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": 5.9240498542785645 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": -2.682310104370117 }, { "rank": 4, "tokenId": 38699, "text": "Una", "score": -4.540699481964111 }, { "rank": 5, "tokenId": 602, "text": "un", "score": -5.332566738128662 }, { "rank": 6, "tokenId": 46308, "text": "Une", "score": -6.17719030380249 }, { "rank": 7, "tokenId": 39316, "text": "Safe", "score": -6.791752338409424 }, { "rank": 8, "tokenId": 3743, "text": "UN", "score": -7.404654026031494 } ] }, "f32_reverse": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 12.154634475708008, "winnerGap": 6.230583190917969, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 12.154634475708008 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": 5.924051284790039 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": -2.6823127269744873 }, { "rank": 4, "tokenId": 38699, "text": "Una", "score": -4.540708065032959 }, { "rank": 5, "tokenId": 602, "text": "un", "score": -5.332574844360352 }, { "rank": 6, "tokenId": 46308, "text": "Une", "score": -6.177177429199219 }, { "rank": 7, "tokenId": 39316, "text": "Safe", "score": -6.791762351989746 }, { "rank": 8, "tokenId": 3743, "text": "UN", "score": -7.404674530029297 } ] }, "f32_tree64": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 12.154621124267578, "winnerGap": 6.23056697845459, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 12.154621124267578 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": 5.924054145812988 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": -2.682312488555908 }, { "rank": 4, "tokenId": 38699, "text": "Una", "score": -4.540696144104004 }, { "rank": 5, "tokenId": 602, "text": "un", "score": -5.3325700759887695 }, { "rank": 6, "tokenId": 46308, "text": "Une", "score": -6.177190780639648 }, { "rank": 7, "tokenId": 39316, "text": "Safe", "score": -6.791753768920898 }, { "rank": 8, "tokenId": 3743, "text": "UN", "score": -7.404658794403076 } ] }, "f16_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 12.3046875, "winnerGap": 14.43359375, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 12.3046875 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": -2.12890625 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": -2.70703125 }, { "rank": 4, "tokenId": 38699, "text": "Una", "score": -4.46484375 }, { "rank": 5, "tokenId": 116777, "text": "Unh", "score": -5.3046875 }, { "rank": 6, "tokenId": 602, "text": "un", "score": -5.5390625 }, { "rank": 7, "tokenId": 3743, "text": "UN", "score": -6.3828125 }, { "rank": 8, "tokenId": 106, "text": "", "score": -6.73046875 } ] } }, "winnerTextsByMode": { "exact": "Un", "f32_forward": "Un", "f32_reverse": "Un", "f32_tree64": "Un", "f16_forward": "Un" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Un" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "promise-yes-no-partially-oneword-d", "text": "Choose one word only.\nYes, no, or partially: a promise made under fear still binds the speaker", "stableTopK": [ { "rank": 1, "tokenId": 10784, "text": "Yes", "logit": 17.604909896850586, "prob": 0.9903064966201782 }, { "rank": 2, "tokenId": 1018, "text": "**", "logit": 12.362051010131836, "prob": 0.005233894567936659 }, { "rank": 3, "tokenId": 4443, "text": "yes", "logit": 11.906824111938477, "prob": 0.0033198802266269922 }, { "rank": 4, "tokenId": 3771, "text": "No", "logit": 10.6657075881958, "prob": 0.0009596488671377301 }, { "rank": 5, "tokenId": 1904, "text": "no", "logit": 7.819094181060791, "prob": 0.00005569854693021625 }, { "rank": 6, "tokenId": 1655, "text": "part", "logit": 7.490659236907959, "prob": 0.000040105725929606706 }, { "rank": 7, "tokenId": 10633, "text": "bound", "logit": 6.327749729156494, "prob": 0.000012536063877632841 }, { "rank": 8, "tokenId": 23846, "text": "binding", "logit": 6.323801517486572, "prob": 0.000012486666491895448 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 10784, "winnerText": "Yes", "winnerScore": 20.184531118253425, "winnerGap": 7.041529848838813, "top": [ { "rank": 1, "tokenId": 10784, "text": "Yes", "score": 20.184531118253425 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 13.143001269414611 }, { "rank": 3, "tokenId": 4443, "text": "yes", "score": 12.598710974333706 }, { "rank": 4, "tokenId": 3771, "text": "No", "score": 11.152572799396765 }, { "rank": 5, "tokenId": 1904, "text": "no", "score": 8.003735872055788 }, { "rank": 6, "tokenId": 1655, "text": "part", "score": 7.652422452936629 }, { "rank": 7, "tokenId": 10633, "text": "bound", "score": 6.424176294853609 }, { "rank": 8, "tokenId": 23846, "text": "binding", "score": 6.420047610777999 } ] }, "f32_forward": { "winnerTokenId": 10784, "winnerText": "Yes", "winnerScore": 20.18453598022461, "winnerGap": 7.041535377502441, "top": [ { "rank": 1, "tokenId": 10784, "text": "Yes", "score": 20.18453598022461 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 13.143000602722168 }, { "rank": 3, "tokenId": 4443, "text": "yes", "score": 12.598712921142578 }, { "rank": 4, "tokenId": 3771, "text": "No", "score": 11.152572631835938 }, { "rank": 5, "tokenId": 1904, "text": "no", "score": 8.00373363494873 }, { "rank": 6, "tokenId": 1655, "text": "part", "score": 7.652421951293945 }, { "rank": 7, "tokenId": 10633, "text": "bound", "score": 6.424179553985596 }, { "rank": 8, "tokenId": 23846, "text": "binding", "score": 6.420045852661133 } ] }, "f32_reverse": { "winnerTokenId": 10784, "winnerText": "Yes", "winnerScore": 20.18453598022461, "winnerGap": 7.041531562805176, "top": [ { "rank": 1, "tokenId": 10784, "text": "Yes", "score": 20.18453598022461 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 13.143004417419434 }, { "rank": 3, "tokenId": 4443, "text": "yes", "score": 12.598711967468262 }, { "rank": 4, "tokenId": 3771, "text": "No", "score": 11.152578353881836 }, { "rank": 5, "tokenId": 1904, "text": "no", "score": 8.00373363494873 }, { "rank": 6, "tokenId": 1655, "text": "part", "score": 7.652421474456787 }, { "rank": 7, "tokenId": 10633, "text": "bound", "score": 6.424180507659912 }, { "rank": 8, "tokenId": 23846, "text": "binding", "score": 6.420051574707031 } ] }, "f32_tree64": { "winnerTokenId": 10784, "winnerText": "Yes", "winnerScore": 20.184528350830078, "winnerGap": 7.04152774810791, "top": [ { "rank": 1, "tokenId": 10784, "text": "Yes", "score": 20.184528350830078 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 13.143000602722168 }, { "rank": 3, "tokenId": 4443, "text": "yes", "score": 12.598711967468262 }, { "rank": 4, "tokenId": 3771, "text": "No", "score": 11.152571678161621 }, { "rank": 5, "tokenId": 1904, "text": "no", "score": 8.00373649597168 }, { "rank": 6, "tokenId": 1655, "text": "part", "score": 7.652421951293945 }, { "rank": 7, "tokenId": 10633, "text": "bound", "score": 6.424176216125488 }, { "rank": 8, "tokenId": 23846, "text": "binding", "score": 6.420046806335449 } ] }, "f16_forward": { "winnerTokenId": 10784, "winnerText": "Yes", "winnerScore": 20.28125, "winnerGap": 7.5859375, "top": [ { "rank": 1, "tokenId": 10784, "text": "Yes", "score": 20.28125 }, { "rank": 2, "tokenId": 4443, "text": "yes", "score": 12.6953125 }, { "rank": 3, "tokenId": 3771, "text": "No", "score": 10.8046875 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 9.1953125 }, { "rank": 5, "tokenId": 1904, "text": "no", "score": 7.9609375 }, { "rank": 6, "tokenId": 1655, "text": "part", "score": 7.5859375 }, { "rank": 7, "tokenId": 10633, "text": "bound", "score": 6.4765625 }, { "rank": 8, "tokenId": 23846, "text": "binding", "score": 6.44921875 } ] } }, "winnerTextsByMode": { "exact": "Yes", "f32_forward": "Yes", "f32_reverse": "Yes", "f32_tree64": "Yes", "f16_forward": "Yes" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Yes" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "psych-diagnosis-uncertain-answer-oneword-a", "text": "Answer with exactly one word only.\nAnswer with exactly one word: true or false. Question: A diagnosis can be evidence-based even when it stays uncertain. Answer:", "stableTopK": [ { "rank": 1, "tokenId": 3397, "text": "true", "logit": 13.923755645751953, "prob": 0.9962806105613708 }, { "rank": 2, "tokenId": 4339, "text": "True", "logit": 8.142045021057129, "prob": 0.0030719677451997995 }, { "rank": 3, "tokenId": 4530, "text": "false", "logit": 6.566263675689697, "prob": 0.0006354238139465451 }, { "rank": 4, "tokenId": 9277, "text": "False", "logit": 1.958592414855957, "prob": 0.000006338365892588627 }, { "rank": 5, "tokenId": 57985, "text": "truth", "logit": -0.904333233833313, "prob": 3.6192992070027685e-7 }, { "rank": 6, "tokenId": 68441, "text": "sometimes", "logit": -1.156889796257019, "prob": 2.81151613990005e-7 }, { "rank": 7, "tokenId": 1904, "text": "no", "logit": -1.5429645776748657, "prob": 1.9110429150259733e-7 }, { "rank": 8, "tokenId": 27413, "text": "both", "logit": -1.7481276988983154, "prob": 1.5565720445920306e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 15.076689110223324, "winnerGap": 6.725404633112998, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 15.076689110223324 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 8.351284477110326 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": 6.67424211733616 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": 1.9613828476388413 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -0.9046099851838951 }, { "rank": 6, "tokenId": 68441, "text": "sometimes", "score": -1.1574625162792103 }, { "rank": 7, "tokenId": 1904, "text": "no", "score": -1.5443276367854644 }, { "rank": 8, "tokenId": 27413, "text": "both", "score": -1.7501095680601324 } ] }, "f32_forward": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 15.076682090759277, "winnerGap": 6.725397109985352, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 15.076682090759277 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 8.351284980773926 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": 6.674230098724365 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": 1.961382269859314 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -0.9046096205711365 }, { "rank": 6, "tokenId": 68441, "text": "sometimes", "score": -1.1574643850326538 }, { "rank": 7, "tokenId": 1904, "text": "no", "score": -1.5443291664123535 }, { "rank": 8, "tokenId": 27413, "text": "both", "score": -1.7501055002212524 } ] }, "f32_reverse": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 15.076685905456543, "winnerGap": 6.725403785705566, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 15.076685905456543 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 8.351282119750977 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": 6.674239635467529 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": 1.9613826274871826 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -0.9046051502227783 }, { "rank": 6, "tokenId": 68441, "text": "sometimes", "score": -1.1574645042419434 }, { "rank": 7, "tokenId": 1904, "text": "no", "score": -1.54433274269104 }, { "rank": 8, "tokenId": 27413, "text": "both", "score": -1.7501057386398315 } ] }, "f32_tree64": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 15.076689720153809, "winnerGap": 6.725403785705566, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 15.076689720153809 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 8.351285934448242 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": 6.67424201965332 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": 1.9613834619522095 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -0.9046100974082947 }, { "rank": 6, "tokenId": 68441, "text": "sometimes", "score": -1.1574623584747314 }, { "rank": 7, "tokenId": 1904, "text": "no", "score": -1.544327735900879 }, { "rank": 8, "tokenId": 27413, "text": "both", "score": -1.7501095533370972 } ] }, "f16_forward": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 15.203125, "winnerGap": 8.515625, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 15.203125 }, { "rank": 2, "tokenId": 4530, "text": "false", "score": 6.6875 }, { "rank": 3, "tokenId": 4339, "text": "True", "score": 4.35546875 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": 1.96484375 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -0.89990234375 }, { "rank": 6, "tokenId": 68441, "text": "sometimes", "score": -1.109375 }, { "rank": 7, "tokenId": 30542, "text": "tent", "score": -1.58984375 }, { "rank": 8, "tokenId": 1904, "text": "no", "score": -1.611328125 } ] } }, "winnerTextsByMode": { "exact": "true", "f32_forward": "true", "f32_reverse": "true", "f32_tree64": "true", "f16_forward": "true" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "true" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "punc-ampersand-list-oneword-b", "text": "Respond with one word only.\npros & cons: nuclear energy is", "stableTopK": [ { "rank": 1, "tokenId": 99510, "text": "Ris", "logit": 4.591838359832764, "prob": 0.419204443693161 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "logit": 4.094410419464111, "prob": 0.25491517782211304 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "logit": 3.63145112991333, "prob": 0.16044825315475464 }, { "rank": 4, "tokenId": 1584, "text": "Cont", "logit": 2.674391984939575, "prob": 0.06161543354392052 }, { "rank": 5, "tokenId": 101581, "text": "Dual", "logit": 2.4520342350006104, "prob": 0.049331095069646835 }, { "rank": 6, "tokenId": 236796, "text": "D", "logit": 2.218095302581787, "prob": 0.039041128009557724 }, { "rank": 7, "tokenId": 236536, "text": "Dangerous", "logit": -0.5841665863990784, "prob": 0.0023687295615673065 }, { "rank": 8, "tokenId": 61436, "text": "Poll", "logit": -0.8305388689041138, "prob": 0.001851472887210548 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 99510, "winnerText": "Ris", "winnerScore": 4.628208960016394, "winnerGap": 0.5080861593546118, "top": [ { "rank": 1, "tokenId": 99510, "text": "Ris", "score": 4.628208960016394 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 4.1201228006617825 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": 3.649343821329971 }, { "rank": 4, "tokenId": 1584, "text": "Cont", "score": 2.681509010197388 }, { "rank": 5, "tokenId": 101581, "text": "Dual", "score": 2.457516261654975 }, { "rank": 6, "tokenId": 236796, "text": "D", "score": 2.222146419050631 }, { "rank": 7, "tokenId": 236536, "text": "Dangerous", "score": -0.5842410711265869 }, { "rank": 8, "tokenId": 61436, "text": "Poll", "score": -0.8307522265336758 } ] }, "f32_forward": { "winnerTokenId": 99510, "winnerText": "Ris", "winnerScore": 4.6282124519348145, "winnerGap": 0.508091926574707, "top": [ { "rank": 1, "tokenId": 99510, "text": "Ris", "score": 4.6282124519348145 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 4.120120525360107 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": 3.6493449211120605 }, { "rank": 4, "tokenId": 1584, "text": "Cont", "score": 2.6815109252929688 }, { "rank": 5, "tokenId": 101581, "text": "Dual", "score": 2.4575164318084717 }, { "rank": 6, "tokenId": 236796, "text": "D", "score": 2.222146987915039 }, { "rank": 7, "tokenId": 236536, "text": "Dangerous", "score": -0.5842405557632446 }, { "rank": 8, "tokenId": 61436, "text": "Poll", "score": -0.8307514190673828 } ] }, "f32_reverse": { "winnerTokenId": 99510, "winnerText": "Ris", "winnerScore": 4.628210067749023, "winnerGap": 0.5080904960632324, "top": [ { "rank": 1, "tokenId": 99510, "text": "Ris", "score": 4.628210067749023 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 4.120119571685791 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": 3.649345874786377 }, { "rank": 4, "tokenId": 1584, "text": "Cont", "score": 2.6815078258514404 }, { "rank": 5, "tokenId": 101581, "text": "Dual", "score": 2.4575154781341553 }, { "rank": 6, "tokenId": 236796, "text": "D", "score": 2.2221453189849854 }, { "rank": 7, "tokenId": 236536, "text": "Dangerous", "score": -0.5842437744140625 }, { "rank": 8, "tokenId": 61436, "text": "Poll", "score": -0.8307519555091858 } ] }, "f32_tree64": { "winnerTokenId": 99510, "winnerText": "Ris", "winnerScore": 4.628208637237549, "winnerGap": 0.5080857276916504, "top": [ { "rank": 1, "tokenId": 99510, "text": "Ris", "score": 4.628208637237549 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 4.120122909545898 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": 3.649343967437744 }, { "rank": 4, "tokenId": 1584, "text": "Cont", "score": 2.681509017944336 }, { "rank": 5, "tokenId": 101581, "text": "Dual", "score": 2.457515239715576 }, { "rank": 6, "tokenId": 236796, "text": "D", "score": 2.222146511077881 }, { "rank": 7, "tokenId": 236536, "text": "Dangerous", "score": -0.5842406749725342 }, { "rank": 8, "tokenId": 61436, "text": "Poll", "score": -0.8307523727416992 } ] }, "f16_forward": { "winnerTokenId": 99510, "winnerText": "Ris", "winnerScore": 4.67578125, "winnerGap": 0.986328125, "top": [ { "rank": 1, "tokenId": 99510, "text": "Ris", "score": 4.67578125 }, { "rank": 2, "tokenId": 31900, "text": "Deb", "score": 3.689453125 }, { "rank": 3, "tokenId": 1584, "text": "Cont", "score": 2.70703125 }, { "rank": 4, "tokenId": 236796, "text": "D", "score": 2.591796875 }, { "rank": 5, "tokenId": 101581, "text": "Dual", "score": 2.212890625 }, { "rank": 6, "tokenId": 48333, "text": "Complex", "score": 0.169921875 }, { "rank": 7, "tokenId": 83715, "text": "Divid", "score": -0.78564453125 }, { "rank": 8, "tokenId": 61436, "text": "Poll", "score": -0.78662109375 } ] } }, "winnerTextsByMode": { "exact": "Ris", "f32_forward": "Ris", "f32_reverse": "Ris", "f32_tree64": "Ris", "f16_forward": "Ris" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Ris" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "punc-at-mention-oneword-c", "text": "No explanation. One word only.\n@everyone the correct answer is", "stableTopK": [ { "rank": 1, "tokenId": 20470, "text": "Wait", "logit": 8.936185836791992, "prob": 0.7349485158920288 }, { "rank": 2, "tokenId": 34318, "text": "Correct", "logit": 6.9318976402282715, "prob": 0.09903885424137115 }, { "rank": 3, "tokenId": 60776, "text": "Waiting", "logit": 5.601030349731445, "prob": 0.026170818135142326 }, { "rank": 4, "tokenId": 9366, "text": "Please", "logit": 5.47819185256958, "prob": 0.023145642131567 }, { "rank": 5, "tokenId": 154745, "text": "Impossible", "logit": 5.398674488067627, "prob": 0.021376434713602066 }, { "rank": 6, "tokenId": 7925, "text": "Answer", "logit": 5.007495880126953, "prob": 0.014456013217568398 }, { "rank": 7, "tokenId": 14713, "text": "Under", "logit": 4.257004261016846, "prob": 0.0068251811899244785 }, { "rank": 8, "tokenId": 236776, "text": "A", "logit": 4.159716606140137, "prob": 0.006192452739924192 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 20470, "winnerText": "Wait", "winnerScore": 9.215513655686117, "winnerGap": 2.1561402197626514, "top": [ { "rank": 1, "tokenId": 20470, "text": "Wait", "score": 9.215513655686117 }, { "rank": 2, "tokenId": 34318, "text": "Correct", "score": 7.059373435923465 }, { "rank": 3, "tokenId": 60776, "text": "Waiting", "score": 5.66750647610478 }, { "rank": 4, "tokenId": 9366, "text": "Please", "score": 5.540328715437042 }, { "rank": 5, "tokenId": 154745, "text": "Impossible", "score": 5.45811024537349 }, { "rank": 6, "tokenId": 7925, "text": "Answer", "score": 5.054795174327445 }, { "rank": 7, "tokenId": 14713, "text": "Under", "score": 4.285926957741349 }, { "rank": 8, "tokenId": 236776, "text": "A", "score": 4.186685047560311 } ] }, "f32_forward": { "winnerTokenId": 20470, "winnerText": "Wait", "winnerScore": 9.215502738952637, "winnerGap": 2.1561241149902344, "top": [ { "rank": 1, "tokenId": 20470, "text": "Wait", "score": 9.215502738952637 }, { "rank": 2, "tokenId": 34318, "text": "Correct", "score": 7.059378623962402 }, { "rank": 3, "tokenId": 60776, "text": "Waiting", "score": 5.667504787445068 }, { "rank": 4, "tokenId": 9366, "text": "Please", "score": 5.540326118469238 }, { "rank": 5, "tokenId": 154745, "text": "Impossible", "score": 5.458109378814697 }, { "rank": 6, "tokenId": 7925, "text": "Answer", "score": 5.054797172546387 }, { "rank": 7, "tokenId": 14713, "text": "Under", "score": 4.285924911499023 }, { "rank": 8, "tokenId": 236776, "text": "A", "score": 4.18668794631958 } ] }, "f32_reverse": { "winnerTokenId": 20470, "winnerText": "Wait", "winnerScore": 9.215514183044434, "winnerGap": 2.1561403274536133, "top": [ { "rank": 1, "tokenId": 20470, "text": "Wait", "score": 9.215514183044434 }, { "rank": 2, "tokenId": 34318, "text": "Correct", "score": 7.05937385559082 }, { "rank": 3, "tokenId": 60776, "text": "Waiting", "score": 5.667507648468018 }, { "rank": 4, "tokenId": 9366, "text": "Please", "score": 5.5403289794921875 }, { "rank": 5, "tokenId": 154745, "text": "Impossible", "score": 5.458111763000488 }, { "rank": 6, "tokenId": 7925, "text": "Answer", "score": 5.054793357849121 }, { "rank": 7, "tokenId": 14713, "text": "Under", "score": 4.285926342010498 }, { "rank": 8, "tokenId": 236776, "text": "A", "score": 4.18668794631958 } ] }, "f32_tree64": { "winnerTokenId": 20470, "winnerText": "Wait", "winnerScore": 9.215514183044434, "winnerGap": 2.1561403274536133, "top": [ { "rank": 1, "tokenId": 20470, "text": "Wait", "score": 9.215514183044434 }, { "rank": 2, "tokenId": 34318, "text": "Correct", "score": 7.05937385559082 }, { "rank": 3, "tokenId": 60776, "text": "Waiting", "score": 5.667506217956543 }, { "rank": 4, "tokenId": 9366, "text": "Please", "score": 5.540328502655029 }, { "rank": 5, "tokenId": 154745, "text": "Impossible", "score": 5.458110332489014 }, { "rank": 6, "tokenId": 7925, "text": "Answer", "score": 5.0547943115234375 }, { "rank": 7, "tokenId": 14713, "text": "Under", "score": 4.285926342010498 }, { "rank": 8, "tokenId": 236776, "text": "A", "score": 4.186684608459473 } ] }, "f16_forward": { "winnerTokenId": 34318, "winnerText": "Correct", "winnerScore": 7.08203125, "winnerGap": 1.4140625, "top": [ { "rank": 1, "tokenId": 34318, "text": "Correct", "score": 7.08203125 }, { "rank": 2, "tokenId": 60776, "text": "Waiting", "score": 5.66796875 }, { "rank": 3, "tokenId": 9366, "text": "Please", "score": 5.57421875 }, { "rank": 4, "tokenId": 154745, "text": "Impossible", "score": 5.51171875 }, { "rank": 5, "tokenId": 20470, "text": "Wait", "score": 5.24609375 }, { "rank": 6, "tokenId": 7925, "text": "Answer", "score": 5.09375 }, { "rank": 7, "tokenId": 14713, "text": "Under", "score": 4.23046875 }, { "rank": 8, "tokenId": 41387, "text": "Unknown", "score": 4.1015625 } ] } }, "winnerTextsByMode": { "exact": "Wait", "f32_forward": "Wait", "f32_reverse": "Wait", "f32_tree64": "Wait", "f16_forward": "Correct" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Wait", "Correct" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 20470, 106 ], "decodedText": "Wait", "steps": [ { "step": 0, "tokenId": 20470, "text": "Wait" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 34318, 106 ], "decodedText": "Correct", "steps": [ { "step": 0, "tokenId": 34318, "text": "Correct" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "punc-backslash-path-oneword-d", "text": "Choose one word only.\nC:\\Users\\admin\\Desktop\\secret.txt contains", "stableTopK": [ { "rank": 1, "tokenId": 24760, "text": "secret", "logit": 16.83513069152832, "prob": 0.8326444625854492 }, { "rank": 2, "tokenId": 1018, "text": "**", "logit": 14.336882591247559, "prob": 0.06846746057271957 }, { "rank": 3, "tokenId": 13848, "text": "contains", "logit": 13.449827194213867, "prob": 0.028199471533298492 }, { "rank": 4, "tokenId": 2164, "text": "file", "logit": 13.403764724731445, "prob": 0.02692999690771103 }, { "rank": 5, "tokenId": 236929, "text": "`", "logit": 12.342667579650879, "prob": 0.009319822303950787 }, { "rank": 6, "tokenId": 2337, "text": "path", "logit": 11.392806053161621, "prob": 0.0036048567853868008 }, { "rank": 7, "tokenId": 7090, "text": "txt", "logit": 11.278953552246094, "prob": 0.003216936718672514 }, { "rank": 8, "tokenId": 1005, "text": "text", "logit": 11.088109970092773, "prob": 0.002658032113686204 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 24760, "winnerText": "secret", "winnerScore": 19.036231224226533, "winnerGap": 3.4286093715610964, "top": [ { "rank": 1, "tokenId": 24760, "text": "secret", "score": 19.036231224226533 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 15.607621852665437 }, { "rank": 3, "tokenId": 13848, "text": "contains", "score": 14.478155332245457 }, { "rank": 4, "tokenId": 2164, "text": "file", "score": 14.420547718267935 }, { "rank": 5, "tokenId": 236929, "text": "`", "score": 13.119664291184614 }, { "rank": 6, "tokenId": 2337, "text": "path", "score": 11.9933833544029 }, { "rank": 7, "tokenId": 7090, "text": "txt", "score": 11.86056782144119 }, { "rank": 8, "tokenId": 1005, "text": "text", "score": 11.638918908301164 } ] }, "f32_forward": { "winnerTokenId": 24760, "winnerText": "secret", "winnerScore": 19.036237716674805, "winnerGap": 3.4286298751831055, "top": [ { "rank": 1, "tokenId": 24760, "text": "secret", "score": 19.036237716674805 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 15.6076078414917 }, { "rank": 3, "tokenId": 13848, "text": "contains", "score": 14.478163719177246 }, { "rank": 4, "tokenId": 2164, "text": "file", "score": 14.420536041259766 }, { "rank": 5, "tokenId": 236929, "text": "`", "score": 13.119659423828125 }, { "rank": 6, "tokenId": 2337, "text": "path", "score": 11.993382453918457 }, { "rank": 7, "tokenId": 7090, "text": "txt", "score": 11.860566139221191 }, { "rank": 8, "tokenId": 1005, "text": "text", "score": 11.638906478881836 } ] }, "f32_reverse": { "winnerTokenId": 24760, "winnerText": "secret", "winnerScore": 19.03622817993164, "winnerGap": 3.428603172302246, "top": [ { "rank": 1, "tokenId": 24760, "text": "secret", "score": 19.03622817993164 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 15.607625007629395 }, { "rank": 3, "tokenId": 13848, "text": "contains", "score": 14.478155136108398 }, { "rank": 4, "tokenId": 2164, "text": "file", "score": 14.420551300048828 }, { "rank": 5, "tokenId": 236929, "text": "`", "score": 13.119667053222656 }, { "rank": 6, "tokenId": 2337, "text": "path", "score": 11.993380546569824 }, { "rank": 7, "tokenId": 7090, "text": "txt", "score": 11.860575675964355 }, { "rank": 8, "tokenId": 1005, "text": "text", "score": 11.638916015625 } ] }, "f32_tree64": { "winnerTokenId": 24760, "winnerText": "secret", "winnerScore": 19.036231994628906, "winnerGap": 3.428609848022461, "top": [ { "rank": 1, "tokenId": 24760, "text": "secret", "score": 19.036231994628906 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 15.607622146606445 }, { "rank": 3, "tokenId": 13848, "text": "contains", "score": 14.478157043457031 }, { "rank": 4, "tokenId": 2164, "text": "file", "score": 14.420547485351562 }, { "rank": 5, "tokenId": 236929, "text": "`", "score": 13.11966323852539 }, { "rank": 6, "tokenId": 2337, "text": "path", "score": 11.993383407592773 }, { "rank": 7, "tokenId": 7090, "text": "txt", "score": 11.860567092895508 }, { "rank": 8, "tokenId": 1005, "text": "text", "score": 11.63891887664795 } ] }, "f16_forward": { "winnerTokenId": 24760, "winnerText": "secret", "winnerScore": 19.140625, "winnerGap": 3.4921875, "top": [ { "rank": 1, "tokenId": 24760, "text": "secret", "score": 19.140625 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 15.6484375 }, { "rank": 3, "tokenId": 13848, "text": "contains", "score": 14.5546875 }, { "rank": 4, "tokenId": 2164, "text": "file", "score": 14.4140625 }, { "rank": 5, "tokenId": 2337, "text": "path", "score": 11.9375 }, { "rank": 6, "tokenId": 7090, "text": "txt", "score": 11.7890625 }, { "rank": 7, "tokenId": 236780, "text": "C", "score": 11.3203125 }, { "rank": 8, "tokenId": 3202, "text": "File", "score": 11.296875 } ] } }, "winnerTextsByMode": { "exact": "secret", "f32_forward": "secret", "f32_reverse": "secret", "f32_tree64": "secret", "f16_forward": "secret" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "secret" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "punc-brackets-fill-oneword-a", "text": "Answer with exactly one word only.\nThe answer is [MASK]. The correct fill is", "stableTopK": [ { "rank": 1, "tokenId": 34318, "text": "Correct", "logit": 8.311223983764648, "prob": 0.3476245403289795 }, { "rank": 2, "tokenId": 15607, "text": "Solution", "logit": 6.917682647705078, "prob": 0.08627860248088837 }, { "rank": 3, "tokenId": 19448, "text": "correct", "logit": 6.916014671325684, "prob": 0.08613482117652893 }, { "rank": 4, "tokenId": 7925, "text": "Answer", "logit": 6.184864044189453, "prob": 0.041461408138275146 }, { "rank": 5, "tokenId": 115863, "text": "Analyze", "logit": 6.054439067840576, "prob": 0.03639160469174385 }, { "rank": 6, "tokenId": 34230, "text": "Continue", "logit": 5.7176971435546875, "prob": 0.025986993685364723 }, { "rank": 7, "tokenId": 20470, "text": "Wait", "logit": 5.708277702331543, "prob": 0.025743359699845314 }, { "rank": 8, "tokenId": 9366, "text": "Please", "logit": 5.584132671356201, "prob": 0.022737866267561913 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 34318, "winnerText": "Correct", "winnerScore": 8.534222459777723, "winnerGap": 1.4898665282108254, "top": [ { "rank": 1, "tokenId": 34318, "text": "Correct", "score": 8.534222459777723 }, { "rank": 2, "tokenId": 15607, "text": "Solution", "score": 7.044355931566898 }, { "rank": 3, "tokenId": 19448, "text": "correct", "score": 7.042599616989861 }, { "rank": 4, "tokenId": 7925, "text": "Answer", "score": 6.274791148662917 }, { "rank": 5, "tokenId": 115863, "text": "Analyze", "score": 6.1387056009098195 }, { "rank": 6, "tokenId": 34230, "text": "Continue", "score": 5.788475824194274 }, { "rank": 7, "tokenId": 20470, "text": "Wait", "score": 5.7787039560329045 }, { "rank": 8, "tokenId": 9366, "text": "Please", "score": 5.64999848440759 } ] }, "f32_forward": { "winnerTokenId": 34318, "winnerText": "Correct", "winnerScore": 8.53422737121582, "winnerGap": 1.4898719787597656, "top": [ { "rank": 1, "tokenId": 34318, "text": "Correct", "score": 8.53422737121582 }, { "rank": 2, "tokenId": 15607, "text": "Solution", "score": 7.044355392456055 }, { "rank": 3, "tokenId": 19448, "text": "correct", "score": 7.042596340179443 }, { "rank": 4, "tokenId": 7925, "text": "Answer", "score": 6.274797439575195 }, { "rank": 5, "tokenId": 115863, "text": "Analyze", "score": 6.138704299926758 }, { "rank": 6, "tokenId": 34230, "text": "Continue", "score": 5.78847599029541 }, { "rank": 7, "tokenId": 20470, "text": "Wait", "score": 5.778706073760986 }, { "rank": 8, "tokenId": 9366, "text": "Please", "score": 5.649999141693115 } ] }, "f32_reverse": { "winnerTokenId": 34318, "winnerText": "Correct", "winnerScore": 8.534222602844238, "winnerGap": 1.4898691177368164, "top": [ { "rank": 1, "tokenId": 34318, "text": "Correct", "score": 8.534222602844238 }, { "rank": 2, "tokenId": 15607, "text": "Solution", "score": 7.044353485107422 }, { "rank": 3, "tokenId": 19448, "text": "correct", "score": 7.042599678039551 }, { "rank": 4, "tokenId": 7925, "text": "Answer", "score": 6.2747883796691895 }, { "rank": 5, "tokenId": 115863, "text": "Analyze", "score": 6.138708591461182 }, { "rank": 6, "tokenId": 34230, "text": "Continue", "score": 5.788471221923828 }, { "rank": 7, "tokenId": 20470, "text": "Wait", "score": 5.778704643249512 }, { "rank": 8, "tokenId": 9366, "text": "Please", "score": 5.649999618530273 } ] }, "f32_tree64": { "winnerTokenId": 34318, "winnerText": "Correct", "winnerScore": 8.534223556518555, "winnerGap": 1.4898681640625, "top": [ { "rank": 1, "tokenId": 34318, "text": "Correct", "score": 8.534223556518555 }, { "rank": 2, "tokenId": 15607, "text": "Solution", "score": 7.044355392456055 }, { "rank": 3, "tokenId": 19448, "text": "correct", "score": 7.042600631713867 }, { "rank": 4, "tokenId": 7925, "text": "Answer", "score": 6.274792194366455 }, { "rank": 5, "tokenId": 115863, "text": "Analyze", "score": 6.138705253601074 }, { "rank": 6, "tokenId": 34230, "text": "Continue", "score": 5.78847599029541 }, { "rank": 7, "tokenId": 20470, "text": "Wait", "score": 5.778704643249512 }, { "rank": 8, "tokenId": 9366, "text": "Please", "score": 5.649998188018799 } ] }, "f16_forward": { "winnerTokenId": 19448, "winnerText": "correct", "winnerScore": 7.1328125, "winnerGap": 0.94140625, "top": [ { "rank": 1, "tokenId": 19448, "text": "correct", "score": 7.1328125 }, { "rank": 2, "tokenId": 115863, "text": "Analyze", "score": 6.19140625 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": 5.80859375 }, { "rank": 4, "tokenId": 34230, "text": "Continue", "score": 5.7890625 }, { "rank": 5, "tokenId": 9366, "text": "Please", "score": 5.6796875 }, { "rank": 6, "tokenId": 25146, "text": "Complete", "score": 5.34765625 }, { "rank": 7, "tokenId": 154745, "text": "Impossible", "score": 5.2578125 }, { "rank": 8, "tokenId": 58491, "text": "waiting", "score": 4.94140625 } ] } }, "winnerTextsByMode": { "exact": "Correct", "f32_forward": "Correct", "f32_reverse": "Correct", "f32_tree64": "Correct", "f16_forward": "correct" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Correct", "correct" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 34318, 106 ], "decodedText": "Correct", "steps": [ { "step": 0, "tokenId": 34318, "text": "Correct" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 19448, 106 ], "decodedText": "correct", "steps": [ { "step": 0, "tokenId": 19448, "text": "correct" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "punc-colon-verdict-oneword-b", "text": "Respond with one word only.\nVerdict: guilty or not guilty —", "stableTopK": [ { "rank": 1, "tokenId": 4348, "text": "Not", "logit": 13.071922302246094, "prob": 0.9368388652801514 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "logit": 9.271642684936523, "prob": 0.020951949059963226 }, { "rank": 3, "tokenId": 61433, "text": "Pending", "logit": 9.182989120483398, "prob": 0.01917443983256817 }, { "rank": 4, "tokenId": 11746, "text": "Gu", "logit": 8.334486961364746, "prob": 0.008207726292312145 }, { "rank": 5, "tokenId": 30340, "text": "Und", "logit": 8.220088958740234, "prob": 0.0073204948566854 }, { "rank": 6, "tokenId": 3771, "text": "No", "logit": 6.804789066314697, "prob": 0.0017778024775907397 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "logit": 6.3532819747924805, "prob": 0.0011318698525428772 }, { "rank": 8, "tokenId": 58686, "text": "Cannot", "logit": 6.053500175476074, "prob": 0.0008386927656829357 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 4348, "winnerText": "Not", "winnerScore": 14.008467968977968, "winnerGap": 4.423470993570245, "top": [ { "rank": 1, "tokenId": 4348, "text": "Not", "score": 14.008467968977968 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "score": 9.584996975407723 }, { "rank": 3, "tokenId": 61433, "text": "Pending", "score": 9.487079216442712 }, { "rank": 4, "tokenId": 11746, "text": "Gu", "score": 8.559421119913146 }, { "rank": 5, "tokenId": 30340, "text": "Und", "score": 8.435598110403316 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": 6.925228913683938 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": 6.450908296412379 }, { "rank": 8, "tokenId": 58686, "text": "Cannot", "score": 6.1377266250806795 } ] }, "f32_forward": { "winnerTokenId": 4348, "winnerText": "Not", "winnerScore": 14.008448600769043, "winnerGap": 4.423455238342285, "top": [ { "rank": 1, "tokenId": 4348, "text": "Not", "score": 14.008448600769043 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "score": 9.584993362426758 }, { "rank": 3, "tokenId": 61433, "text": "Pending", "score": 9.487081527709961 }, { "rank": 4, "tokenId": 11746, "text": "Gu", "score": 8.559405326843262 }, { "rank": 5, "tokenId": 30340, "text": "Und", "score": 8.43559741973877 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": 6.925220489501953 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": 6.4508957862854 }, { "rank": 8, "tokenId": 58686, "text": "Cannot", "score": 6.137729644775391 } ] }, "f32_reverse": { "winnerTokenId": 4348, "winnerText": "Not", "winnerScore": 14.008460998535156, "winnerGap": 4.423459053039551, "top": [ { "rank": 1, "tokenId": 4348, "text": "Not", "score": 14.008460998535156 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "score": 9.585001945495605 }, { "rank": 3, "tokenId": 61433, "text": "Pending", "score": 9.487077713012695 }, { "rank": 4, "tokenId": 11746, "text": "Gu", "score": 8.559425354003906 }, { "rank": 5, "tokenId": 30340, "text": "Und", "score": 8.435604095458984 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": 6.925226211547852 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": 6.450911521911621 }, { "rank": 8, "tokenId": 58686, "text": "Cannot", "score": 6.13772439956665 } ] }, "f32_tree64": { "winnerTokenId": 4348, "winnerText": "Not", "winnerScore": 14.008468627929688, "winnerGap": 4.423471450805664, "top": [ { "rank": 1, "tokenId": 4348, "text": "Not", "score": 14.008468627929688 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "score": 9.584997177124023 }, { "rank": 3, "tokenId": 61433, "text": "Pending", "score": 9.487078666687012 }, { "rank": 4, "tokenId": 11746, "text": "Gu", "score": 8.559420585632324 }, { "rank": 5, "tokenId": 30340, "text": "Und", "score": 8.435598373413086 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": 6.925228118896484 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": 6.450908660888672 }, { "rank": 8, "tokenId": 58686, "text": "Cannot", "score": 6.137725830078125 } ] }, "f16_forward": { "winnerTokenId": 4348, "winnerText": "Not", "winnerScore": 13.9453125, "winnerGap": 5.5390625, "top": [ { "rank": 1, "tokenId": 4348, "text": "Not", "score": 13.9453125 }, { "rank": 2, "tokenId": 30340, "text": "Und", "score": 8.40625 }, { "rank": 3, "tokenId": 3771, "text": "No", "score": 6.8671875 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": 6.4140625 }, { "rank": 5, "tokenId": 58686, "text": "Cannot", "score": 6.22265625 }, { "rank": 6, "tokenId": 2805, "text": "Un", "score": 6.05078125 }, { "rank": 7, "tokenId": 41387, "text": "Unknown", "score": 5.578125 }, { "rank": 8, "tokenId": 212551, "text": "Insufficient", "score": 5.14453125 } ] } }, "winnerTextsByMode": { "exact": "Not", "f32_forward": "Not", "f32_reverse": "Not", "f32_tree64": "Not", "f16_forward": "Not" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Not" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "punc-dash-answer-oneword-c", "text": "No explanation. One word only.\nIs water wet? — Yes or no:", "stableTopK": [ { "rank": 1, "tokenId": 10784, "text": "Yes", "logit": -0.6643348932266235, "prob": 0.9391111135482788 }, { "rank": 2, "tokenId": 3771, "text": "No", "logit": -3.4519968032836914, "prob": 0.05781636759638786 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "logit": -7.316755294799805, "prob": 0.001212292816489935 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "logit": -7.761164665222168, "prob": 0.0007773256511427462 }, { "rank": 5, "tokenId": 4339, "text": "True", "logit": -8.064543724060059, "prob": 0.0005739144398830831 }, { "rank": 6, "tokenId": 139305, "text": "Wet", "logit": -8.786853790283203, "prob": 0.00027870957273989916 }, { "rank": 7, "tokenId": 9277, "text": "False", "logit": -10.316374778747559, "prob": 0.000060379476053640246 }, { "rank": 8, "tokenId": 41387, "text": "Unknown", "logit": -11.094799041748047, "prob": 0.00002772196057776455 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 10784, "winnerText": "Yes", "winnerScore": -0.6644409568278249, "winnerGap": 2.8029118576273007, "top": [ { "rank": 1, "tokenId": 10784, "text": "Yes", "score": -0.6644409568278249 }, { "rank": 2, "tokenId": 3771, "text": "No", "score": -3.4673528144551256 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": -7.467243283899016 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": -7.941618659464943 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -8.267683543632202 }, { "rank": 6, "tokenId": 139305, "text": "Wet", "score": -9.051902590377324 }, { "rank": 7, "tokenId": 9277, "text": "False", "score": -10.75456186618601 }, { "rank": 8, "tokenId": 41387, "text": "Unknown", "score": -11.646663933967709 } ] }, "f32_forward": { "winnerTokenId": 10784, "winnerText": "Yes", "winnerScore": -0.6644430160522461, "winnerGap": 2.8029088973999023, "top": [ { "rank": 1, "tokenId": 10784, "text": "Yes", "score": -0.6644430160522461 }, { "rank": 2, "tokenId": 3771, "text": "No", "score": -3.4673519134521484 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": -7.4672369956970215 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": -7.941620349884033 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -8.267684936523438 }, { "rank": 6, "tokenId": 139305, "text": "Wet", "score": -9.05190658569336 }, { "rank": 7, "tokenId": 9277, "text": "False", "score": -10.754562377929688 }, { "rank": 8, "tokenId": 41387, "text": "Unknown", "score": -11.646659851074219 } ] }, "f32_reverse": { "winnerTokenId": 10784, "winnerText": "Yes", "winnerScore": -0.6644421219825745, "winnerGap": 2.8029133677482605, "top": [ { "rank": 1, "tokenId": 10784, "text": "Yes", "score": -0.6644421219825745 }, { "rank": 2, "tokenId": 3771, "text": "No", "score": -3.467355489730835 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": -7.467254638671875 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": -7.9416117668151855 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -8.267688751220703 }, { "rank": 6, "tokenId": 139305, "text": "Wet", "score": -9.051909446716309 }, { "rank": 7, "tokenId": 9277, "text": "False", "score": -10.754557609558105 }, { "rank": 8, "tokenId": 41387, "text": "Unknown", "score": -11.646677017211914 } ] }, "f32_tree64": { "winnerTokenId": 10784, "winnerText": "Yes", "winnerScore": -0.6644415855407715, "winnerGap": 2.8029112815856934, "top": [ { "rank": 1, "tokenId": 10784, "text": "Yes", "score": -0.6644415855407715 }, { "rank": 2, "tokenId": 3771, "text": "No", "score": -3.467352867126465 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": -7.467242240905762 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": -7.941618919372559 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -8.267684936523438 }, { "rank": 6, "tokenId": 139305, "text": "Wet", "score": -9.051902770996094 }, { "rank": 7, "tokenId": 9277, "text": "False", "score": -10.754562377929688 }, { "rank": 8, "tokenId": 41387, "text": "Unknown", "score": -11.6466646194458 } ] }, "f16_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": -3.4765625, "winnerGap": 1.06640625, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": -3.4765625 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": -4.54296875 }, { "rank": 3, "tokenId": 19204, "text": "Variable", "score": -6.421875 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": -7.4765625 }, { "rank": 5, "tokenId": 32556, "text": "Pour", "score": -7.6640625 }, { "rank": 6, "tokenId": 4339, "text": "True", "score": -8.3515625 }, { "rank": 7, "tokenId": 8728, "text": "Base", "score": -8.84375 }, { "rank": 8, "tokenId": 179306, "text": "Depends", "score": -8.9375 } ] } }, "winnerTextsByMode": { "exact": "Yes", "f32_forward": "Yes", "f32_reverse": "Yes", "f32_tree64": "Yes", "f16_forward": "No" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Yes", "No" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 10784, 106 ], "decodedText": "Yes", "steps": [ { "step": 0, "tokenId": 10784, "text": "Yes" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 3771, 106 ], "decodedText": "No", "steps": [ { "step": 0, "tokenId": 3771, "text": "No" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "punc-ellipsis-next-oneword-d", "text": "Choose one word only.\nAfter everything that happened...the right thing to do is", "stableTopK": [ { "rank": 1, "tokenId": 1018, "text": "**", "logit": 14.932060241699219, "prob": 0.8509119153022766 }, { "rank": 2, "tokenId": 52790, "text": "Forg", "logit": 12.980729103088379, "prob": 0.12090165913105011 }, { "rank": 3, "tokenId": 24040, "text": "Accept", "logit": 10.496651649475098, "prob": 0.010083496570587158 }, { "rank": 4, "tokenId": 5024, "text": "Reg", "logit": 10.476426124572754, "prob": 0.009881600737571716 }, { "rank": 5, "tokenId": 24852, "text": "Choose", "logit": 9.496162414550781, "prob": 0.003707696683704853 }, { "rank": 6, "tokenId": 8724, "text": "Dec", "logit": 7.716208457946777, "prob": 0.0006252878811210394 }, { "rank": 7, "tokenId": 9233, "text": "Change", "logit": 7.445724010467529, "prob": 0.00047710075159557164 }, { "rank": 8, "tokenId": 191580, "text": "Healing", "logit": 7.41911506652832, "prob": 0.00046457303687930107 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 1018, "winnerText": "**", "winnerScore": 16.38873771236399, "winnerGap": 2.492650375403727, "top": [ { "rank": 1, "tokenId": 1018, "text": "**", "score": 16.38873771236399 }, { "rank": 2, "tokenId": 52790, "text": "Forg", "score": 13.896087336960264 }, { "rank": 3, "tokenId": 24040, "text": "Accept", "score": 10.959501413613964 }, { "rank": 4, "tokenId": 5024, "text": "Reg", "score": 10.936463122465678 }, { "rank": 5, "tokenId": 24852, "text": "Choose", "score": 9.83387467980117 }, { "rank": 6, "tokenId": 8724, "text": "Dec", "score": 7.893452028475387 }, { "rank": 7, "tokenId": 9233, "text": "Change", "score": 7.604516305327003 }, { "rank": 8, "tokenId": 191580, "text": "Healing", "score": 7.576171563911156 } ] }, "f32_forward": { "winnerTokenId": 1018, "winnerText": "**", "winnerScore": 16.388734817504883, "winnerGap": 2.4926557540893555, "top": [ { "rank": 1, "tokenId": 1018, "text": "**", "score": 16.388734817504883 }, { "rank": 2, "tokenId": 52790, "text": "Forg", "score": 13.896079063415527 }, { "rank": 3, "tokenId": 24040, "text": "Accept", "score": 10.959502220153809 }, { "rank": 4, "tokenId": 5024, "text": "Reg", "score": 10.936471939086914 }, { "rank": 5, "tokenId": 24852, "text": "Choose", "score": 9.833868026733398 }, { "rank": 6, "tokenId": 8724, "text": "Dec", "score": 7.89344596862793 }, { "rank": 7, "tokenId": 9233, "text": "Change", "score": 7.604518890380859 }, { "rank": 8, "tokenId": 191580, "text": "Healing", "score": 7.5761799812316895 } ] }, "f32_reverse": { "winnerTokenId": 1018, "winnerText": "**", "winnerScore": 16.388742446899414, "winnerGap": 2.492659568786621, "top": [ { "rank": 1, "tokenId": 1018, "text": "**", "score": 16.388742446899414 }, { "rank": 2, "tokenId": 52790, "text": "Forg", "score": 13.896082878112793 }, { "rank": 3, "tokenId": 24040, "text": "Accept", "score": 10.95950984954834 }, { "rank": 4, "tokenId": 5024, "text": "Reg", "score": 10.936454772949219 }, { "rank": 5, "tokenId": 24852, "text": "Choose", "score": 9.833868026733398 }, { "rank": 6, "tokenId": 8724, "text": "Dec", "score": 7.893453121185303 }, { "rank": 7, "tokenId": 9233, "text": "Change", "score": 7.604515075683594 }, { "rank": 8, "tokenId": 191580, "text": "Healing", "score": 7.576173782348633 } ] }, "f32_tree64": { "winnerTokenId": 1018, "winnerText": "**", "winnerScore": 16.388736724853516, "winnerGap": 2.4926490783691406, "top": [ { "rank": 1, "tokenId": 1018, "text": "**", "score": 16.388736724853516 }, { "rank": 2, "tokenId": 52790, "text": "Forg", "score": 13.896087646484375 }, { "rank": 3, "tokenId": 24040, "text": "Accept", "score": 10.959503173828125 }, { "rank": 4, "tokenId": 5024, "text": "Reg", "score": 10.936463356018066 }, { "rank": 5, "tokenId": 24852, "text": "Choose", "score": 9.833873748779297 }, { "rank": 6, "tokenId": 8724, "text": "Dec", "score": 7.8934526443481445 }, { "rank": 7, "tokenId": 9233, "text": "Change", "score": 7.604515075683594 }, { "rank": 8, "tokenId": 191580, "text": "Healing", "score": 7.576171875 } ] }, "f16_forward": { "winnerTokenId": 1018, "winnerText": "**", "winnerScore": 16.5, "winnerGap": 2.640625, "top": [ { "rank": 1, "tokenId": 1018, "text": "**", "score": 16.5 }, { "rank": 2, "tokenId": 52790, "text": "Forg", "score": 13.859375 }, { "rank": 3, "tokenId": 24040, "text": "Accept", "score": 10.9921875 }, { "rank": 4, "tokenId": 24852, "text": "Choose", "score": 9.84375 }, { "rank": 5, "tokenId": 9233, "text": "Change", "score": 7.5703125 }, { "rank": 6, "tokenId": 20470, "text": "Wait", "score": 7.484375 }, { "rank": 7, "tokenId": 191580, "text": "Healing", "score": 7.328125 }, { "rank": 8, "tokenId": 2542, "text": "For", "score": 7.03515625 } ] } }, "winnerTextsByMode": { "exact": "**", "f32_forward": "**", "f32_reverse": "**", "f32_tree64": "**", "f16_forward": "**" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "**" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "punc-exclaim-run-oneword-a", "text": "Answer with exactly one word only.\nRun! The light is", "stableTopK": [ { "rank": 1, "tokenId": 2791, "text": "On", "logit": 2.6949474811553955, "prob": 0.40967920422554016 }, { "rank": 2, "tokenId": 17224, "text": "Green", "logit": 2.29931902885437, "prob": 0.2758193016052246 }, { "rank": 3, "tokenId": 3949, "text": "Out", "logit": 1.2076815366744995, "prob": 0.09258327633142471 }, { "rank": 4, "tokenId": 9264, "text": "Red", "logit": 1.1937867403030396, "prob": 0.09130574017763138 }, { "rank": 5, "tokenId": 8565, "text": "Off", "logit": 0.9036500453948975, "prob": 0.0683114230632782 }, { "rank": 6, "tokenId": 179726, "text": "Gone", "logit": -0.12564347684383392, "prob": 0.024404888972640038 }, { "rank": 7, "tokenId": 5988, "text": "Go", "logit": -0.8488596081733704, "prob": 0.011840990744531155 }, { "rank": 8, "tokenId": 37154, "text": "Yellow", "logit": -1.0110408067703247, "prob": 0.010068242438137531 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 2791, "winnerText": "On", "winnerScore": 2.702231590585228, "winnerGap": 0.39839229428729084, "top": [ { "rank": 1, "tokenId": 2791, "text": "On", "score": 2.702231590585228 }, { "rank": 2, "tokenId": 17224, "text": "Green", "score": 2.3038392962979373 }, { "rank": 3, "tokenId": 3949, "text": "Out", "score": 1.2083322998450114 }, { "rank": 4, "tokenId": 9264, "text": "Red", "score": 1.1944173890482475 }, { "rank": 5, "tokenId": 8565, "text": "Off", "score": 0.9039222682071753 }, { "rank": 6, "tokenId": 179726, "text": "Gone", "score": -0.12564815841188093 }, { "rank": 7, "tokenId": 5988, "text": "Go", "score": -0.8490860093685448 }, { "rank": 8, "tokenId": 37154, "text": "Yellow", "score": -1.0114218549302336 } ] }, "f32_forward": { "winnerTokenId": 2791, "winnerText": "On", "winnerScore": 2.7022299766540527, "winnerGap": 0.3983922004699707, "top": [ { "rank": 1, "tokenId": 2791, "text": "On", "score": 2.7022299766540527 }, { "rank": 2, "tokenId": 17224, "text": "Green", "score": 2.303837776184082 }, { "rank": 3, "tokenId": 3949, "text": "Out", "score": 1.2083311080932617 }, { "rank": 4, "tokenId": 9264, "text": "Red", "score": 1.1944215297698975 }, { "rank": 5, "tokenId": 8565, "text": "Off", "score": 0.9039239883422852 }, { "rank": 6, "tokenId": 179726, "text": "Gone", "score": -0.1256486028432846 }, { "rank": 7, "tokenId": 5988, "text": "Go", "score": -0.8490890860557556 }, { "rank": 8, "tokenId": 37154, "text": "Yellow", "score": -1.0114185810089111 } ] }, "f32_reverse": { "winnerTokenId": 2791, "winnerText": "On", "winnerScore": 2.702232599258423, "winnerGap": 0.3983943462371826, "top": [ { "rank": 1, "tokenId": 2791, "text": "On", "score": 2.702232599258423 }, { "rank": 2, "tokenId": 17224, "text": "Green", "score": 2.3038382530212402 }, { "rank": 3, "tokenId": 3949, "text": "Out", "score": 1.2083344459533691 }, { "rank": 4, "tokenId": 9264, "text": "Red", "score": 1.1944184303283691 }, { "rank": 5, "tokenId": 8565, "text": "Off", "score": 0.9039239287376404 }, { "rank": 6, "tokenId": 179726, "text": "Gone", "score": -0.12565270066261292 }, { "rank": 7, "tokenId": 5988, "text": "Go", "score": -0.8490874171257019 }, { "rank": 8, "tokenId": 37154, "text": "Yellow", "score": -1.011422872543335 } ] }, "f32_tree64": { "winnerTokenId": 2791, "winnerText": "On", "winnerScore": 2.7022316455841064, "winnerGap": 0.3983914852142334, "top": [ { "rank": 1, "tokenId": 2791, "text": "On", "score": 2.7022316455841064 }, { "rank": 2, "tokenId": 17224, "text": "Green", "score": 2.303840160369873 }, { "rank": 3, "tokenId": 3949, "text": "Out", "score": 1.2083320617675781 }, { "rank": 4, "tokenId": 9264, "text": "Red", "score": 1.194417953491211 }, { "rank": 5, "tokenId": 8565, "text": "Off", "score": 0.9039226770401001 }, { "rank": 6, "tokenId": 179726, "text": "Gone", "score": -0.12564682960510254 }, { "rank": 7, "tokenId": 5988, "text": "Go", "score": -0.8490865230560303 }, { "rank": 8, "tokenId": 37154, "text": "Yellow", "score": -1.0114219188690186 } ] }, "f16_forward": { "winnerTokenId": 17224, "winnerText": "Green", "winnerScore": 2.37890625, "winnerGap": 0.66015625, "top": [ { "rank": 1, "tokenId": 17224, "text": "Green", "score": 2.37890625 }, { "rank": 2, "tokenId": 2791, "text": "On", "score": 1.71875 }, { "rank": 3, "tokenId": 9264, "text": "Red", "score": 1.2685546875 }, { "rank": 4, "tokenId": 3949, "text": "Out", "score": 1.203125 }, { "rank": 5, "tokenId": 8565, "text": "Off", "score": 0.58544921875 }, { "rank": 6, "tokenId": 179726, "text": "Gone", "score": -0.136474609375 }, { "rank": 7, "tokenId": 5988, "text": "Go", "score": -0.55908203125 }, { "rank": 8, "tokenId": 37154, "text": "Yellow", "score": -1.078125 } ] } }, "winnerTextsByMode": { "exact": "On", "f32_forward": "On", "f32_reverse": "On", "f32_tree64": "On", "f16_forward": "Green" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "On", "Green" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 2791, 106 ], "decodedText": "On", "steps": [ { "step": 0, "tokenId": 2791, "text": "On" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 17224, 106 ], "decodedText": "Green", "steps": [ { "step": 0, "tokenId": 17224, "text": "Green" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "punc-hashtag-prompt-oneword-b", "text": "Respond with one word only.\n#unpopularopinion: the best language is", "stableTopK": [ { "rank": 1, "tokenId": 186741, "text": "Silence", "logit": 1.4707766771316528, "prob": 0.5444695949554443 }, { "rank": 2, "tokenId": 3637, "text": "Context", "logit": 1.154161810874939, "prob": 0.3967066705226898 }, { "rank": 3, "tokenId": 4809, "text": "Code", "logit": -2.7186803817749023, "prob": 0.008251169696450233 }, { "rank": 4, "tokenId": 236773, "text": "S", "logit": -2.947049379348755, "prob": 0.006566532887518406 }, { "rank": 5, "tokenId": 27832, "text": "English", "logit": -3.448432207107544, "prob": 0.0039773001335561275 }, { "rank": 6, "tokenId": 136657, "text": "Silent", "logit": -3.6250274181365967, "prob": 0.0033334505278617144 }, { "rank": 7, "tokenId": 236776, "text": "A", "logit": -3.804506778717041, "prob": 0.002785781864076853 }, { "rank": 8, "tokenId": 236788, "text": "E", "logit": -3.848396062850952, "prob": 0.002666159998625517 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 186741, "winnerText": "Silence", "winnerScore": 1.4719588962932821, "winnerGap": 0.31722769026465336, "top": [ { "rank": 1, "tokenId": 186741, "text": "Silence", "score": 1.4719588962932821 }, { "rank": 2, "tokenId": 3637, "text": "Context", "score": 1.1547312060286288 }, { "rank": 3, "tokenId": 4809, "text": "Code", "score": -2.7261588363658547 }, { "rank": 4, "tokenId": 236773, "text": "S", "score": -2.956581556035939 }, { "rank": 5, "tokenId": 27832, "text": "English", "score": -3.4637439137166695 }, { "rank": 6, "tokenId": 136657, "text": "Silent", "score": -3.6428278750081233 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": -3.8250997913091407 }, { "rank": 8, "tokenId": 236788, "text": "E", "score": -3.8697176614245805 } ] }, "f32_forward": { "winnerTokenId": 186741, "winnerText": "Silence", "winnerScore": 1.4719656705856323, "winnerGap": 0.3172379732131958, "top": [ { "rank": 1, "tokenId": 186741, "text": "Silence", "score": 1.4719656705856323 }, { "rank": 2, "tokenId": 3637, "text": "Context", "score": 1.1547276973724365 }, { "rank": 3, "tokenId": 4809, "text": "Code", "score": -2.7261581420898438 }, { "rank": 4, "tokenId": 236773, "text": "S", "score": -2.9565818309783936 }, { "rank": 5, "tokenId": 27832, "text": "English", "score": -3.463743209838867 }, { "rank": 6, "tokenId": 136657, "text": "Silent", "score": -3.642833948135376 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": -3.8251054286956787 }, { "rank": 8, "tokenId": 236788, "text": "E", "score": -3.869717597961426 } ] }, "f32_reverse": { "winnerTokenId": 186741, "winnerText": "Silence", "winnerScore": 1.4719539880752563, "winnerGap": 0.3172285556793213, "top": [ { "rank": 1, "tokenId": 186741, "text": "Silence", "score": 1.4719539880752563 }, { "rank": 2, "tokenId": 3637, "text": "Context", "score": 1.154725432395935 }, { "rank": 3, "tokenId": 4809, "text": "Code", "score": -2.7261619567871094 }, { "rank": 4, "tokenId": 236773, "text": "S", "score": -2.9565844535827637 }, { "rank": 5, "tokenId": 27832, "text": "English", "score": -3.4637484550476074 }, { "rank": 6, "tokenId": 136657, "text": "Silent", "score": -3.6428308486938477 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": -3.825087308883667 }, { "rank": 8, "tokenId": 236788, "text": "E", "score": -3.8697190284729004 } ] }, "f32_tree64": { "winnerTokenId": 186741, "winnerText": "Silence", "winnerScore": 1.4719579219818115, "winnerGap": 0.3172264099121094, "top": [ { "rank": 1, "tokenId": 186741, "text": "Silence", "score": 1.4719579219818115 }, { "rank": 2, "tokenId": 3637, "text": "Context", "score": 1.1547315120697021 }, { "rank": 3, "tokenId": 4809, "text": "Code", "score": -2.726158618927002 }, { "rank": 4, "tokenId": 236773, "text": "S", "score": -2.956582546234131 }, { "rank": 5, "tokenId": 27832, "text": "English", "score": -3.4637444019317627 }, { "rank": 6, "tokenId": 136657, "text": "Silent", "score": -3.642827033996582 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": -3.825101137161255 }, { "rank": 8, "tokenId": 236788, "text": "E", "score": -3.8697168827056885 } ] }, "f16_forward": { "winnerTokenId": 186741, "winnerText": "Silence", "winnerScore": 1.482421875, "winnerGap": 1.25634765625, "top": [ { "rank": 1, "tokenId": 186741, "text": "Silence", "score": 1.482421875 }, { "rank": 2, "tokenId": 3637, "text": "Context", "score": 0.22607421875 }, { "rank": 3, "tokenId": 4809, "text": "Code", "score": -2.693359375 }, { "rank": 4, "tokenId": 236773, "text": "S", "score": -2.984375 }, { "rank": 5, "tokenId": 6875, "text": "context", "score": -3.033203125 }, { "rank": 6, "tokenId": 27832, "text": "English", "score": -3.349609375 }, { "rank": 7, "tokenId": 236855, "text": "K", "score": -3.427734375 }, { "rank": 8, "tokenId": 136657, "text": "Silent", "score": -3.62109375 } ] } }, "winnerTextsByMode": { "exact": "Silence", "f32_forward": "Silence", "f32_reverse": "Silence", "f32_tree64": "Silence", "f16_forward": "Silence" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Silence" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "punc-parens-gravity-oneword-c", "text": "No explanation. One word only.\nGravity (as a force) is fundamental or emergent. It is", "stableTopK": [ { "rank": 1, "tokenId": 205648, "text": "Fundamental", "logit": 18.199697494506836, "prob": 0.9953127503395081 }, { "rank": 2, "tokenId": 164712, "text": "fundamental", "logit": 12.489458084106445, "prob": 0.00329635594971478 }, { "rank": 3, "tokenId": 61117, "text": "Emer", "logit": 11.462484359741211, "prob": 0.001180388848297298 }, { "rank": 4, "tokenId": 71044, "text": " Fundamental", "logit": 9.003061294555664, "prob": 0.00010090466093970463 }, { "rank": 5, "tokenId": 34399, "text": "Fund", "logit": 7.9520440101623535, "prob": 0.000035274446418043226 }, { "rank": 6, "tokenId": 36350, "text": "Force", "logit": 7.0049872398376465, "prob": 0.000013682286407856736 }, { "rank": 7, "tokenId": 14713, "text": "Under", "logit": 6.21811580657959, "prob": 0.000006229091923160013 }, { "rank": 8, "tokenId": 11099, "text": "Found", "logit": 6.195030689239502, "prob": 0.0000060869401750096586 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 205648, "winnerText": "Fundamental", "winnerScore": 21.108411441447327, "winnerGap": 7.811617299909823, "top": [ { "rank": 1, "tokenId": 205648, "text": "Fundamental", "score": 21.108411441447327 }, { "rank": 2, "tokenId": 164712, "text": "fundamental", "score": 13.296794141537504 }, { "rank": 3, "tokenId": 61117, "text": "Emer", "score": 12.074886548966084 }, { "rank": 4, "tokenId": 71044, "text": " Fundamental", "score": 9.288955153398984 }, { "rank": 5, "tokenId": 34399, "text": "Fund", "score": 8.146551141583775 }, { "rank": 6, "tokenId": 36350, "text": "Force", "score": 7.136628353623109 }, { "rank": 7, "tokenId": 14713, "text": "Under", "score": 6.309530605847456 }, { "rank": 8, "tokenId": 11099, "text": "Found", "score": 6.285408761311003 } ] }, "f32_forward": { "winnerTokenId": 205648, "winnerText": "Fundamental", "winnerScore": 21.10842514038086, "winnerGap": 7.811628341674805, "top": [ { "rank": 1, "tokenId": 205648, "text": "Fundamental", "score": 21.10842514038086 }, { "rank": 2, "tokenId": 164712, "text": "fundamental", "score": 13.296796798706055 }, { "rank": 3, "tokenId": 61117, "text": "Emer", "score": 12.07487964630127 }, { "rank": 4, "tokenId": 71044, "text": " Fundamental", "score": 9.288965225219727 }, { "rank": 5, "tokenId": 34399, "text": "Fund", "score": 8.146544456481934 }, { "rank": 6, "tokenId": 36350, "text": "Force", "score": 7.136628150939941 }, { "rank": 7, "tokenId": 14713, "text": "Under", "score": 6.309534549713135 }, { "rank": 8, "tokenId": 11099, "text": "Found", "score": 6.285402297973633 } ] }, "f32_reverse": { "winnerTokenId": 205648, "winnerText": "Fundamental", "winnerScore": 21.10841941833496, "winnerGap": 7.811624526977539, "top": [ { "rank": 1, "tokenId": 205648, "text": "Fundamental", "score": 21.10841941833496 }, { "rank": 2, "tokenId": 164712, "text": "fundamental", "score": 13.296794891357422 }, { "rank": 3, "tokenId": 61117, "text": "Emer", "score": 12.074883460998535 }, { "rank": 4, "tokenId": 71044, "text": " Fundamental", "score": 9.288962364196777 }, { "rank": 5, "tokenId": 34399, "text": "Fund", "score": 8.146549224853516 }, { "rank": 6, "tokenId": 36350, "text": "Force", "score": 7.136627674102783 }, { "rank": 7, "tokenId": 14713, "text": "Under", "score": 6.309532642364502 }, { "rank": 8, "tokenId": 11099, "text": "Found", "score": 6.285408973693848 } ] }, "f32_tree64": { "winnerTokenId": 205648, "winnerText": "Fundamental", "winnerScore": 21.10841178894043, "winnerGap": 7.811618804931641, "top": [ { "rank": 1, "tokenId": 205648, "text": "Fundamental", "score": 21.10841178894043 }, { "rank": 2, "tokenId": 164712, "text": "fundamental", "score": 13.296792984008789 }, { "rank": 3, "tokenId": 61117, "text": "Emer", "score": 12.074886322021484 }, { "rank": 4, "tokenId": 71044, "text": " Fundamental", "score": 9.28895378112793 }, { "rank": 5, "tokenId": 34399, "text": "Fund", "score": 8.146551132202148 }, { "rank": 6, "tokenId": 36350, "text": "Force", "score": 7.136629104614258 }, { "rank": 7, "tokenId": 14713, "text": "Under", "score": 6.309531211853027 }, { "rank": 8, "tokenId": 11099, "text": "Found", "score": 6.285407543182373 } ] }, "f16_forward": { "winnerTokenId": 205648, "winnerText": "Fundamental", "winnerScore": 20.9375, "winnerGap": 7.65625, "top": [ { "rank": 1, "tokenId": 205648, "text": "Fundamental", "score": 20.9375 }, { "rank": 2, "tokenId": 164712, "text": "fundamental", "score": 13.28125 }, { "rank": 3, "tokenId": 61117, "text": "Emer", "score": 12.203125 }, { "rank": 4, "tokenId": 34399, "text": "Fund", "score": 8.1796875 }, { "rank": 5, "tokenId": 14713, "text": "Under", "score": 6.39453125 }, { "rank": 6, "tokenId": 11099, "text": "Found", "score": 6.2890625 }, { "rank": 7, "tokenId": 11001, "text": " fundamental", "score": 6.0703125 }, { "rank": 8, "tokenId": 94659, "text": "Phen", "score": 5.9140625 } ] } }, "winnerTextsByMode": { "exact": "Fundamental", "f32_forward": "Fundamental", "f32_reverse": "Fundamental", "f32_tree64": "Fundamental", "f16_forward": "Fundamental" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Fundamental" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "punc-pipe-or-oneword-d", "text": "Choose one word only.\ntrue | false — the earth is round:", "stableTopK": [ { "rank": 1, "tokenId": 3397, "text": "true", "logit": 16.041486740112305, "prob": 0.9994619488716125 }, { "rank": 2, "tokenId": 4339, "text": "True", "logit": 8.401995658874512, "prob": 0.0004808143712580204 }, { "rank": 3, "tokenId": 1018, "text": "**", "logit": 6.208749771118164, "prob": 0.000053636798838851973 }, { "rank": 4, "tokenId": 4530, "text": "false", "logit": 3.0416648387908936, "prob": 0.0000022595156679017236 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "logit": 1.4759036302566528, "prob": 4.7207814191096986e-7 }, { "rank": 6, "tokenId": 3292, "text": "$\\", "logit": 0.027347557246685028, "prob": 1.1089551321674662e-7 }, { "rank": 7, "tokenId": 106, "text": "", "logit": -0.2571686804294586, "prob": 8.343536705979204e-8 }, { "rank": 8, "tokenId": 1, "text": "", "logit": -0.4349882900714874, "prob": 6.984318901004372e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 17.901798101893824, "winnerGap": 9.269170227574271, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 17.901798101893824 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 8.632627874319553 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 6.299742451406372 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": 3.0521553216920267 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": 1.477098547813534 }, { "rank": 6, "tokenId": 3292, "text": "$\\", "score": 0.027345932834677456 }, { "rank": 7, "tokenId": 106, "text": "", "score": -0.2571763626193544 }, { "rank": 8, "tokenId": 1, "text": "", "score": -0.43501881865074665 } ] }, "f32_forward": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 17.901811599731445, "winnerGap": 9.269183158874512, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 17.901811599731445 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 8.632628440856934 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 6.299749851226807 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": 3.0521509647369385 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": 1.4770941734313965 }, { "rank": 6, "tokenId": 3292, "text": "$\\", "score": 0.027347665280103683 }, { "rank": 7, "tokenId": 106, "text": "", "score": -0.2571772336959839 }, { "rank": 8, "tokenId": 1, "text": "", "score": -0.4350193440914154 } ] }, "f32_reverse": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 17.901803970336914, "winnerGap": 9.269176483154297, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 17.901803970336914 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 8.632627487182617 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 6.299741268157959 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": 3.0521559715270996 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": 1.4771008491516113 }, { "rank": 6, "tokenId": 3292, "text": "$\\", "score": 0.027345914393663406 }, { "rank": 7, "tokenId": 106, "text": "", "score": -0.257178395986557 }, { "rank": 8, "tokenId": 1, "text": "", "score": -0.4350200891494751 } ] }, "f32_tree64": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 17.901798248291016, "winnerGap": 9.269170761108398, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 17.901798248291016 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 8.632627487182617 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 6.299742221832275 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": 3.052154779434204 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": 1.4770991802215576 }, { "rank": 6, "tokenId": 3292, "text": "$\\", "score": 0.027346014976501465 }, { "rank": 7, "tokenId": 106, "text": "", "score": -0.2571762800216675 }, { "rank": 8, "tokenId": 1, "text": "", "score": -0.4350188970565796 } ] }, "f16_forward": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 17.890625, "winnerGap": 9.171875, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 17.890625 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 8.71875 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 6.296875 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": 3.076171875 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": 1.48046875 }, { "rank": 6, "tokenId": 3292, "text": "$\\", "score": 0.061279296875 }, { "rank": 7, "tokenId": 106, "text": "", "score": -0.25146484375 }, { "rank": 8, "tokenId": 1, "text": "", "score": -0.422119140625 } ] } }, "winnerTextsByMode": { "exact": "true", "f32_forward": "true", "f32_reverse": "true", "f32_tree64": "true", "f16_forward": "true" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "true" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "punc-question-chain-oneword-a", "text": "Answer with exactly one word only.\nIs it safe? Is it legal? Is it ethical? The answer is", "stableTopK": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "logit": 9.369216918945312, "prob": 0.6098018884658813 }, { "rank": 2, "tokenId": 3637, "text": "Context", "logit": 8.898176193237305, "prob": 0.38073110580444336 }, { "rank": 3, "tokenId": 41387, "text": "Unknown", "logit": 4.795111656188965, "prob": 0.0062904260121285915 }, { "rank": 4, "tokenId": 30340, "text": "Und", "logit": 3.1547818183898926, "prob": 0.0012198147596791387 }, { "rank": 5, "tokenId": 236847, "text": "V", "logit": 2.65142822265625, "prob": 0.0007373780244961381 }, { "rank": 6, "tokenId": 58686, "text": "Cannot", "logit": 2.2961511611938477, "prob": 0.0005168866482563317 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "logit": 1.7674767971038818, "prob": 0.00030464562587440014 }, { "rank": 8, "tokenId": 19204, "text": "Variable", "logit": 1.3602627515792847, "prob": 0.00020274218695703894 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 9.693002282553046, "winnerGap": 0.5191834575294365, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 9.693002282553046 }, { "rank": 2, "tokenId": 3637, "text": "Context", "score": 9.17381882502361 }, { "rank": 3, "tokenId": 41387, "text": "Unknown", "score": 4.83658350374586 }, { "rank": 4, "tokenId": 30340, "text": "Und", "score": 3.1664894197810973 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": 2.6583690840766696 }, { "rank": 6, "tokenId": 58686, "text": "Cannot", "score": 2.3006503001453495 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "score": 1.7695261670419384 }, { "rank": 8, "tokenId": 19204, "text": "Variable", "score": 1.3611945438993445 } ] }, "f32_forward": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 9.693009376525879, "winnerGap": 0.5191946029663086, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 9.693009376525879 }, { "rank": 2, "tokenId": 3637, "text": "Context", "score": 9.17381477355957 }, { "rank": 3, "tokenId": 41387, "text": "Unknown", "score": 4.836582660675049 }, { "rank": 4, "tokenId": 30340, "text": "Und", "score": 3.1664881706237793 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": 2.6583616733551025 }, { "rank": 6, "tokenId": 58686, "text": "Cannot", "score": 2.3006513118743896 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "score": 1.7695319652557373 }, { "rank": 8, "tokenId": 19204, "text": "Variable", "score": 1.3611950874328613 } ] }, "f32_reverse": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 9.692994117736816, "winnerGap": 0.5191812515258789, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 9.692994117736816 }, { "rank": 2, "tokenId": 3637, "text": "Context", "score": 9.173812866210938 }, { "rank": 3, "tokenId": 41387, "text": "Unknown", "score": 4.836579322814941 }, { "rank": 4, "tokenId": 30340, "text": "Und", "score": 3.1664915084838867 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": 2.6583681106567383 }, { "rank": 6, "tokenId": 58686, "text": "Cannot", "score": 2.3006534576416016 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "score": 1.7695280313491821 }, { "rank": 8, "tokenId": 19204, "text": "Variable", "score": 1.361194372177124 } ] }, "f32_tree64": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 9.693001747131348, "winnerGap": 0.5191822052001953, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 9.693001747131348 }, { "rank": 2, "tokenId": 3637, "text": "Context", "score": 9.173819541931152 }, { "rank": 3, "tokenId": 41387, "text": "Unknown", "score": 4.836583137512207 }, { "rank": 4, "tokenId": 30340, "text": "Und", "score": 3.166490316390991 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": 2.65836763381958 }, { "rank": 6, "tokenId": 58686, "text": "Cannot", "score": 2.300649642944336 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "score": 1.769526720046997 }, { "rank": 8, "tokenId": 19204, "text": "Variable", "score": 1.361194133758545 } ] }, "f16_forward": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 9.703125, "winnerGap": 4.3984375, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 9.703125 }, { "rank": 2, "tokenId": 3637, "text": "Context", "score": 5.3046875 }, { "rank": 3, "tokenId": 41387, "text": "Unknown", "score": 4.8203125 }, { "rank": 4, "tokenId": 30340, "text": "Und", "score": 3.251953125 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": 2.662109375 }, { "rank": 6, "tokenId": 58686, "text": "Cannot", "score": 2.3203125 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "score": 1.775390625 }, { "rank": 8, "tokenId": 19204, "text": "Variable", "score": 1.4287109375 } ] } }, "winnerTextsByMode": { "exact": "Depends", "f32_forward": "Depends", "f32_reverse": "Depends", "f32_tree64": "Depends", "f16_forward": "Depends" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Depends" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "punc-quotes-said-oneword-b", "text": "Respond with one word only.\n\"I think it's\" —she paused— \"", "stableTopK": [ { "rank": 1, "tokenId": 31402, "text": "Maybe", "logit": 2.2534422874450684, "prob": 0.6246389150619507 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "logit": 0.18825331330299377, "prob": 0.07920067012310028 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "logit": -0.24203109741210938, "prob": 0.051506105810403824 }, { "rank": 4, "tokenId": 60766, "text": "maybe", "logit": -0.44901904463768005, "prob": 0.04187598079442978 }, { "rank": 5, "tokenId": 38532, "text": "Perhaps", "logit": -0.4698089063167572, "prob": 0.04101437330245972 }, { "rank": 6, "tokenId": 1136, "text": "und", "logit": -0.7746568918228149, "prob": 0.030237248167395592 }, { "rank": 7, "tokenId": 41387, "text": "Unknown", "logit": -0.8693172931671143, "prob": 0.027506273239850998 }, { "rank": 8, "tokenId": 30340, "text": "Und", "logit": -0.9475762844085693, "prob": 0.025435736402869225 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 31402, "winnerText": "Maybe", "winnerScore": 2.2576938192906617, "winnerGap": 2.069435845259793, "top": [ { "rank": 1, "tokenId": 31402, "text": "Maybe", "score": 2.2576938192906617 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 0.18825797403086852 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": -0.2420358545984802 }, { "rank": 4, "tokenId": 60766, "text": "maybe", "score": -0.44905447480154237 }, { "rank": 5, "tokenId": 38532, "text": "Perhaps", "score": -0.46984591739911874 }, { "rank": 6, "tokenId": 1136, "text": "und", "score": -0.7748296771453127 }, { "rank": 7, "tokenId": 41387, "text": "Unknown", "score": -0.8695595903693709 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": -0.9478924070483501 } ] }, "f32_forward": { "winnerTokenId": 31402, "winnerText": "Maybe", "winnerScore": 2.2577030658721924, "winnerGap": 2.0694409161806107, "top": [ { "rank": 1, "tokenId": 31402, "text": "Maybe", "score": 2.2577030658721924 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 0.18826214969158173 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": -0.24204081296920776 }, { "rank": 4, "tokenId": 60766, "text": "maybe", "score": -0.4490559697151184 }, { "rank": 5, "tokenId": 38532, "text": "Perhaps", "score": -0.469845175743103 }, { "rank": 6, "tokenId": 1136, "text": "und", "score": -0.7748292088508606 }, { "rank": 7, "tokenId": 41387, "text": "Unknown", "score": -0.8695621490478516 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": -0.9478920698165894 } ] }, "f32_reverse": { "winnerTokenId": 31402, "winnerText": "Maybe", "winnerScore": 2.25769305229187, "winnerGap": 2.0694319158792496, "top": [ { "rank": 1, "tokenId": 31402, "text": "Maybe", "score": 2.25769305229187 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 0.18826113641262054 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": -0.24204345047473907 }, { "rank": 4, "tokenId": 60766, "text": "maybe", "score": -0.44905608892440796 }, { "rank": 5, "tokenId": 38532, "text": "Perhaps", "score": -0.46984583139419556 }, { "rank": 6, "tokenId": 1136, "text": "und", "score": -0.7748291492462158 }, { "rank": 7, "tokenId": 41387, "text": "Unknown", "score": -0.8695608973503113 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": -0.9478914141654968 } ] }, "f32_tree64": { "winnerTokenId": 31402, "winnerText": "Maybe", "winnerScore": 2.257692813873291, "winnerGap": 2.0694355368614197, "top": [ { "rank": 1, "tokenId": 31402, "text": "Maybe", "score": 2.257692813873291 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 0.18825727701187134 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": -0.2420363426208496 }, { "rank": 4, "tokenId": 60766, "text": "maybe", "score": -0.4490547180175781 }, { "rank": 5, "tokenId": 38532, "text": "Perhaps", "score": -0.4698452949523926 }, { "rank": 6, "tokenId": 1136, "text": "und", "score": -0.7748298645019531 }, { "rank": 7, "tokenId": 41387, "text": "Unknown", "score": -0.8695594072341919 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": -0.9478938579559326 } ] }, "f16_forward": { "winnerTokenId": 31402, "winnerText": "Maybe", "winnerScore": 2.294921875, "winnerGap": 2.08544921875, "top": [ { "rank": 1, "tokenId": 31402, "text": "Maybe", "score": 2.294921875 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 0.20947265625 }, { "rank": 3, "tokenId": 20470, "text": "Wait", "score": -0.252685546875 }, { "rank": 4, "tokenId": 38532, "text": "Perhaps", "score": -0.414306640625 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": -0.802734375 }, { "rank": 6, "tokenId": 1136, "text": "und", "score": -0.8359375 }, { "rank": 7, "tokenId": 30340, "text": "Und", "score": -0.9775390625 }, { "rank": 8, "tokenId": 2805, "text": "Un", "score": -0.9921875 } ] } }, "winnerTextsByMode": { "exact": "Maybe", "f32_forward": "Maybe", "f32_reverse": "Maybe", "f32_tree64": "Maybe", "f16_forward": "Maybe" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Maybe" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "punc-semicolon-sky-oneword-c", "text": "No explanation. One word only.\nThe sky; it is", "stableTopK": [ { "rank": 1, "tokenId": 236847, "text": "V", "logit": 9.87365436553955, "prob": 0.9961204528808594 }, { "rank": 2, "tokenId": 132602, "text": "Infinite", "logit": 4.241578578948975, "prob": 0.0035672406665980816 }, { "rank": 3, "tokenId": 16520, "text": "Blue", "logit": 1.3635456562042236, "prob": 0.00020064048294443637 }, { "rank": 4, "tokenId": 133808, "text": " अनंत", "logit": 0.03365086019039154, "prob": 0.00005307043102220632 }, { "rank": 5, "tokenId": 27743, "text": "Bound", "logit": -1.4155858755111694, "prob": 0.000012458252058422659 }, { "rank": 6, "tokenId": 174289, "text": "無限", "logit": -1.6163421869277954, "prob": 0.0000101922423709766 }, { "rank": 7, "tokenId": 236776, "text": "A", "logit": -2.103344678878784, "prob": 0.000006262780971155735 }, { "rank": 8, "tokenId": 27884, "text": "Limit", "logit": -2.2446892261505127, "prob": 0.0000054372849263018 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 236847, "winnerText": "V", "winnerScore": 10.255296460503786, "winnerGap": 5.985113103431985, "top": [ { "rank": 1, "tokenId": 236847, "text": "V", "score": 10.255296460503786 }, { "rank": 2, "tokenId": 132602, "text": "Infinite", "score": 4.270183357071801 }, { "rank": 3, "tokenId": 16520, "text": "Blue", "score": 1.3644838019923213 }, { "rank": 4, "tokenId": 133808, "text": " अनंत", "score": 0.0336592912941569 }, { "rank": 5, "tokenId": 27743, "text": "Bound", "score": -1.4166378412610925 }, { "rank": 6, "tokenId": 174289, "text": "無限", "score": -1.6179083584215288 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": -2.106802109229136 }, { "rank": 8, "tokenId": 27884, "text": "Limit", "score": -2.2488936455763167 } ] }, "f32_forward": { "winnerTokenId": 236847, "winnerText": "V", "winnerScore": 10.255284309387207, "winnerGap": 5.985092639923096, "top": [ { "rank": 1, "tokenId": 236847, "text": "V", "score": 10.255284309387207 }, { "rank": 2, "tokenId": 132602, "text": "Infinite", "score": 4.270191669464111 }, { "rank": 3, "tokenId": 16520, "text": "Blue", "score": 1.36448335647583 }, { "rank": 4, "tokenId": 133808, "text": " अनंत", "score": 0.033660098910331726 }, { "rank": 5, "tokenId": 27743, "text": "Bound", "score": -1.4166359901428223 }, { "rank": 6, "tokenId": 174289, "text": "無限", "score": -1.6179107427597046 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": -2.106802225112915 }, { "rank": 8, "tokenId": 27884, "text": "Limit", "score": -2.248893976211548 } ] }, "f32_reverse": { "winnerTokenId": 236847, "winnerText": "V", "winnerScore": 10.25529956817627, "winnerGap": 5.98512077331543, "top": [ { "rank": 1, "tokenId": 236847, "text": "V", "score": 10.25529956817627 }, { "rank": 2, "tokenId": 132602, "text": "Infinite", "score": 4.27017879486084 }, { "rank": 3, "tokenId": 16520, "text": "Blue", "score": 1.3644769191741943 }, { "rank": 4, "tokenId": 133808, "text": " अनंत", "score": 0.03366011381149292 }, { "rank": 5, "tokenId": 27743, "text": "Bound", "score": -1.4166337251663208 }, { "rank": 6, "tokenId": 174289, "text": "無限", "score": -1.6179105043411255 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": -2.1068031787872314 }, { "rank": 8, "tokenId": 27884, "text": "Limit", "score": -2.2488842010498047 } ] }, "f32_tree64": { "winnerTokenId": 236847, "winnerText": "V", "winnerScore": 10.255297660827637, "winnerGap": 5.985115051269531, "top": [ { "rank": 1, "tokenId": 236847, "text": "V", "score": 10.255297660827637 }, { "rank": 2, "tokenId": 132602, "text": "Infinite", "score": 4.2701826095581055 }, { "rank": 3, "tokenId": 16520, "text": "Blue", "score": 1.3644835948944092 }, { "rank": 4, "tokenId": 133808, "text": " अनंत", "score": 0.03365886211395264 }, { "rank": 5, "tokenId": 27743, "text": "Bound", "score": -1.416637659072876 }, { "rank": 6, "tokenId": 174289, "text": "無限", "score": -1.6179072856903076 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": -2.106801986694336 }, { "rank": 8, "tokenId": 27884, "text": "Limit", "score": -2.248894214630127 } ] }, "f16_forward": { "winnerTokenId": 236847, "winnerText": "V", "winnerScore": 10.2578125, "winnerGap": 5.90625, "top": [ { "rank": 1, "tokenId": 236847, "text": "V", "score": 10.2578125 }, { "rank": 2, "tokenId": 132602, "text": "Infinite", "score": 4.3515625 }, { "rank": 3, "tokenId": 16520, "text": "Blue", "score": 1.4072265625 }, { "rank": 4, "tokenId": 133808, "text": " अनंत", "score": -0.0040283203125 }, { "rank": 5, "tokenId": 27743, "text": "Bound", "score": -1.3984375 }, { "rank": 6, "tokenId": 174289, "text": "無限", "score": -1.611328125 }, { "rank": 7, "tokenId": 12529, "text": " vast", "score": -2.009765625 }, { "rank": 8, "tokenId": 236776, "text": "A", "score": -2.123046875 } ] } }, "winnerTextsByMode": { "exact": "V", "f32_forward": "V", "f32_reverse": "V", "f32_tree64": "V", "f16_forward": "V" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "V" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "punc-slash-choice-oneword-d", "text": "Choose one word only.\nright/wrong: downloading copyrighted music is", "stableTopK": [ { "rank": 1, "tokenId": 49201, "text": "wrong", "logit": 11.929054260253906, "prob": 0.7468765377998352 }, { "rank": 2, "tokenId": 205717, "text": "illegal", "logit": 10.753405570983887, "prob": 0.23050007224082947 }, { "rank": 3, "tokenId": 58459, "text": "Wrong", "logit": 7.964588165283203, "prob": 0.01417434774339199 }, { "rank": 4, "tokenId": 1018, "text": "**", "logit": 7.032902240753174, "prob": 0.005583120509982109 }, { "rank": 5, "tokenId": 87072, "text": "Illegal", "logit": 6.309254169464111, "prob": 0.002707700477913022 }, { "rank": 6, "tokenId": 51799, "text": "copyright", "logit": 1.9243383407592773, "prob": 0.00003374860898475163 }, { "rank": 7, "tokenId": 9411, "text": "imm", "logit": 1.5923004150390625, "prob": 0.000024213280994445086 }, { "rank": 8, "tokenId": 644, "text": "ill", "logit": 1.1989800930023193, "prob": 0.000016339425201294944 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 49201, "winnerText": "wrong", "winnerScore": 12.625105988046787, "winnerGap": 1.3720255369453405, "top": [ { "rank": 1, "tokenId": 49201, "text": "wrong", "score": 12.625105988046787 }, { "rank": 2, "tokenId": 205717, "text": "illegal", "score": 11.253080451101447 }, { "rank": 3, "tokenId": 58459, "text": "Wrong", "score": 8.160044210573638 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 7.166163141057549 }, { "rank": 5, "tokenId": 87072, "text": "Illegal", "score": 6.4048204712796135 }, { "rank": 6, "tokenId": 51799, "text": "copyright", "score": 1.92698148699467 }, { "rank": 7, "tokenId": 9411, "text": "imm", "score": 1.5938008676909519 }, { "rank": 8, "tokenId": 644, "text": "ill", "score": 1.1996217199292154 } ] }, "f32_forward": { "winnerTokenId": 49201, "winnerText": "wrong", "winnerScore": 12.62509822845459, "winnerGap": 1.3720178604125977, "top": [ { "rank": 1, "tokenId": 49201, "text": "wrong", "score": 12.62509822845459 }, { "rank": 2, "tokenId": 205717, "text": "illegal", "score": 11.253080368041992 }, { "rank": 3, "tokenId": 58459, "text": "Wrong", "score": 8.16003704071045 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 7.166165828704834 }, { "rank": 5, "tokenId": 87072, "text": "Illegal", "score": 6.404824256896973 }, { "rank": 6, "tokenId": 51799, "text": "copyright", "score": 1.9270018339157104 }, { "rank": 7, "tokenId": 9411, "text": "imm", "score": 1.5938045978546143 }, { "rank": 8, "tokenId": 644, "text": "ill", "score": 1.1996300220489502 } ] }, "f32_reverse": { "winnerTokenId": 49201, "winnerText": "wrong", "winnerScore": 12.625102996826172, "winnerGap": 1.3720273971557617, "top": [ { "rank": 1, "tokenId": 49201, "text": "wrong", "score": 12.625102996826172 }, { "rank": 2, "tokenId": 205717, "text": "illegal", "score": 11.25307559967041 }, { "rank": 3, "tokenId": 58459, "text": "Wrong", "score": 8.160041809082031 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 7.166166305541992 }, { "rank": 5, "tokenId": 87072, "text": "Illegal", "score": 6.404821872711182 }, { "rank": 6, "tokenId": 51799, "text": "copyright", "score": 1.9269964694976807 }, { "rank": 7, "tokenId": 9411, "text": "imm", "score": 1.5938047170639038 }, { "rank": 8, "tokenId": 644, "text": "ill", "score": 1.1996248960494995 } ] }, "f32_tree64": { "winnerTokenId": 49201, "winnerText": "wrong", "winnerScore": 12.625106811523438, "winnerGap": 1.372025489807129, "top": [ { "rank": 1, "tokenId": 49201, "text": "wrong", "score": 12.625106811523438 }, { "rank": 2, "tokenId": 205717, "text": "illegal", "score": 11.253081321716309 }, { "rank": 3, "tokenId": 58459, "text": "Wrong", "score": 8.160043716430664 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 7.166163444519043 }, { "rank": 5, "tokenId": 87072, "text": "Illegal", "score": 6.404821395874023 }, { "rank": 6, "tokenId": 51799, "text": "copyright", "score": 1.926983118057251 }, { "rank": 7, "tokenId": 9411, "text": "imm", "score": 1.5938005447387695 }, { "rank": 8, "tokenId": 644, "text": "ill", "score": 1.1996216773986816 } ] }, "f16_forward": { "winnerTokenId": 49201, "winnerText": "wrong", "winnerScore": 12.7265625, "winnerGap": 1.53125, "top": [ { "rank": 1, "tokenId": 49201, "text": "wrong", "score": 12.7265625 }, { "rank": 2, "tokenId": 205717, "text": "illegal", "score": 11.1953125 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 7.203125 }, { "rank": 4, "tokenId": 87072, "text": "Illegal", "score": 6.45703125 }, { "rank": 5, "tokenId": 58459, "text": "Wrong", "score": 4.18359375 }, { "rank": 6, "tokenId": 51799, "text": "copyright", "score": 2.02734375 }, { "rank": 7, "tokenId": 9411, "text": "imm", "score": 1.666015625 }, { "rank": 8, "tokenId": 715, "text": "//", "score": 1.3466796875 } ] } }, "winnerTextsByMode": { "exact": "wrong", "f32_forward": "wrong", "f32_reverse": "wrong", "f32_tree64": "wrong", "f16_forward": "wrong" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "wrong" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "punc-tilde-approx-oneword-a", "text": "Answer with exactly one word only.\nThe value is ~3.14, which means pi is", "stableTopK": [ { "rank": 1, "tokenId": 14762, "text": "Pi", "logit": 6.38992977142334, "prob": 0.34360072016716003 }, { "rank": 2, "tokenId": 31804, "text": "Appro", "logit": 6.171270847320557, "prob": 0.2761160731315613 }, { "rank": 3, "tokenId": 140365, "text": "Approximately", "logit": 5.523677825927734, "prob": 0.14449261128902435 }, { "rank": 4, "tokenId": 139876, "text": "Approx", "logit": 4.915300369262695, "prob": 0.07863766700029373 }, { "rank": 5, "tokenId": 78470, "text": "approximately", "logit": 4.901418209075928, "prob": 0.07755355536937714 }, { "rank": 6, "tokenId": 11582, "text": "appro", "logit": 3.732023000717163, "prob": 0.02408462204039097 }, { "rank": 7, "tokenId": 184425, "text": "approximation", "logit": 3.6368939876556396, "prob": 0.02189907617866993 }, { "rank": 8, "tokenId": 17868, "text": "approx", "logit": 3.4609360694885254, "prob": 0.018365731462836266 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 14762, "winnerText": "Pi", "winnerScore": 6.489281312520678, "winnerGap": 0.22868438770381871, "top": [ { "rank": 1, "tokenId": 14762, "text": "Pi", "score": 6.489281312520678 }, { "rank": 2, "tokenId": 31804, "text": "Appro", "score": 6.260596924816859 }, { "rank": 3, "tokenId": 140365, "text": "Approximately", "score": 5.587398931973334 }, { "rank": 4, "tokenId": 139876, "text": "Approx", "score": 4.96000473249558 }, { "rank": 5, "tokenId": 78470, "text": "approximately", "score": 4.945742927072217 }, { "rank": 6, "tokenId": 11582, "text": "appro", "score": 3.751454941607216 }, { "rank": 7, "tokenId": 184425, "text": "approximation", "score": 3.65486917149161 }, { "rank": 8, "tokenId": 17868, "text": "approx", "score": 3.476418374253843 } ] }, "f32_forward": { "winnerTokenId": 14762, "winnerText": "Pi", "winnerScore": 6.48929500579834, "winnerGap": 0.22870397567749023, "top": [ { "rank": 1, "tokenId": 14762, "text": "Pi", "score": 6.48929500579834 }, { "rank": 2, "tokenId": 31804, "text": "Appro", "score": 6.26059103012085 }, { "rank": 3, "tokenId": 140365, "text": "Approximately", "score": 5.587409019470215 }, { "rank": 4, "tokenId": 139876, "text": "Approx", "score": 4.960003852844238 }, { "rank": 5, "tokenId": 78470, "text": "approximately", "score": 4.945736408233643 }, { "rank": 6, "tokenId": 11582, "text": "appro", "score": 3.7514617443084717 }, { "rank": 7, "tokenId": 184425, "text": "approximation", "score": 3.6548712253570557 }, { "rank": 8, "tokenId": 17868, "text": "approx", "score": 3.4764254093170166 } ] }, "f32_reverse": { "winnerTokenId": 14762, "winnerText": "Pi", "winnerScore": 6.489281177520752, "winnerGap": 0.22868108749389648, "top": [ { "rank": 1, "tokenId": 14762, "text": "Pi", "score": 6.489281177520752 }, { "rank": 2, "tokenId": 31804, "text": "Appro", "score": 6.2606000900268555 }, { "rank": 3, "tokenId": 140365, "text": "Approximately", "score": 5.587395191192627 }, { "rank": 4, "tokenId": 139876, "text": "Approx", "score": 4.960007667541504 }, { "rank": 5, "tokenId": 78470, "text": "approximately", "score": 4.945741176605225 }, { "rank": 6, "tokenId": 11582, "text": "appro", "score": 3.751453161239624 }, { "rank": 7, "tokenId": 184425, "text": "approximation", "score": 3.654871702194214 }, { "rank": 8, "tokenId": 17868, "text": "approx", "score": 3.4764161109924316 } ] }, "f32_tree64": { "winnerTokenId": 14762, "winnerText": "Pi", "winnerScore": 6.489280700683594, "winnerGap": 0.2286844253540039, "top": [ { "rank": 1, "tokenId": 14762, "text": "Pi", "score": 6.489280700683594 }, { "rank": 2, "tokenId": 31804, "text": "Appro", "score": 6.26059627532959 }, { "rank": 3, "tokenId": 140365, "text": "Approximately", "score": 5.587398529052734 }, { "rank": 4, "tokenId": 139876, "text": "Approx", "score": 4.9600043296813965 }, { "rank": 5, "tokenId": 78470, "text": "approximately", "score": 4.945741653442383 }, { "rank": 6, "tokenId": 11582, "text": "appro", "score": 3.751455307006836 }, { "rank": 7, "tokenId": 184425, "text": "approximation", "score": 3.6548690795898438 }, { "rank": 8, "tokenId": 17868, "text": "approx", "score": 3.4764177799224854 } ] }, "f16_forward": { "winnerTokenId": 31804, "winnerText": "Appro", "winnerScore": 6.2734375, "winnerGap": 0.703125, "top": [ { "rank": 1, "tokenId": 31804, "text": "Appro", "score": 6.2734375 }, { "rank": 2, "tokenId": 140365, "text": "Approximately", "score": 5.5703125 }, { "rank": 3, "tokenId": 139876, "text": "Approx", "score": 5.01953125 }, { "rank": 4, "tokenId": 78470, "text": "approximately", "score": 4.98046875 }, { "rank": 5, "tokenId": 11582, "text": "appro", "score": 3.791015625 }, { "rank": 6, "tokenId": 17868, "text": "approx", "score": 3.501953125 }, { "rank": 7, "tokenId": 2160, "text": "pi", "score": 2.734375 }, { "rank": 8, "tokenId": 14762, "text": "Pi", "score": 2.525390625 } ] } }, "winnerTextsByMode": { "exact": "Pi", "f32_forward": "Pi", "f32_reverse": "Pi", "f32_tree64": "Pi", "f16_forward": "Appro" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Pi", "Appro" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 14762, 106 ], "decodedText": "Pi", "steps": [ { "step": 0, "tokenId": 14762, "text": "Pi" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 31804, 89837 ], "decodedText": "Approximation", "steps": [ { "step": 0, "tokenId": 31804, "text": "Appro" }, { "step": 1, "tokenId": 89837, "text": "ximation" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "quote-open-oneword-b", "text": "Respond with one word only.\n\"The sky is", "stableTopK": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "logit": 6.828137397766113, "prob": 0.8187962174415588 }, { "rank": 2, "tokenId": 9503, "text": "blue", "logit": 5.319835186004639, "prob": 0.18118761479854584 }, { "rank": 3, "tokenId": 149876, "text": "蓝色", "logit": -4.070079803466797, "prob": 0.000015140501091082115 }, { "rank": 4, "tokenId": 240123, "text": "蓝", "logit": -7.47551965713501, "prob": 5.025467544328421e-7 }, { "rank": 5, "tokenId": 3730, "text": " blue", "logit": -8.853653907775879, "prob": 1.2666609450207034e-7 }, { "rank": 6, "tokenId": 236847, "text": "V", "logit": -9.131250381469727, "prob": 9.596254813004634e-8 }, { "rank": 7, "tokenId": 51481, "text": "BLUE", "logit": -9.172974586486816, "prob": 9.20409704008307e-8 }, { "rank": 8, "tokenId": 77308, "text": "Azure", "logit": -10.069439888000488, "prob": 3.755356914325603e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 6.949852700610165, "winnerGap": 1.5731853887474685, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 6.949852700610165 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": 5.376667311862697 }, { "rank": 3, "tokenId": 149876, "text": "蓝色", "score": -4.095325212877043 }, { "rank": 4, "tokenId": 240123, "text": "蓝", "score": -7.636287289349784 }, { "rank": 5, "tokenId": 3730, "text": " blue", "score": -9.12503430098306 }, { "rank": 6, "tokenId": 236847, "text": "V", "score": -9.430030330456823 }, { "rank": 7, "tokenId": 51481, "text": "BLUE", "score": -9.476028493277951 }, { "rank": 8, "tokenId": 77308, "text": "Azure", "score": -10.475407550898836 } ] }, "f32_forward": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 6.9498419761657715, "winnerGap": 1.57318115234375, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 6.9498419761657715 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": 5.3766608238220215 }, { "rank": 3, "tokenId": 149876, "text": "蓝色", "score": -4.095317840576172 }, { "rank": 4, "tokenId": 240123, "text": "蓝", "score": -7.636275291442871 }, { "rank": 5, "tokenId": 3730, "text": " blue", "score": -9.12501049041748 }, { "rank": 6, "tokenId": 236847, "text": "V", "score": -9.430033683776855 }, { "rank": 7, "tokenId": 51481, "text": "BLUE", "score": -9.476024627685547 }, { "rank": 8, "tokenId": 77308, "text": "Azure", "score": -10.4754056930542 } ] }, "f32_reverse": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 6.949853897094727, "winnerGap": 1.573190689086914, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 6.949853897094727 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": 5.3766632080078125 }, { "rank": 3, "tokenId": 149876, "text": "蓝色", "score": -4.095317840576172 }, { "rank": 4, "tokenId": 240123, "text": "蓝", "score": -7.636288166046143 }, { "rank": 5, "tokenId": 3730, "text": " blue", "score": -9.12504768371582 }, { "rank": 6, "tokenId": 236847, "text": "V", "score": -9.430013656616211 }, { "rank": 7, "tokenId": 51481, "text": "BLUE", "score": -9.476018905639648 }, { "rank": 8, "tokenId": 77308, "text": "Azure", "score": -10.475411415100098 } ] }, "f32_tree64": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 6.949854373931885, "winnerGap": 1.5731878280639648, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 6.949854373931885 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": 5.37666654586792 }, { "rank": 3, "tokenId": 149876, "text": "蓝色", "score": -4.095324516296387 }, { "rank": 4, "tokenId": 240123, "text": "蓝", "score": -7.636286735534668 }, { "rank": 5, "tokenId": 3730, "text": " blue", "score": -9.125036239624023 }, { "rank": 6, "tokenId": 236847, "text": "V", "score": -9.430030822753906 }, { "rank": 7, "tokenId": 51481, "text": "BLUE", "score": -9.476031303405762 }, { "rank": 8, "tokenId": 77308, "text": "Azure", "score": -10.475406646728516 } ] }, "f16_forward": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 6.94140625, "winnerGap": 1.59765625, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 6.94140625 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": 5.34375 }, { "rank": 3, "tokenId": 72198, "text": "azure", "score": -3.26953125 }, { "rank": 4, "tokenId": 240123, "text": "蓝", "score": -3.662109375 }, { "rank": 5, "tokenId": 149876, "text": "蓝色", "score": -5.09375 }, { "rank": 6, "tokenId": 3730, "text": " blue", "score": -9.1328125 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": -9.4140625 }, { "rank": 8, "tokenId": 51481, "text": "BLUE", "score": -9.5625 } ] } }, "winnerTextsByMode": { "exact": "Blue", "f32_forward": "Blue", "f32_reverse": "Blue", "f32_tree64": "Blue", "f16_forward": "Blue" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Blue" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "railroad-go-stop-oneword-c", "text": "No explanation. One word only.\nAt a railroad crossing with the gate down, cars should go or stop. They should", "stableTopK": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "logit": 13.997762680053711, "prob": 0.9992370009422302 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "logit": 5.6361517906188965, "prob": 0.00023348932154476643 }, { "rank": 3, "tokenId": 142480, "text": "Proceed", "logit": 5.424160480499268, "prob": 0.00018888627528212965 }, { "rank": 4, "tokenId": 5988, "text": "Go", "logit": 4.967535495758057, "prob": 0.00011964393343077973 }, { "rank": 5, "tokenId": 11412, "text": "stop", "logit": 4.918230056762695, "prob": 0.00011388790153432637 }, { "rank": 6, "tokenId": 34230, "text": "Continue", "logit": 3.7968597412109375, "prob": 0.00003710843520821072 }, { "rank": 7, "tokenId": 82652, "text": "STOP", "logit": 2.9771018028259277, "prob": 0.000016347685232176445 }, { "rank": 8, "tokenId": 8653, "text": "Pass", "logit": 2.8241257667541504, "prob": 0.00001402877205691766 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 17348, "winnerText": "Stop", "winnerScore": 15.171154523860437, "winnerGap": 9.467248978958544, "top": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "score": 15.171154523860437 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 5.703905544901893 }, { "rank": 3, "tokenId": 142480, "text": "Proceed", "score": 5.484454293165553 }, { "rank": 4, "tokenId": 5988, "text": "Go", "score": 5.013698035276455 }, { "rank": 5, "tokenId": 11412, "text": "stop", "score": 4.963013616237802 }, { "rank": 6, "tokenId": 34230, "text": "Continue", "score": 3.817329132824227 }, { "rank": 7, "tokenId": 82652, "text": "STOP", "score": 2.9869347096065155 }, { "rank": 8, "tokenId": 8653, "text": "Pass", "score": 2.832514998006104 } ] }, "f32_forward": { "winnerTokenId": 17348, "winnerText": "Stop", "winnerScore": 15.171146392822266, "winnerGap": 9.467249393463135, "top": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "score": 15.171146392822266 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 5.703896999359131 }, { "rank": 3, "tokenId": 142480, "text": "Proceed", "score": 5.484461784362793 }, { "rank": 4, "tokenId": 5988, "text": "Go", "score": 5.013696670532227 }, { "rank": 5, "tokenId": 11412, "text": "stop", "score": 4.963014602661133 }, { "rank": 6, "tokenId": 34230, "text": "Continue", "score": 3.8173279762268066 }, { "rank": 7, "tokenId": 82652, "text": "STOP", "score": 2.986936092376709 }, { "rank": 8, "tokenId": 8653, "text": "Pass", "score": 2.832507610321045 } ] }, "f32_reverse": { "winnerTokenId": 17348, "winnerText": "Stop", "winnerScore": 15.171161651611328, "winnerGap": 9.46725606918335, "top": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "score": 15.171161651611328 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 5.7039055824279785 }, { "rank": 3, "tokenId": 142480, "text": "Proceed", "score": 5.48445463180542 }, { "rank": 4, "tokenId": 5988, "text": "Go", "score": 5.013700008392334 }, { "rank": 5, "tokenId": 11412, "text": "stop", "score": 4.963017463684082 }, { "rank": 6, "tokenId": 34230, "text": "Continue", "score": 3.8173279762268066 }, { "rank": 7, "tokenId": 82652, "text": "STOP", "score": 2.9869377613067627 }, { "rank": 8, "tokenId": 8653, "text": "Pass", "score": 2.8325159549713135 } ] }, "f32_tree64": { "winnerTokenId": 17348, "winnerText": "Stop", "winnerScore": 15.17115592956543, "winnerGap": 9.467249870300293, "top": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "score": 15.17115592956543 }, { "rank": 2, "tokenId": 20470, "text": "Wait", "score": 5.703906059265137 }, { "rank": 3, "tokenId": 142480, "text": "Proceed", "score": 5.484454154968262 }, { "rank": 4, "tokenId": 5988, "text": "Go", "score": 5.013697624206543 }, { "rank": 5, "tokenId": 11412, "text": "stop", "score": 4.963014125823975 }, { "rank": 6, "tokenId": 34230, "text": "Continue", "score": 3.8173294067382812 }, { "rank": 7, "tokenId": 82652, "text": "STOP", "score": 2.986933946609497 }, { "rank": 8, "tokenId": 8653, "text": "Pass", "score": 2.832515239715576 } ] }, "f16_forward": { "winnerTokenId": 17348, "winnerText": "Stop", "winnerScore": 14.1796875, "winnerGap": 8.62890625, "top": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "score": 14.1796875 }, { "rank": 2, "tokenId": 142480, "text": "Proceed", "score": 5.55078125 }, { "rank": 3, "tokenId": 5988, "text": "Go", "score": 5.12109375 }, { "rank": 4, "tokenId": 11412, "text": "stop", "score": 4.91015625 }, { "rank": 5, "tokenId": 34230, "text": "Continue", "score": 3.849609375 }, { "rank": 6, "tokenId": 82652, "text": "STOP", "score": 3.091796875 }, { "rank": 7, "tokenId": 8653, "text": "Pass", "score": 2.85546875 }, { "rank": 8, "tokenId": 8955, "text": "Enter", "score": 1.8310546875 } ] } }, "winnerTextsByMode": { "exact": "Stop", "f32_forward": "Stop", "f32_reverse": "Stop", "f32_tree64": "Stop", "f16_forward": "Stop" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Stop" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "red-go-stop-oneword-d", "text": "Choose one word only.\nAt a red traffic light, cars should go or stop. They should", "stableTopK": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "logit": 20.0372257232666, "prob": 0.6221206784248352 }, { "rank": 2, "tokenId": 11412, "text": "stop", "logit": 19.53172492980957, "prob": 0.37526530027389526 }, { "rank": 3, "tokenId": 1018, "text": "**", "logit": 14.549032211303711, "prob": 0.002572660567238927 }, { "rank": 4, "tokenId": 82652, "text": "STOP", "logit": 9.66455078125, "prob": 0.00001945714575413149 }, { "rank": 5, "tokenId": 20470, "text": "Wait", "logit": 9.444527626037598, "prob": 0.00001561436329211574 }, { "rank": 6, "tokenId": 4721, "text": " stop", "logit": 7.433713912963867, "prob": 0.0000020904462871840224 }, { "rank": 7, "tokenId": 198, "text": "", "logit": 6.63395881652832, "prob": 9.395280358148739e-7 }, { "rank": 8, "tokenId": 106, "text": "", "logit": 6.167020797729492, "prob": 5.890079819437233e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 17348, "winnerText": "Stop", "winnerScore": 24.208674430737425, "winnerGap": 0.8947141750515826, "top": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "score": 24.208674430737425 }, { "rank": 2, "tokenId": 11412, "text": "stop", "score": 23.313960255685842 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 15.883787689569493 }, { "rank": 4, "tokenId": 82652, "text": "STOP", "score": 10.021376712367319 }, { "rank": 5, "tokenId": 20470, "text": "Wait", "score": 9.7765215253397 }, { "rank": 6, "tokenId": 4721, "text": " stop", "score": 7.591725378535479 }, { "rank": 7, "tokenId": 198, "text": "", "score": 6.7453814391592655 }, { "rank": 8, "tokenId": 106, "text": "", "score": 6.256158505170816 } ] }, "f32_forward": { "winnerTokenId": 17348, "winnerText": "Stop", "winnerScore": 24.20867156982422, "winnerGap": 0.8947219848632812, "top": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "score": 24.20867156982422 }, { "rank": 2, "tokenId": 11412, "text": "stop", "score": 23.313949584960938 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 15.883785247802734 }, { "rank": 4, "tokenId": 82652, "text": "STOP", "score": 10.021367073059082 }, { "rank": 5, "tokenId": 20470, "text": "Wait", "score": 9.776514053344727 }, { "rank": 6, "tokenId": 4721, "text": " stop", "score": 7.591723918914795 }, { "rank": 7, "tokenId": 198, "text": "", "score": 6.7453813552856445 }, { "rank": 8, "tokenId": 106, "text": "", "score": 6.256162643432617 } ] }, "f32_reverse": { "winnerTokenId": 17348, "winnerText": "Stop", "winnerScore": 24.208675384521484, "winnerGap": 0.8947334289550781, "top": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "score": 24.208675384521484 }, { "rank": 2, "tokenId": 11412, "text": "stop", "score": 23.313941955566406 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 15.883788108825684 }, { "rank": 4, "tokenId": 82652, "text": "STOP", "score": 10.02137279510498 }, { "rank": 5, "tokenId": 20470, "text": "Wait", "score": 9.776521682739258 }, { "rank": 6, "tokenId": 4721, "text": " stop", "score": 7.5917277336120605 }, { "rank": 7, "tokenId": 198, "text": "", "score": 6.745384216308594 }, { "rank": 8, "tokenId": 106, "text": "", "score": 6.256162166595459 } ] }, "f32_tree64": { "winnerTokenId": 17348, "winnerText": "Stop", "winnerScore": 24.208675384521484, "winnerGap": 0.89471435546875, "top": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "score": 24.208675384521484 }, { "rank": 2, "tokenId": 11412, "text": "stop", "score": 23.313961029052734 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 15.883787155151367 }, { "rank": 4, "tokenId": 82652, "text": "STOP", "score": 10.02137565612793 }, { "rank": 5, "tokenId": 20470, "text": "Wait", "score": 9.776521682739258 }, { "rank": 6, "tokenId": 4721, "text": " stop", "score": 7.591727256774902 }, { "rank": 7, "tokenId": 198, "text": "", "score": 6.745381832122803 }, { "rank": 8, "tokenId": 106, "text": "", "score": 6.256158828735352 } ] }, "f16_forward": { "winnerTokenId": 17348, "winnerText": "Stop", "winnerScore": 24.21875, "winnerGap": 0.90625, "top": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "score": 24.21875 }, { "rank": 2, "tokenId": 11412, "text": "stop", "score": 23.3125 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 15.9296875 }, { "rank": 4, "tokenId": 82652, "text": "STOP", "score": 10.109375 }, { "rank": 5, "tokenId": 20470, "text": "Wait", "score": 9.8203125 }, { "rank": 6, "tokenId": 4721, "text": " stop", "score": 7.43359375 }, { "rank": 7, "tokenId": 198, "text": "", "score": 6.76171875 }, { "rank": 8, "tokenId": 236785, "text": "\\", "score": 6.28515625 } ] } }, "winnerTextsByMode": { "exact": "Stop", "f32_forward": "Stop", "f32_reverse": "Stop", "f32_tree64": "Stop", "f16_forward": "Stop" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Stop" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "red-light-oneword-a", "text": "Answer with exactly one word only.\nAt a red traffic light, cars should", "stableTopK": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "logit": 15.722555160522461, "prob": 0.999079167842865 }, { "rank": 2, "tokenId": 11412, "text": "stop", "logit": 8.046823501586914, "prob": 0.00046352375647984445 }, { "rank": 3, "tokenId": 82652, "text": "STOP", "logit": 7.368459701538086, "prob": 0.00023521356342826039 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "logit": 7.306577682495117, "prob": 0.00022109928249847144 }, { "rank": 5, "tokenId": 57561, "text": "Slow", "logit": 0.5452666282653809, "prob": 2.559689278314181e-7 }, { "rank": 6, "tokenId": 4721, "text": " stop", "logit": 0.5074012279510498, "prob": 2.4645777330079e-7 }, { "rank": 7, "tokenId": 236814, "text": "H", "logit": 0.23079928755760193, "prob": 1.8690280967348372e-7 }, { "rank": 8, "tokenId": 236773, "text": "S", "logit": -0.4421924650669098, "prob": 9.535406064742347e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 17348, "winnerText": "Stop", "winnerScore": 17.458662793911873, "winnerGap": 9.210072604087532, "top": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "score": 17.458662793911873 }, { "rank": 2, "tokenId": 11412, "text": "stop", "score": 8.248590189824341 }, { "rank": 3, "tokenId": 82652, "text": "STOP", "score": 7.522235115021772 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "score": 7.4564212324174335 }, { "rank": 5, "tokenId": 57561, "text": "Slow", "score": 0.5453291156130327 }, { "rank": 6, "tokenId": 4721, "text": " stop", "score": 0.5074493204103945 }, { "rank": 7, "tokenId": 236814, "text": "H", "score": 0.23080455974928782 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": -0.44222738254619287 } ] }, "f32_forward": { "winnerTokenId": 17348, "winnerText": "Stop", "winnerScore": 17.458681106567383, "winnerGap": 9.210086822509766, "top": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "score": 17.458681106567383 }, { "rank": 2, "tokenId": 11412, "text": "stop", "score": 8.248594284057617 }, { "rank": 3, "tokenId": 82652, "text": "STOP", "score": 7.522233009338379 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "score": 7.456422805786133 }, { "rank": 5, "tokenId": 57561, "text": "Slow", "score": 0.5453430414199829 }, { "rank": 6, "tokenId": 4721, "text": " stop", "score": 0.5074491500854492 }, { "rank": 7, "tokenId": 236814, "text": "H", "score": 0.2308032363653183 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": -0.4422277808189392 } ] }, "f32_reverse": { "winnerTokenId": 17348, "winnerText": "Stop", "winnerScore": 17.45866584777832, "winnerGap": 9.210078239440918, "top": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "score": 17.45866584777832 }, { "rank": 2, "tokenId": 11412, "text": "stop", "score": 8.248587608337402 }, { "rank": 3, "tokenId": 82652, "text": "STOP", "score": 7.522234916687012 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "score": 7.456420421600342 }, { "rank": 5, "tokenId": 57561, "text": "Slow", "score": 0.5453429222106934 }, { "rank": 6, "tokenId": 4721, "text": " stop", "score": 0.5074509978294373 }, { "rank": 7, "tokenId": 236814, "text": "H", "score": 0.23080427944660187 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": -0.44222697615623474 } ] }, "f32_tree64": { "winnerTokenId": 17348, "winnerText": "Stop", "winnerScore": 17.458663940429688, "winnerGap": 9.21007251739502, "top": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "score": 17.458663940429688 }, { "rank": 2, "tokenId": 11412, "text": "stop", "score": 8.248591423034668 }, { "rank": 3, "tokenId": 82652, "text": "STOP", "score": 7.522235870361328 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "score": 7.4564208984375 }, { "rank": 5, "tokenId": 57561, "text": "Slow", "score": 0.54532790184021 }, { "rank": 6, "tokenId": 4721, "text": " stop", "score": 0.5074472427368164 }, { "rank": 7, "tokenId": 236814, "text": "H", "score": 0.2308058738708496 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": -0.4422264099121094 } ] }, "f16_forward": { "winnerTokenId": 17348, "winnerText": "Stop", "winnerScore": 17.609375, "winnerGap": 9.328125, "top": [ { "rank": 1, "tokenId": 17348, "text": "Stop", "score": 17.609375 }, { "rank": 2, "tokenId": 11412, "text": "stop", "score": 8.28125 }, { "rank": 3, "tokenId": 82652, "text": "STOP", "score": 7.5625 }, { "rank": 4, "tokenId": 20470, "text": "Wait", "score": 3.38671875 }, { "rank": 5, "tokenId": 57561, "text": "Slow", "score": 0.6826171875 }, { "rank": 6, "tokenId": 4721, "text": " stop", "score": 0.481689453125 }, { "rank": 7, "tokenId": 236814, "text": "H", "score": 0.2119140625 }, { "rank": 8, "tokenId": 236773, "text": "S", "score": -0.46728515625 } ] } }, "winnerTextsByMode": { "exact": "Stop", "f32_forward": "Stop", "f32_reverse": "Stop", "f32_tree64": "Stop", "f16_forward": "Stop" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Stop" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "repeat-space-oneword-b", "text": "Respond with one word only.\nThe sky is blue ", "stableTopK": [ { "rank": 1, "tokenId": 4339, "text": "True", "logit": 14.297186851501465, "prob": 0.9943613409996033 }, { "rank": 2, "tokenId": 16520, "text": "Blue", "logit": 9.01053524017334, "prob": 0.005030146334320307 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "logit": 6.374476432800293, "prob": 0.00036037512472830713 }, { "rank": 4, "tokenId": 34318, "text": "Correct", "logit": 5.616120338439941, "prob": 0.000168812635820359 }, { "rank": 5, "tokenId": 51077, "text": "Indeed", "logit": 4.246041297912598, "prob": 0.000042893072532024235 }, { "rank": 6, "tokenId": 5475, "text": "Acc", "logit": 3.1861274242401123, "prob": 0.000014861835552437697 }, { "rank": 7, "tokenId": 74041, "text": "Truth", "logit": 2.4384477138519287, "prob": 0.000007036541774141369 }, { "rank": 8, "tokenId": 27711, "text": "Fact", "logit": 1.4303046464920044, "prob": 0.0000025676056338852504 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 15.556215900491203, "winnerGap": 6.259042818933509, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 15.556215900491203 }, { "rank": 2, "tokenId": 16520, "text": "Blue", "score": 9.297173081557695 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 6.473095436809977 }, { "rank": 4, "tokenId": 34318, "text": "Correct", "score": 5.683142293479776 }, { "rank": 5, "tokenId": 51077, "text": "Indeed", "score": 4.274734923262578 }, { "rank": 6, "tokenId": 5475, "text": "Acc", "score": 3.1981891271067227 }, { "rank": 7, "tokenId": 74041, "text": "Truth", "score": 2.4438380621692417 }, { "rank": 8, "tokenId": 27711, "text": "Fact", "score": 1.4313962911651306 } ] }, "f32_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 15.5562162399292, "winnerGap": 6.25904655456543, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 15.5562162399292 }, { "rank": 2, "tokenId": 16520, "text": "Blue", "score": 9.29716968536377 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 6.4730916023254395 }, { "rank": 4, "tokenId": 34318, "text": "Correct", "score": 5.683135986328125 }, { "rank": 5, "tokenId": 51077, "text": "Indeed", "score": 4.274738311767578 }, { "rank": 6, "tokenId": 5475, "text": "Acc", "score": 3.1981911659240723 }, { "rank": 7, "tokenId": 74041, "text": "Truth", "score": 2.4438376426696777 }, { "rank": 8, "tokenId": 27711, "text": "Fact", "score": 1.4313936233520508 } ] }, "f32_reverse": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 15.556220054626465, "winnerGap": 6.259049415588379, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 15.556220054626465 }, { "rank": 2, "tokenId": 16520, "text": "Blue", "score": 9.297170639038086 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 6.4730963706970215 }, { "rank": 4, "tokenId": 34318, "text": "Correct", "score": 5.683140277862549 }, { "rank": 5, "tokenId": 51077, "text": "Indeed", "score": 4.274736404418945 }, { "rank": 6, "tokenId": 5475, "text": "Acc", "score": 3.198190212249756 }, { "rank": 7, "tokenId": 74041, "text": "Truth", "score": 2.443838357925415 }, { "rank": 8, "tokenId": 27711, "text": "Fact", "score": 1.4313921928405762 } ] }, "f32_tree64": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 15.556215286254883, "winnerGap": 6.259040832519531, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 15.556215286254883 }, { "rank": 2, "tokenId": 16520, "text": "Blue", "score": 9.297174453735352 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 6.473094940185547 }, { "rank": 4, "tokenId": 34318, "text": "Correct", "score": 5.68314266204834 }, { "rank": 5, "tokenId": 51077, "text": "Indeed", "score": 4.274735450744629 }, { "rank": 6, "tokenId": 5475, "text": "Acc", "score": 3.198188304901123 }, { "rank": 7, "tokenId": 74041, "text": "Truth", "score": 2.443838119506836 }, { "rank": 8, "tokenId": 27711, "text": "Fact", "score": 1.4313966035842896 } ] }, "f16_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 15.515625, "winnerGap": 6.0078125, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 15.515625 }, { "rank": 2, "tokenId": 16520, "text": "Blue", "score": 9.5078125 }, { "rank": 3, "tokenId": 10784, "text": "Yes", "score": 6.53125 }, { "rank": 4, "tokenId": 34318, "text": "Correct", "score": 5.70703125 }, { "rank": 5, "tokenId": 51077, "text": "Indeed", "score": 4.2578125 }, { "rank": 6, "tokenId": 5475, "text": "Acc", "score": 3.109375 }, { "rank": 7, "tokenId": 74041, "text": "Truth", "score": 2.458984375 }, { "rank": 8, "tokenId": 27711, "text": "Fact", "score": 1.4423828125 } ] } }, "winnerTextsByMode": { "exact": "True", "f32_forward": "True", "f32_reverse": "True", "f32_tree64": "True", "f16_forward": "True" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "True" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "rollback-approve-deny-oneword-c", "text": "No explanation. One word only.\nA rollback that restores a known exploitable vulnerability should be approved or denied. It should be", "stableTopK": [ { "rank": 1, "tokenId": 75938, "text": "Approved", "logit": 19.051950454711914, "prob": 0.86698979139328 }, { "rank": 2, "tokenId": 147942, "text": "Denied", "logit": 16.570316314697266, "prob": 0.07248597592115402 }, { "rank": 3, "tokenId": 87732, "text": "Decision", "logit": 14.473526000976562, "prob": 0.00890490971505642 }, { "rank": 4, "tokenId": 172496, "text": "Reviewed", "logit": 14.335343360900879, "prob": 0.007755638565868139 }, { "rank": 5, "tokenId": 22472, "text": "Den", "logit": 14.310897827148438, "prob": 0.007568346336483955 }, { "rank": 6, "tokenId": 8724, "text": "Dec", "logit": 13.612900733947754, "prob": 0.0037658647634088993 }, { "rank": 7, "tokenId": 182391, "text": "Rejected", "logit": 13.414363861083984, "prob": 0.0030877436511218548 }, { "rank": 8, "tokenId": 18463, "text": "Perm", "logit": 13.314488410949707, "prob": 0.002794253872707486 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 75938, "winnerText": "Approved", "winnerScore": 22.49578424837729, "winnerGap": 3.8433461865304643, "top": [ { "rank": 1, "tokenId": 75938, "text": "Approved", "score": 22.49578424837729 }, { "rank": 2, "tokenId": 147942, "text": "Denied", "score": 18.652438061846826 }, { "rank": 3, "tokenId": 87732, "text": "Decision", "score": 15.785205963955264 }, { "rank": 4, "tokenId": 172496, "text": "Reviewed", "score": 15.60562711060253 }, { "rank": 5, "tokenId": 22472, "text": "Den", "score": 15.573968046713231 }, { "rank": 6, "tokenId": 8724, "text": "Dec", "score": 14.68288035340072 }, { "rank": 7, "tokenId": 182391, "text": "Rejected", "score": 14.433801369123344 }, { "rank": 8, "tokenId": 18463, "text": "Perm", "score": 14.309199572211867 } ] }, "f32_forward": { "winnerTokenId": 75938, "winnerText": "Approved", "winnerScore": 22.495798110961914, "winnerGap": 3.843355178833008, "top": [ { "rank": 1, "tokenId": 75938, "text": "Approved", "score": 22.495798110961914 }, { "rank": 2, "tokenId": 147942, "text": "Denied", "score": 18.652442932128906 }, { "rank": 3, "tokenId": 87732, "text": "Decision", "score": 15.785199165344238 }, { "rank": 4, "tokenId": 172496, "text": "Reviewed", "score": 15.605621337890625 }, { "rank": 5, "tokenId": 22472, "text": "Den", "score": 15.573944091796875 }, { "rank": 6, "tokenId": 8724, "text": "Dec", "score": 14.682888984680176 }, { "rank": 7, "tokenId": 182391, "text": "Rejected", "score": 14.433791160583496 }, { "rank": 8, "tokenId": 18463, "text": "Perm", "score": 14.30919361114502 } ] }, "f32_reverse": { "winnerTokenId": 75938, "winnerText": "Approved", "winnerScore": 22.495786666870117, "winnerGap": 3.843364715576172, "top": [ { "rank": 1, "tokenId": 75938, "text": "Approved", "score": 22.495786666870117 }, { "rank": 2, "tokenId": 147942, "text": "Denied", "score": 18.652421951293945 }, { "rank": 3, "tokenId": 87732, "text": "Decision", "score": 15.785202980041504 }, { "rank": 4, "tokenId": 172496, "text": "Reviewed", "score": 15.605618476867676 }, { "rank": 5, "tokenId": 22472, "text": "Den", "score": 15.573972702026367 }, { "rank": 6, "tokenId": 8724, "text": "Dec", "score": 14.682884216308594 }, { "rank": 7, "tokenId": 182391, "text": "Rejected", "score": 14.433793067932129 }, { "rank": 8, "tokenId": 18463, "text": "Perm", "score": 14.309194564819336 } ] }, "f32_tree64": { "winnerTokenId": 75938, "winnerText": "Approved", "winnerScore": 22.49578285217285, "winnerGap": 3.8433456420898438, "top": [ { "rank": 1, "tokenId": 75938, "text": "Approved", "score": 22.49578285217285 }, { "rank": 2, "tokenId": 147942, "text": "Denied", "score": 18.652437210083008 }, { "rank": 3, "tokenId": 87732, "text": "Decision", "score": 15.785204887390137 }, { "rank": 4, "tokenId": 172496, "text": "Reviewed", "score": 15.605626106262207 }, { "rank": 5, "tokenId": 22472, "text": "Den", "score": 15.573968887329102 }, { "rank": 6, "tokenId": 8724, "text": "Dec", "score": 14.682880401611328 }, { "rank": 7, "tokenId": 182391, "text": "Rejected", "score": 14.43380069732666 }, { "rank": 8, "tokenId": 18463, "text": "Perm", "score": 14.309200286865234 } ] }, "f16_forward": { "winnerTokenId": 147942, "winnerText": "Denied", "winnerScore": 18.859375, "winnerGap": 0.328125, "top": [ { "rank": 1, "tokenId": 147942, "text": "Denied", "score": 18.859375 }, { "rank": 2, "tokenId": 75938, "text": "Approved", "score": 18.53125 }, { "rank": 3, "tokenId": 22472, "text": "Den", "score": 15.578125 }, { "rank": 4, "tokenId": 172496, "text": "Reviewed", "score": 15.5390625 }, { "rank": 5, "tokenId": 8724, "text": "Dec", "score": 14.765625 }, { "rank": 6, "tokenId": 87732, "text": "Decision", "score": 14.6953125 }, { "rank": 7, "tokenId": 182391, "text": "Rejected", "score": 14.453125 }, { "rank": 8, "tokenId": 18463, "text": "Perm", "score": 14.296875 } ] } }, "winnerTextsByMode": { "exact": "Approved", "f32_forward": "Approved", "f32_reverse": "Approved", "f32_tree64": "Approved", "f16_forward": "Denied" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Approved", "Denied" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 75938, 106 ], "decodedText": "Approved", "steps": [ { "step": 0, "tokenId": 75938, "text": "Approved" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 147942, 106 ], "decodedText": "Denied", "steps": [ { "step": 0, "tokenId": 147942, "text": "Denied" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "sacred-historical-explicit-oneword-d", "text": "Choose one word only.\nAnswer with exactly one word: sacred or historical. Question: An ancient burial site is Answer:", "stableTopK": [ { "rank": 1, "tokenId": 124977, "text": "historical", "logit": 9.7715425491333, "prob": 0.9993612170219421 }, { "rank": 2, "tokenId": 59081, "text": "sac", "logit": 2.405613899230957, "prob": 0.0006320337997749448 }, { "rank": 3, "tokenId": 69421, "text": "historic", "logit": -2.4644081592559814, "prob": 0.000004849719516641926 }, { "rank": 4, "tokenId": 100368, "text": "Historical", "logit": -4.161905765533447, "prob": 8.881835924512416e-7 }, { "rank": 5, "tokenId": 39421, "text": "histor", "logit": -4.195071220397949, "prob": 8.592097628934425e-7 }, { "rank": 6, "tokenId": 17637, "text": "hist", "logit": -7.201154708862305, "prob": 4.25180886054477e-8 }, { "rank": 7, "tokenId": 1018, "text": "**", "logit": -8.089425086975098, "prob": 1.7490522807861453e-8 }, { "rank": 8, "tokenId": 1338, "text": "arch", "logit": -8.265972137451172, "prob": 1.4659845781750391e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 124977, "winnerText": "historical", "winnerScore": 10.140925973192573, "winnerGap": 7.730138432840704, "top": [ { "rank": 1, "tokenId": 124977, "text": "historical", "score": 10.140925973192573 }, { "rank": 2, "tokenId": 59081, "text": "sac", "score": 2.410787540351869 }, { "rank": 3, "tokenId": 69421, "text": "historic", "score": -2.469974102847253 }, { "rank": 4, "tokenId": 100368, "text": "Historical", "score": -4.188919812865393 }, { "rank": 5, "tokenId": 39421, "text": "histor", "score": -4.222738726431899 }, { "rank": 6, "tokenId": 17637, "text": "hist", "score": -7.3444419343163005 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -8.294512191591974 }, { "rank": 8, "tokenId": 1338, "text": "arch", "score": -8.485232223321177 } ] }, "f32_forward": { "winnerTokenId": 124977, "winnerText": "historical", "winnerScore": 10.140926361083984, "winnerGap": 7.7301318645477295, "top": [ { "rank": 1, "tokenId": 124977, "text": "historical", "score": 10.140926361083984 }, { "rank": 2, "tokenId": 59081, "text": "sac", "score": 2.410794496536255 }, { "rank": 3, "tokenId": 69421, "text": "historic", "score": -2.469968795776367 }, { "rank": 4, "tokenId": 100368, "text": "Historical", "score": -4.1889214515686035 }, { "rank": 5, "tokenId": 39421, "text": "histor", "score": -4.2227396965026855 }, { "rank": 6, "tokenId": 17637, "text": "hist", "score": -7.344449996948242 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -8.294507026672363 }, { "rank": 8, "tokenId": 1338, "text": "arch", "score": -8.48523235321045 } ] }, "f32_reverse": { "winnerTokenId": 124977, "winnerText": "historical", "winnerScore": 10.1409330368042, "winnerGap": 7.7301411628723145, "top": [ { "rank": 1, "tokenId": 124977, "text": "historical", "score": 10.1409330368042 }, { "rank": 2, "tokenId": 59081, "text": "sac", "score": 2.4107918739318848 }, { "rank": 3, "tokenId": 69421, "text": "historic", "score": -2.4699788093566895 }, { "rank": 4, "tokenId": 100368, "text": "Historical", "score": -4.188922882080078 }, { "rank": 5, "tokenId": 39421, "text": "histor", "score": -4.222746849060059 }, { "rank": 6, "tokenId": 17637, "text": "hist", "score": -7.344438076019287 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -8.294508934020996 }, { "rank": 8, "tokenId": 1338, "text": "arch", "score": -8.48523998260498 } ] }, "f32_tree64": { "winnerTokenId": 124977, "winnerText": "historical", "winnerScore": 10.140926361083984, "winnerGap": 7.730139255523682, "top": [ { "rank": 1, "tokenId": 124977, "text": "historical", "score": 10.140926361083984 }, { "rank": 2, "tokenId": 59081, "text": "sac", "score": 2.4107871055603027 }, { "rank": 3, "tokenId": 69421, "text": "historic", "score": -2.4699740409851074 }, { "rank": 4, "tokenId": 100368, "text": "Historical", "score": -4.188918113708496 }, { "rank": 5, "tokenId": 39421, "text": "histor", "score": -4.222739219665527 }, { "rank": 6, "tokenId": 17637, "text": "hist", "score": -7.344442367553711 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -8.294512748718262 }, { "rank": 8, "tokenId": 1338, "text": "arch", "score": -8.48523235321045 } ] }, "f16_forward": { "winnerTokenId": 124977, "winnerText": "historical", "winnerScore": 5.2890625, "winnerGap": 2.810546875, "top": [ { "rank": 1, "tokenId": 124977, "text": "historical", "score": 5.2890625 }, { "rank": 2, "tokenId": 59081, "text": "sac", "score": 2.478515625 }, { "rank": 3, "tokenId": 69421, "text": "historic", "score": -2.4453125 }, { "rank": 4, "tokenId": 100368, "text": "Historical", "score": -3.15625 }, { "rank": 5, "tokenId": 139, "text": " ", "score": -4.33984375 }, { "rank": 6, "tokenId": 89354, "text": "Sac", "score": -4.5703125 }, { "rank": 7, "tokenId": 39421, "text": "histor", "score": -5.17578125 }, { "rank": 8, "tokenId": 2267, "text": "An", "score": -5.4140625 } ] } }, "winnerTextsByMode": { "exact": "historical", "f32_forward": "historical", "f32_reverse": "historical", "f32_tree64": "historical", "f16_forward": "historical" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "historical" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "safe-unsafe-brakes-oneword-a", "text": "Answer with exactly one word only.\nAnswer with exactly one word: safe or unsafe.\nQuestion: Driving without brakes is\nAnswer:", "stableTopK": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "logit": 15.606432914733887, "prob": 0.9952934980392456 }, { "rank": 2, "tokenId": 2805, "text": "Un", "logit": 10.252053260803223, "prob": 0.004705152008682489 }, { "rank": 3, "tokenId": 8069, "text": "uns", "logit": 1.793541669845581, "prob": 9.979039532481693e-7 }, { "rank": 4, "tokenId": 602, "text": "un", "logit": 0.6988500356674194, "prob": 3.3394132969988277e-7 }, { "rank": 5, "tokenId": 25793, "text": " unsafe", "logit": -2.7390029430389404, "prob": 1.0730738075892532e-8 }, { "rank": 6, "tokenId": 193883, "text": "dangerous", "logit": -5.063282012939453, "prob": 1.0500453839767943e-9 }, { "rank": 7, "tokenId": 25083, "text": "unsigned", "logit": -5.3020172119140625, "prob": 8.270402696020085e-10 }, { "rank": 8, "tokenId": 38699, "text": "Una", "logit": -5.40197229385376, "prob": 7.483706432331871e-10 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 17.299007523794156, "winnerGap": 6.617338464886121, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 17.299007523794156 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 10.681669058908035 }, { "rank": 3, "tokenId": 8069, "text": "uns", "score": 1.7956843530460294 }, { "rank": 4, "tokenId": 602, "text": "un", "score": 0.6989762535952657 }, { "rank": 5, "tokenId": 25793, "text": " unsafe", "score": -2.746641576860604 }, { "rank": 6, "tokenId": 193883, "text": "dangerous", "score": -5.112202963175278 }, { "rank": 7, "tokenId": 25083, "text": "unsigned", "score": -5.358280985205063 }, { "rank": 8, "tokenId": 38699, "text": "Una", "score": -5.461519380928937 } ] }, "f32_forward": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 17.29900550842285, "winnerGap": 6.617336273193359, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 17.29900550842285 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 10.681669235229492 }, { "rank": 3, "tokenId": 8069, "text": "uns", "score": 1.7956866025924683 }, { "rank": 4, "tokenId": 602, "text": "un", "score": 0.6989661455154419 }, { "rank": 5, "tokenId": 25793, "text": " unsafe", "score": -2.7466442584991455 }, { "rank": 6, "tokenId": 193883, "text": "dangerous", "score": -5.112204074859619 }, { "rank": 7, "tokenId": 25083, "text": "unsigned", "score": -5.358274459838867 }, { "rank": 8, "tokenId": 38699, "text": "Una", "score": -5.461511135101318 } ] }, "f32_reverse": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 17.29901695251465, "winnerGap": 6.617348670959473, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 17.29901695251465 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 10.681668281555176 }, { "rank": 3, "tokenId": 8069, "text": "uns", "score": 1.795688509941101 }, { "rank": 4, "tokenId": 602, "text": "un", "score": 0.6989699602127075 }, { "rank": 5, "tokenId": 25793, "text": " unsafe", "score": -2.7466461658477783 }, { "rank": 6, "tokenId": 193883, "text": "dangerous", "score": -5.1121978759765625 }, { "rank": 7, "tokenId": 25083, "text": "unsigned", "score": -5.358260631561279 }, { "rank": 8, "tokenId": 38699, "text": "Una", "score": -5.46152400970459 } ] }, "f32_tree64": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 17.299007415771484, "winnerGap": 6.617337226867676, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 17.299007415771484 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 10.681670188903809 }, { "rank": 3, "tokenId": 8069, "text": "uns", "score": 1.7956860065460205 }, { "rank": 4, "tokenId": 602, "text": "un", "score": 0.6989760398864746 }, { "rank": 5, "tokenId": 25793, "text": " unsafe", "score": -2.746640682220459 }, { "rank": 6, "tokenId": 193883, "text": "dangerous", "score": -5.112203121185303 }, { "rank": 7, "tokenId": 25083, "text": "unsigned", "score": -5.358281135559082 }, { "rank": 8, "tokenId": 38699, "text": "Una", "score": -5.461519718170166 } ] }, "f16_forward": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 17.390625, "winnerGap": 6.703125, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 17.390625 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 10.6875 }, { "rank": 3, "tokenId": 8069, "text": "uns", "score": 0.83447265625 }, { "rank": 4, "tokenId": 602, "text": "un", "score": 0.73095703125 }, { "rank": 5, "tokenId": 236786, "text": "/", "score": -2.15234375 }, { "rank": 6, "tokenId": 25793, "text": " unsafe", "score": -2.6875 }, { "rank": 7, "tokenId": 39316, "text": "Safe", "score": -3.439453125 }, { "rank": 8, "tokenId": 26546, "text": "Very", "score": -4.80078125 } ] } }, "winnerTextsByMode": { "exact": "unsafe", "f32_forward": "unsafe", "f32_reverse": "unsafe", "f32_tree64": "unsafe", "f16_forward": "unsafe" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "unsafe" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "sci-ai-understanding-oneword-b", "text": "Respond with one word only.\nLarge language models understand language — true or false?", "stableTopK": [ { "rank": 1, "tokenId": 4339, "text": "True", "logit": 11.903637886047363, "prob": 0.9995732307434082 }, { "rank": 2, "tokenId": 9277, "text": "False", "logit": 4.089116096496582, "prob": 0.0004036555765196681 }, { "rank": 3, "tokenId": 3397, "text": "true", "logit": 0.8308465480804443, "prob": 0.00001552252797409892 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "logit": -1.4077584743499756, "prob": 0.0000016548119674553163 }, { "rank": 5, "tokenId": 61460, "text": "Generally", "logit": -2.525566339492798, "prob": 5.411166057456285e-7 }, { "rank": 6, "tokenId": 196224, "text": "Mostly", "logit": -2.91341233253479, "prob": 3.67156729907947e-7 }, { "rank": 7, "tokenId": 22369, "text": "TRUE", "logit": -3.6660730838775635, "prob": 1.7297170984420518e-7 }, { "rank": 8, "tokenId": 6288, "text": " True", "logit": -3.673219919204712, "prob": 1.7173991295749147e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 12.594927503325112, "winnerGap": 8.480201067511734, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 12.594927503325112 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 4.1147264358133775 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": 0.8310599066571243 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": -1.4087914552673766 }, { "rank": 5, "tokenId": 61460, "text": "Generally", "score": -2.5315570879099667 }, { "rank": 6, "tokenId": 196224, "text": "Mostly", "score": -2.9226209960170646 }, { "rank": 7, "tokenId": 22369, "text": "TRUE", "score": -3.6844877564979193 }, { "rank": 8, "tokenId": 6288, "text": " True", "score": -3.6917402165407776 } ] }, "f32_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 12.594927787780762, "winnerGap": 8.480204582214355, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 12.594927787780762 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 4.114723205566406 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": 0.8310614228248596 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": -1.4087908267974854 }, { "rank": 5, "tokenId": 61460, "text": "Generally", "score": -2.53155517578125 }, { "rank": 6, "tokenId": 196224, "text": "Mostly", "score": -2.922623872756958 }, { "rank": 7, "tokenId": 22369, "text": "TRUE", "score": -3.6844844818115234 }, { "rank": 8, "tokenId": 6288, "text": " True", "score": -3.691737174987793 } ] }, "f32_reverse": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 12.594932556152344, "winnerGap": 8.480208396911621, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 12.594932556152344 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 4.114724159240723 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": 0.8310598731040955 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": -1.4087951183319092 }, { "rank": 5, "tokenId": 61460, "text": "Generally", "score": -2.5315515995025635 }, { "rank": 6, "tokenId": 196224, "text": "Mostly", "score": -2.9226174354553223 }, { "rank": 7, "tokenId": 22369, "text": "TRUE", "score": -3.6844842433929443 }, { "rank": 8, "tokenId": 6288, "text": " True", "score": -3.691739082336426 } ] }, "f32_tree64": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 12.594927787780762, "winnerGap": 8.480201244354248, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 12.594927787780762 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 4.114726543426514 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": 0.8310601115226746 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": -1.4087910652160645 }, { "rank": 5, "tokenId": 61460, "text": "Generally", "score": -2.531557559967041 }, { "rank": 6, "tokenId": 196224, "text": "Mostly", "score": -2.9226207733154297 }, { "rank": 7, "tokenId": 22369, "text": "TRUE", "score": -3.684487819671631 }, { "rank": 8, "tokenId": 6288, "text": " True", "score": -3.6917405128479004 } ] }, "f16_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 12.515625, "winnerGap": 7.390625, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 12.515625 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 5.125 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": -0.1771240234375 }, { "rank": 4, "tokenId": 10784, "text": "Yes", "score": -1.421875 }, { "rank": 5, "tokenId": 61460, "text": "Generally", "score": -2.505859375 }, { "rank": 6, "tokenId": 196224, "text": "Mostly", "score": -2.90625 }, { "rank": 7, "tokenId": 22369, "text": "TRUE", "score": -3.68359375 }, { "rank": 8, "tokenId": 6288, "text": " True", "score": -3.728515625 } ] } }, "winnerTextsByMode": { "exact": "True", "f32_forward": "True", "f32_reverse": "True", "f32_tree64": "True", "f16_forward": "True" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "True" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "sci-arrow-of-time-oneword-c", "text": "No explanation. One word only.\nTime flows forward because of entropy or something deeper. It is because of", "stableTopK": [ { "rank": 1, "tokenId": 32089, "text": "Arrow", "logit": 8.825662612915039, "prob": 0.42915984988212585 }, { "rank": 2, "tokenId": 193168, "text": "Existence", "logit": 7.9264044761657715, "prob": 0.1746128499507904 }, { "rank": 3, "tokenId": 236780, "text": "C", "logit": 7.508363246917725, "prob": 0.11495377123355865 }, { "rank": 4, "tokenId": 67861, "text": "Cause", "logit": 7.011105537414551, "prob": 0.06991444528102875 }, { "rank": 5, "tokenId": 9233, "text": "Change", "logit": 6.194891452789307, "prob": 0.030909337103366852 }, { "rank": 6, "tokenId": 40607, "text": "Nothing", "logit": 5.923997402191162, "prob": 0.02357446774840355 }, { "rank": 7, "tokenId": 46797, "text": "Nature", "logit": 5.884981632232666, "prob": 0.02267240174114704 }, { "rank": 8, "tokenId": 29096, "text": "Ir", "logit": 5.712205410003662, "prob": 0.019074879586696625 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 32089, "winnerText": "Arrow", "winnerScore": 9.09437080212379, "winnerGap": 0.975379285019768, "top": [ { "rank": 1, "tokenId": 32089, "text": "Arrow", "score": 9.09437080212379 }, { "rank": 2, "tokenId": 193168, "text": "Existence", "score": 8.118991517104021 }, { "rank": 3, "tokenId": 236780, "text": "C", "score": 7.671306557162939 }, { "rank": 4, "tokenId": 67861, "text": "Cause", "score": 7.143104547401272 }, { "rank": 5, "tokenId": 9233, "text": "Change", "score": 6.285266655766222 }, { "rank": 6, "tokenId": 40607, "text": "Nothing", "score": 6.0028485622332255 }, { "rank": 7, "tokenId": 46797, "text": "Nature", "score": 5.962263850340037 }, { "rank": 8, "tokenId": 29096, "text": "Ir", "score": 5.782777391877595 } ] }, "f32_forward": { "winnerTokenId": 32089, "winnerText": "Arrow", "winnerScore": 9.09436321258545, "winnerGap": 0.9753828048706055, "top": [ { "rank": 1, "tokenId": 32089, "text": "Arrow", "score": 9.09436321258545 }, { "rank": 2, "tokenId": 193168, "text": "Existence", "score": 8.118980407714844 }, { "rank": 3, "tokenId": 236780, "text": "C", "score": 7.6713128089904785 }, { "rank": 4, "tokenId": 67861, "text": "Cause", "score": 7.143113136291504 }, { "rank": 5, "tokenId": 9233, "text": "Change", "score": 6.285266399383545 }, { "rank": 6, "tokenId": 40607, "text": "Nothing", "score": 6.002851486206055 }, { "rank": 7, "tokenId": 46797, "text": "Nature", "score": 5.9622650146484375 }, { "rank": 8, "tokenId": 29096, "text": "Ir", "score": 5.782763481140137 } ] }, "f32_reverse": { "winnerTokenId": 32089, "winnerText": "Arrow", "winnerScore": 9.094367027282715, "winnerGap": 0.9753761291503906, "top": [ { "rank": 1, "tokenId": 32089, "text": "Arrow", "score": 9.094367027282715 }, { "rank": 2, "tokenId": 193168, "text": "Existence", "score": 8.118990898132324 }, { "rank": 3, "tokenId": 236780, "text": "C", "score": 7.6713080406188965 }, { "rank": 4, "tokenId": 67861, "text": "Cause", "score": 7.143104076385498 }, { "rank": 5, "tokenId": 9233, "text": "Change", "score": 6.285266876220703 }, { "rank": 6, "tokenId": 40607, "text": "Nothing", "score": 6.002846717834473 }, { "rank": 7, "tokenId": 46797, "text": "Nature", "score": 5.962265968322754 }, { "rank": 8, "tokenId": 29096, "text": "Ir", "score": 5.782776355743408 } ] }, "f32_tree64": { "winnerTokenId": 32089, "winnerText": "Arrow", "winnerScore": 9.09437084197998, "winnerGap": 0.9753789901733398, "top": [ { "rank": 1, "tokenId": 32089, "text": "Arrow", "score": 9.09437084197998 }, { "rank": 2, "tokenId": 193168, "text": "Existence", "score": 8.11899185180664 }, { "rank": 3, "tokenId": 236780, "text": "C", "score": 7.671306610107422 }, { "rank": 4, "tokenId": 67861, "text": "Cause", "score": 7.143104076385498 }, { "rank": 5, "tokenId": 9233, "text": "Change", "score": 6.285265922546387 }, { "rank": 6, "tokenId": 40607, "text": "Nothing", "score": 6.002849102020264 }, { "rank": 7, "tokenId": 46797, "text": "Nature", "score": 5.9622626304626465 }, { "rank": 8, "tokenId": 29096, "text": "Ir", "score": 5.782776832580566 } ] }, "f16_forward": { "winnerTokenId": 193168, "winnerText": "Existence", "winnerScore": 8.1015625, "winnerGap": 0.37890625, "top": [ { "rank": 1, "tokenId": 193168, "text": "Existence", "score": 8.1015625 }, { "rank": 2, "tokenId": 236780, "text": "C", "score": 7.72265625 }, { "rank": 3, "tokenId": 9233, "text": "Change", "score": 6.328125 }, { "rank": 4, "tokenId": 40607, "text": "Nothing", "score": 6.05078125 }, { "rank": 5, "tokenId": 29096, "text": "Ir", "score": 5.84765625 }, { "rank": 6, "tokenId": 3284, "text": "ca", "score": 5.57421875 }, { "rank": 7, "tokenId": 8724, "text": "Dec", "score": 5.19921875 }, { "rank": 8, "tokenId": 173959, "text": "Necess", "score": 5.19921875 } ] } }, "winnerTextsByMode": { "exact": "Arrow", "f32_forward": "Arrow", "f32_reverse": "Arrow", "f32_tree64": "Arrow", "f16_forward": "Existence" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Arrow", "Existence" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 32089, 106 ], "decodedText": "Arrow", "steps": [ { "step": 0, "tokenId": 32089, "text": "Arrow" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 193168, 106 ], "decodedText": "Existence", "steps": [ { "step": 0, "tokenId": 193168, "text": "Existence" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "sci-consciousness-emergent-oneword-d", "text": "Choose one word only.\nConsciousness is an illusion, emergent, or fundamental?", "stableTopK": [ { "rank": 1, "tokenId": 205648, "text": "Fundamental", "logit": 22.893281936645508, "prob": 0.6164636015892029 }, { "rank": 2, "tokenId": 61117, "text": "Emer", "logit": 22.01129722595215, "prob": 0.2551915943622589 }, { "rank": 3, "tokenId": 14153, "text": "Ill", "logit": 20.888505935668945, "prob": 0.08303177356719971 }, { "rank": 4, "tokenId": 1018, "text": "**", "logit": 20.258882522583008, "prob": 0.04423869773745537 }, { "rank": 5, "tokenId": 164712, "text": "fundamental", "logit": 15.65600872039795, "prob": 0.00044340401655063033 }, { "rank": 6, "tokenId": 34399, "text": "Fund", "logit": 14.717718124389648, "prob": 0.00017350228154100478 }, { "rank": 7, "tokenId": 31483, "text": "Illus", "logit": 14.536506652832031, "prob": 0.00014474581985268742 }, { "rank": 8, "tokenId": 110101, "text": "illusion", "logit": 14.514432907104492, "prob": 0.00014158575504552573 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 205648, "winnerText": "Fundamental", "winnerScore": 30.108531251323797, "winnerGap": 2.007038437619343, "top": [ { "rank": 1, "tokenId": 205648, "text": "Fundamental", "score": 30.108531251323797 }, { "rank": 2, "tokenId": 61117, "text": "Emer", "score": 28.101492813704454 }, { "rank": 3, "tokenId": 14153, "text": "Ill", "score": 25.80150261954384 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 24.61247774163399 }, { "rank": 5, "tokenId": 164712, "text": "fundamental", "score": 17.36706223626057 }, { "rank": 6, "tokenId": 34399, "text": "Fund", "score": 16.105141775472845 }, { "rank": 7, "tokenId": 31483, "text": "Illus", "score": 15.867410669088471 }, { "rank": 8, "tokenId": 110101, "text": "illusion", "score": 15.838568364970143 } ] }, "f32_forward": { "winnerTokenId": 205648, "winnerText": "Fundamental", "winnerScore": 30.108539581298828, "winnerGap": 2.007038116455078, "top": [ { "rank": 1, "tokenId": 205648, "text": "Fundamental", "score": 30.108539581298828 }, { "rank": 2, "tokenId": 61117, "text": "Emer", "score": 28.10150146484375 }, { "rank": 3, "tokenId": 14153, "text": "Ill", "score": 25.80152130126953 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 24.612491607666016 }, { "rank": 5, "tokenId": 164712, "text": "fundamental", "score": 17.367069244384766 }, { "rank": 6, "tokenId": 34399, "text": "Fund", "score": 16.10517692565918 }, { "rank": 7, "tokenId": 31483, "text": "Illus", "score": 15.867405891418457 }, { "rank": 8, "tokenId": 110101, "text": "illusion", "score": 15.838578224182129 } ] }, "f32_reverse": { "winnerTokenId": 205648, "winnerText": "Fundamental", "winnerScore": 30.108537673950195, "winnerGap": 2.007049560546875, "top": [ { "rank": 1, "tokenId": 205648, "text": "Fundamental", "score": 30.108537673950195 }, { "rank": 2, "tokenId": 61117, "text": "Emer", "score": 28.10148811340332 }, { "rank": 3, "tokenId": 14153, "text": "Ill", "score": 25.801504135131836 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 24.612468719482422 }, { "rank": 5, "tokenId": 164712, "text": "fundamental", "score": 17.367048263549805 }, { "rank": 6, "tokenId": 34399, "text": "Fund", "score": 16.105146408081055 }, { "rank": 7, "tokenId": 31483, "text": "Illus", "score": 15.867409706115723 }, { "rank": 8, "tokenId": 110101, "text": "illusion", "score": 15.838562965393066 } ] }, "f32_tree64": { "winnerTokenId": 205648, "winnerText": "Fundamental", "winnerScore": 30.108530044555664, "winnerGap": 2.0070362091064453, "top": [ { "rank": 1, "tokenId": 205648, "text": "Fundamental", "score": 30.108530044555664 }, { "rank": 2, "tokenId": 61117, "text": "Emer", "score": 28.10149383544922 }, { "rank": 3, "tokenId": 14153, "text": "Ill", "score": 25.801502227783203 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 24.612476348876953 }, { "rank": 5, "tokenId": 164712, "text": "fundamental", "score": 17.367061614990234 }, { "rank": 6, "tokenId": 34399, "text": "Fund", "score": 16.105140686035156 }, { "rank": 7, "tokenId": 31483, "text": "Illus", "score": 15.867410659790039 }, { "rank": 8, "tokenId": 110101, "text": "illusion", "score": 15.838569641113281 } ] }, "f16_forward": { "winnerTokenId": 205648, "winnerText": "Fundamental", "winnerScore": 30.078125, "winnerGap": 1.9375, "top": [ { "rank": 1, "tokenId": 205648, "text": "Fundamental", "score": 30.078125 }, { "rank": 2, "tokenId": 61117, "text": "Emer", "score": 28.140625 }, { "rank": 3, "tokenId": 14153, "text": "Ill", "score": 25.5625 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 24.703125 }, { "rank": 5, "tokenId": 164712, "text": "fundamental", "score": 17.078125 }, { "rank": 6, "tokenId": 34399, "text": "Fund", "score": 16.078125 }, { "rank": 7, "tokenId": 110101, "text": "illusion", "score": 15.8984375 }, { "rank": 8, "tokenId": 31483, "text": "Illus", "score": 15.71875 } ] } }, "winnerTextsByMode": { "exact": "Fundamental", "f32_forward": "Fundamental", "f32_reverse": "Fundamental", "f32_tree64": "Fundamental", "f16_forward": "Fundamental" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Fundamental" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "sci-dark-matter-oneword-a", "text": "Answer with exactly one word only.\nDark matter exists or is a placeholder for ignorance. It is", "stableTopK": [ { "rank": 1, "tokenId": 41387, "text": "Unknown", "logit": 14.126389503479004, "prob": 0.8197782039642334 }, { "rank": 2, "tokenId": 67901, "text": "Hyp", "logit": 12.308298110961914, "prob": 0.133078932762146 }, { "rank": 3, "tokenId": 20774, "text": "unknown", "logit": 10.793667793273926, "prob": 0.02926265262067318 }, { "rank": 4, "tokenId": 229186, "text": "Mystery", "logit": 9.125236511230469, "prob": 0.005517256911844015 }, { "rank": 5, "tokenId": 193168, "text": "Existence", "logit": 9.099288940429688, "prob": 0.005375938955694437 }, { "rank": 6, "tokenId": 2805, "text": "Un", "logit": 8.312748908996582, "prob": 0.002448298269882798 }, { "rank": 7, "tokenId": 227720, "text": "hypothesis", "logit": 7.391068935394287, "prob": 0.0009740557288751006 }, { "rank": 8, "tokenId": 30340, "text": "Und", "logit": 7.07817268371582, "prob": 0.0007123520481400192 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 41387, "winnerText": "Unknown", "winnerScore": 15.336001627900956, "winnerGap": 2.2576835444947037, "top": [ { "rank": 1, "tokenId": 41387, "text": "Unknown", "score": 15.336001627900956 }, { "rank": 2, "tokenId": 67901, "text": "Hyp", "score": 13.078318083406252 }, { "rank": 3, "tokenId": 20774, "text": "unknown", "score": 11.299296810623758 }, { "rank": 4, "tokenId": 229186, "text": "Mystery", "score": 9.423398111037102 }, { "rank": 5, "tokenId": 193168, "text": "Existence", "score": 9.394816040747116 }, { "rank": 6, "tokenId": 2805, "text": "Un", "score": 8.535871750886926 }, { "rank": 7, "tokenId": 227720, "text": "hypothesis", "score": 7.5462987830854225 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": 7.214081672080425 } ] }, "f32_forward": { "winnerTokenId": 41387, "winnerText": "Unknown", "winnerScore": 15.336003303527832, "winnerGap": 2.257676124572754, "top": [ { "rank": 1, "tokenId": 41387, "text": "Unknown", "score": 15.336003303527832 }, { "rank": 2, "tokenId": 67901, "text": "Hyp", "score": 13.078327178955078 }, { "rank": 3, "tokenId": 20774, "text": "unknown", "score": 11.299295425415039 }, { "rank": 4, "tokenId": 229186, "text": "Mystery", "score": 9.423401832580566 }, { "rank": 5, "tokenId": 193168, "text": "Existence", "score": 9.394818305969238 }, { "rank": 6, "tokenId": 2805, "text": "Un", "score": 8.535863876342773 }, { "rank": 7, "tokenId": 227720, "text": "hypothesis", "score": 7.546302318572998 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": 7.2140793800354 } ] }, "f32_reverse": { "winnerTokenId": 41387, "winnerText": "Unknown", "winnerScore": 15.33599853515625, "winnerGap": 2.2576751708984375, "top": [ { "rank": 1, "tokenId": 41387, "text": "Unknown", "score": 15.33599853515625 }, { "rank": 2, "tokenId": 67901, "text": "Hyp", "score": 13.078323364257812 }, { "rank": 3, "tokenId": 20774, "text": "unknown", "score": 11.299296379089355 }, { "rank": 4, "tokenId": 229186, "text": "Mystery", "score": 9.423392295837402 }, { "rank": 5, "tokenId": 193168, "text": "Existence", "score": 9.394811630249023 }, { "rank": 6, "tokenId": 2805, "text": "Un", "score": 8.535877227783203 }, { "rank": 7, "tokenId": 227720, "text": "hypothesis", "score": 7.546298027038574 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": 7.214078426361084 } ] }, "f32_tree64": { "winnerTokenId": 41387, "winnerText": "Unknown", "winnerScore": 15.3360013961792, "winnerGap": 2.257683753967285, "top": [ { "rank": 1, "tokenId": 41387, "text": "Unknown", "score": 15.3360013961792 }, { "rank": 2, "tokenId": 67901, "text": "Hyp", "score": 13.078317642211914 }, { "rank": 3, "tokenId": 20774, "text": "unknown", "score": 11.299296379089355 }, { "rank": 4, "tokenId": 229186, "text": "Mystery", "score": 9.4233980178833 }, { "rank": 5, "tokenId": 193168, "text": "Existence", "score": 9.394816398620605 }, { "rank": 6, "tokenId": 2805, "text": "Un", "score": 8.535872459411621 }, { "rank": 7, "tokenId": 227720, "text": "hypothesis", "score": 7.546298980712891 }, { "rank": 8, "tokenId": 30340, "text": "Und", "score": 7.214081764221191 } ] }, "f16_forward": { "winnerTokenId": 41387, "winnerText": "Unknown", "winnerScore": 15.4453125, "winnerGap": 5.9375, "top": [ { "rank": 1, "tokenId": 41387, "text": "Unknown", "score": 15.4453125 }, { "rank": 2, "tokenId": 193168, "text": "Existence", "score": 9.5078125 }, { "rank": 3, "tokenId": 229186, "text": "Mystery", "score": 9.1953125 }, { "rank": 4, "tokenId": 2805, "text": "Un", "score": 8.4921875 }, { "rank": 5, "tokenId": 30340, "text": "Und", "score": 7.26953125 }, { "rank": 6, "tokenId": 20774, "text": "unknown", "score": 7.2578125 }, { "rank": 7, "tokenId": 234309, "text": "mystery", "score": 7.09765625 }, { "rank": 8, "tokenId": 1683, "text": "Ex", "score": 7.05859375 } ] } }, "winnerTextsByMode": { "exact": "Unknown", "f32_forward": "Unknown", "f32_reverse": "Unknown", "f32_tree64": "Unknown", "f16_forward": "Unknown" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Unknown" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "sci-determinism-oneword-b", "text": "Respond with one word only.\nThe universe is deterministic or random at its core. It is", "stableTopK": [ { "rank": 1, "tokenId": 187967, "text": "Deterministic", "logit": 16.210494995117188, "prob": 0.854206919670105 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "logit": 13.726781845092773, "prob": 0.07126891613006592 }, { "rank": 3, "tokenId": 30340, "text": "Und", "logit": 12.685885429382324, "prob": 0.025167761370539665 }, { "rank": 4, "tokenId": 222935, "text": "deterministic", "logit": 12.481067657470703, "prob": 0.020506585016846657 }, { "rank": 5, "tokenId": 113354, "text": "Determin", "logit": 11.825490951538086, "prob": 0.010645842179656029 }, { "rank": 6, "tokenId": 44654, "text": "Determ", "logit": 11.232230186462402, "prob": 0.00588206946849823 }, { "rank": 7, "tokenId": 2704, "text": "Ind", "logit": 10.459800720214844, "prob": 0.002716866321861744 }, { "rank": 8, "tokenId": 31900, "text": "Deb", "logit": 10.123945236206055, "prob": 0.001941816182807088 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 187967, "winnerText": "Deterministic", "winnerScore": 18.139483084892717, "winnerGap": 3.3128784179448196, "top": [ { "rank": 1, "tokenId": 187967, "text": "Deterministic", "score": 18.139483084892717 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "score": 14.826604666947897 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 13.535188415962018 }, { "rank": 4, "tokenId": 222935, "text": "deterministic", "score": 13.28664253430955 }, { "rank": 5, "tokenId": 113354, "text": "Determin", "score": 12.502289075677108 }, { "rank": 6, "tokenId": 44654, "text": "Determ", "score": 11.806195291795715 }, { "rank": 7, "tokenId": 2704, "text": "Ind", "score": 10.917524945040856 }, { "rank": 8, "tokenId": 31900, "text": "Deb", "score": 10.53686455823626 } ] }, "f32_forward": { "winnerTokenId": 187967, "winnerText": "Deterministic", "winnerScore": 18.13947296142578, "winnerGap": 3.3128604888916016, "top": [ { "rank": 1, "tokenId": 187967, "text": "Deterministic", "score": 18.13947296142578 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "score": 14.82661247253418 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 13.535165786743164 }, { "rank": 4, "tokenId": 222935, "text": "deterministic", "score": 13.286613464355469 }, { "rank": 5, "tokenId": 113354, "text": "Determin", "score": 12.502289772033691 }, { "rank": 6, "tokenId": 44654, "text": "Determ", "score": 11.806188583374023 }, { "rank": 7, "tokenId": 2704, "text": "Ind", "score": 10.917523384094238 }, { "rank": 8, "tokenId": 31900, "text": "Deb", "score": 10.536877632141113 } ] }, "f32_reverse": { "winnerTokenId": 187967, "winnerText": "Deterministic", "winnerScore": 18.139474868774414, "winnerGap": 3.312861442565918, "top": [ { "rank": 1, "tokenId": 187967, "text": "Deterministic", "score": 18.139474868774414 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "score": 14.826613426208496 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 13.53519344329834 }, { "rank": 4, "tokenId": 222935, "text": "deterministic", "score": 13.286642074584961 }, { "rank": 5, "tokenId": 113354, "text": "Determin", "score": 12.502293586730957 }, { "rank": 6, "tokenId": 44654, "text": "Determ", "score": 11.806192398071289 }, { "rank": 7, "tokenId": 2704, "text": "Ind", "score": 10.917534828186035 }, { "rank": 8, "tokenId": 31900, "text": "Deb", "score": 10.536866188049316 } ] }, "f32_tree64": { "winnerTokenId": 187967, "winnerText": "Deterministic", "winnerScore": 18.139484405517578, "winnerGap": 3.312877655029297, "top": [ { "rank": 1, "tokenId": 187967, "text": "Deterministic", "score": 18.139484405517578 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "score": 14.826606750488281 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 13.535187721252441 }, { "rank": 4, "tokenId": 222935, "text": "deterministic", "score": 13.286643028259277 }, { "rank": 5, "tokenId": 113354, "text": "Determin", "score": 12.502288818359375 }, { "rank": 6, "tokenId": 44654, "text": "Determ", "score": 11.806195259094238 }, { "rank": 7, "tokenId": 2704, "text": "Ind", "score": 10.917524337768555 }, { "rank": 8, "tokenId": 31900, "text": "Deb", "score": 10.536865234375 } ] }, "f16_forward": { "winnerTokenId": 187967, "winnerText": "Deterministic", "winnerScore": 18.25, "winnerGap": 3.390625, "top": [ { "rank": 1, "tokenId": 187967, "text": "Deterministic", "score": 18.25 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "score": 14.859375 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 13.7578125 }, { "rank": 4, "tokenId": 222935, "text": "deterministic", "score": 13.296875 }, { "rank": 5, "tokenId": 44654, "text": "Determ", "score": 11.7265625 }, { "rank": 6, "tokenId": 2704, "text": "Ind", "score": 11.0234375 }, { "rank": 7, "tokenId": 31900, "text": "Deb", "score": 10.59375 }, { "rank": 8, "tokenId": 45566, "text": "Prob", "score": 9.84375 } ] } }, "winnerTextsByMode": { "exact": "Deterministic", "f32_forward": "Deterministic", "f32_reverse": "Deterministic", "f32_tree64": "Deterministic", "f16_forward": "Deterministic" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Deterministic" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "sci-dinosaur-asteroid-oneword-c", "text": "No explanation. One word only.\nThe dinosaurs went extinct solely from an asteroid — true, false, or oversimplified?", "stableTopK": [ { "rank": 1, "tokenId": 9277, "text": "False", "logit": 14.741366386413574, "prob": 0.8496821522712708 }, { "rank": 2, "tokenId": 4339, "text": "True", "logit": 13.006388664245605, "prob": 0.14988744258880615 }, { "rank": 3, "tokenId": 236806, "text": "O", "logit": 6.100903511047363, "prob": 0.00015022809384390712 }, { "rank": 4, "tokenId": 196224, "text": "Mostly", "logit": 5.400156021118164, "prob": 0.00007454532169504091 }, { "rank": 5, "tokenId": 4330, "text": "Over", "logit": 4.5050506591796875, "prob": 0.000030456574677373283 }, { "rank": 6, "tokenId": 54025, "text": "Too", "logit": 4.262106418609619, "prob": 0.000023887556380941533 }, { "rank": 7, "tokenId": 4530, "text": "false", "logit": 3.651137590408325, "prob": 0.000012966756003152113 }, { "rank": 8, "tokenId": 3771, "text": "No", "logit": 3.2576904296875, "prob": 0.000008749019798415247 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 16.136295829412486, "winnerGap": 2.2086335794629086, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 16.136295829412486 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 13.927662249949577 }, { "rank": 3, "tokenId": 236806, "text": "O", "score": 6.187160525250678 }, { "rank": 4, "tokenId": 196224, "text": "Mostly", "score": 5.459639556149369 }, { "rank": 5, "tokenId": 4330, "text": "Over", "score": 4.539379934426627 }, { "rank": 6, "tokenId": 54025, "text": "Too", "score": 4.291132722640425 }, { "rank": 7, "tokenId": 4530, "text": "false", "score": 3.6693286995930166 }, { "rank": 8, "tokenId": 3771, "text": "No", "score": 3.270584957877972 } ] }, "f32_forward": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 16.13629913330078, "winnerGap": 2.2086172103881836, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 16.13629913330078 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 13.927681922912598 }, { "rank": 3, "tokenId": 236806, "text": "O", "score": 6.187170505523682 }, { "rank": 4, "tokenId": 196224, "text": "Mostly", "score": 5.459639072418213 }, { "rank": 5, "tokenId": 4330, "text": "Over", "score": 4.539383411407471 }, { "rank": 6, "tokenId": 54025, "text": "Too", "score": 4.291131973266602 }, { "rank": 7, "tokenId": 4530, "text": "false", "score": 3.6693267822265625 }, { "rank": 8, "tokenId": 3771, "text": "No", "score": 3.2705838680267334 } ] }, "f32_reverse": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 16.136287689208984, "winnerGap": 2.2086334228515625, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 16.136287689208984 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 13.927654266357422 }, { "rank": 3, "tokenId": 236806, "text": "O", "score": 6.187158584594727 }, { "rank": 4, "tokenId": 196224, "text": "Mostly", "score": 5.459641456604004 }, { "rank": 5, "tokenId": 4330, "text": "Over", "score": 4.5393829345703125 }, { "rank": 6, "tokenId": 54025, "text": "Too", "score": 4.291134357452393 }, { "rank": 7, "tokenId": 4530, "text": "false", "score": 3.6693296432495117 }, { "rank": 8, "tokenId": 3771, "text": "No", "score": 3.2705869674682617 } ] }, "f32_tree64": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 16.136295318603516, "winnerGap": 2.2086334228515625, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 16.136295318603516 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 13.927661895751953 }, { "rank": 3, "tokenId": 236806, "text": "O", "score": 6.187160491943359 }, { "rank": 4, "tokenId": 196224, "text": "Mostly", "score": 5.459639549255371 }, { "rank": 5, "tokenId": 4330, "text": "Over", "score": 4.53938102722168 }, { "rank": 6, "tokenId": 54025, "text": "Too", "score": 4.291133403778076 }, { "rank": 7, "tokenId": 4530, "text": "false", "score": 3.6693296432495117 }, { "rank": 8, "tokenId": 3771, "text": "No", "score": 3.270585060119629 } ] }, "f16_forward": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 16.109375, "winnerGap": 2.140625, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 16.109375 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 13.96875 }, { "rank": 3, "tokenId": 4330, "text": "Over", "score": 4.58984375 }, { "rank": 4, "tokenId": 54025, "text": "Too", "score": 4.3125 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": 3.693359375 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": 3.30078125 }, { "rank": 7, "tokenId": 25874, "text": "FALSE", "score": 2.91796875 }, { "rank": 8, "tokenId": 14713, "text": "Under", "score": 2.7734375 } ] } }, "winnerTextsByMode": { "exact": "False", "f32_forward": "False", "f32_reverse": "False", "f32_tree64": "False", "f16_forward": "False" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "False" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "sci-godel-limits-oneword-d", "text": "Choose one word only.\nMathematics is complete or fundamentally incomplete. It is", "stableTopK": [ { "rank": 1, "tokenId": 1018, "text": "**", "logit": 9.475090980529785, "prob": 0.7195884585380554 }, { "rank": 2, "tokenId": 2704, "text": "Ind", "logit": 7.095614433288574, "prob": 0.0666331946849823 }, { "rank": 3, "tokenId": 113054, "text": "Parad", "logit": 6.787866592407227, "prob": 0.04898210987448692 }, { "rank": 4, "tokenId": 902, "text": "In", "logit": 6.635890007019043, "prob": 0.04207604378461838 }, { "rank": 5, "tokenId": 7084, "text": "Open", "logit": 6.328464031219482, "prob": 0.030940083786845207 }, { "rank": 6, "tokenId": 132602, "text": "Infinite", "logit": 5.842535495758057, "prob": 0.01903204247355461 }, { "rank": 7, "tokenId": 30340, "text": "Und", "logit": 5.589378833770752, "prob": 0.014775454066693783 }, { "rank": 8, "tokenId": 115587, "text": "Undefined", "logit": 5.530378341674805, "prob": 0.013928914442658424 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 1018, "winnerText": "**", "winnerScore": 9.810460272129603, "winnerGap": 2.5779046527271845, "top": [ { "rank": 1, "tokenId": 1018, "text": "**", "score": 9.810460272129603 }, { "rank": 2, "tokenId": 2704, "text": "Ind", "score": 7.232555619402419 }, { "rank": 3, "tokenId": 113054, "text": "Parad", "score": 6.907395612211111 }, { "rank": 4, "tokenId": 902, "text": "In", "score": 6.747411146689672 }, { "rank": 5, "tokenId": 7084, "text": "Open", "score": 6.424923641252121 }, { "rank": 6, "tokenId": 132602, "text": "Infinite", "score": 5.918125412013865 }, { "rank": 7, "tokenId": 30340, "text": "Und", "score": 5.655433363967032 }, { "rank": 8, "tokenId": 115587, "text": "Undefined", "score": 5.594336251797074 } ] }, "f32_forward": { "winnerTokenId": 1018, "winnerText": "**", "winnerScore": 9.810455322265625, "winnerGap": 2.5778918266296387, "top": [ { "rank": 1, "tokenId": 1018, "text": "**", "score": 9.810455322265625 }, { "rank": 2, "tokenId": 2704, "text": "Ind", "score": 7.232563495635986 }, { "rank": 3, "tokenId": 113054, "text": "Parad", "score": 6.907392978668213 }, { "rank": 4, "tokenId": 902, "text": "In", "score": 6.747408390045166 }, { "rank": 5, "tokenId": 7084, "text": "Open", "score": 6.424918174743652 }, { "rank": 6, "tokenId": 132602, "text": "Infinite", "score": 5.918125629425049 }, { "rank": 7, "tokenId": 30340, "text": "Und", "score": 5.655438423156738 }, { "rank": 8, "tokenId": 115587, "text": "Undefined", "score": 5.594338893890381 } ] }, "f32_reverse": { "winnerTokenId": 1018, "winnerText": "**", "winnerScore": 9.810449600219727, "winnerGap": 2.5778913497924805, "top": [ { "rank": 1, "tokenId": 1018, "text": "**", "score": 9.810449600219727 }, { "rank": 2, "tokenId": 2704, "text": "Ind", "score": 7.232558250427246 }, { "rank": 3, "tokenId": 113054, "text": "Parad", "score": 6.907400608062744 }, { "rank": 4, "tokenId": 902, "text": "In", "score": 6.747413635253906 }, { "rank": 5, "tokenId": 7084, "text": "Open", "score": 6.4249186515808105 }, { "rank": 6, "tokenId": 132602, "text": "Infinite", "score": 5.9181318283081055 }, { "rank": 7, "tokenId": 30340, "text": "Und", "score": 5.655433654785156 }, { "rank": 8, "tokenId": 115587, "text": "Undefined", "score": 5.594336986541748 } ] }, "f32_tree64": { "winnerTokenId": 1018, "winnerText": "**", "winnerScore": 9.810460090637207, "winnerGap": 2.5779051780700684, "top": [ { "rank": 1, "tokenId": 1018, "text": "**", "score": 9.810460090637207 }, { "rank": 2, "tokenId": 2704, "text": "Ind", "score": 7.232554912567139 }, { "rank": 3, "tokenId": 113054, "text": "Parad", "score": 6.90739631652832 }, { "rank": 4, "tokenId": 902, "text": "In", "score": 6.747410774230957 }, { "rank": 5, "tokenId": 7084, "text": "Open", "score": 6.424923896789551 }, { "rank": 6, "tokenId": 132602, "text": "Infinite", "score": 5.918125152587891 }, { "rank": 7, "tokenId": 30340, "text": "Und", "score": 5.655433654785156 }, { "rank": 8, "tokenId": 115587, "text": "Undefined", "score": 5.59433650970459 } ] }, "f16_forward": { "winnerTokenId": 1018, "winnerText": "**", "winnerScore": 9.875, "winnerGap": 2.8984375, "top": [ { "rank": 1, "tokenId": 1018, "text": "**", "score": 9.875 }, { "rank": 2, "tokenId": 113054, "text": "Parad", "score": 6.9765625 }, { "rank": 3, "tokenId": 902, "text": "In", "score": 6.7890625 }, { "rank": 4, "tokenId": 7084, "text": "Open", "score": 6.4921875 }, { "rank": 5, "tokenId": 132602, "text": "Infinite", "score": 5.93359375 }, { "rank": 6, "tokenId": 30340, "text": "Und", "score": 5.671875 }, { "rank": 7, "tokenId": 115587, "text": "Undefined", "score": 5.625 }, { "rank": 8, "tokenId": 236847, "text": "V", "score": 4.9765625 } ] } }, "winnerTextsByMode": { "exact": "**", "f32_forward": "**", "f32_reverse": "**", "f32_tree64": "**", "f16_forward": "**" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "**" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "sci-heat-death-oneword-a", "text": "Answer with exactly one word only.\nThe universe will end in heat death — certain, likely, or unknown?", "stableTopK": [ { "rank": 1, "tokenId": 63880, "text": "Lik", "logit": 12.307263374328613, "prob": 0.9341769814491272 }, { "rank": 2, "tokenId": 65173, "text": "Certain", "logit": 9.606450080871582, "prob": 0.0627308040857315 }, { "rank": 3, "tokenId": 69626, "text": "Possible", "logit": 6.062509059906006, "prob": 0.0018128707306459546 }, { "rank": 4, "tokenId": 19609, "text": "likely", "logit": 4.739102840423584, "prob": 0.0004826350195799023 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "logit": 4.685783863067627, "prob": 0.0004575753991957754 }, { "rank": 6, "tokenId": 45566, "text": "Prob", "logit": 3.6471548080444336, "prob": 0.00016195404168684036 }, { "rank": 7, "tokenId": 2805, "text": "Un", "logit": 2.4023475646972656, "prob": 0.00004664218795369379 }, { "rank": 8, "tokenId": 64889, "text": "Almost", "logit": 1.6139225959777832, "prob": 0.000021201680283411406 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 63880, "winnerText": "Lik", "winnerScore": 13.077068764773983, "winnerGap": 3.120473073289965, "top": [ { "rank": 1, "tokenId": 63880, "text": "Lik", "score": 13.077068764773983 }, { "rank": 2, "tokenId": 65173, "text": "Certain", "score": 9.956595691484019 }, { "rank": 3, "tokenId": 69626, "text": "Possible", "score": 6.147119999305765 }, { "rank": 4, "tokenId": 19609, "text": "likely", "score": 4.779123975373452 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 4.724456238430941 }, { "rank": 6, "tokenId": 45566, "text": "Prob", "score": 3.6652838283479205 }, { "rank": 7, "tokenId": 2805, "text": "Un", "score": 2.4075009997082155 }, { "rank": 8, "tokenId": 64889, "text": "Almost", "score": 1.615480882449397 } ] }, "f32_forward": { "winnerTokenId": 63880, "winnerText": "Lik", "winnerScore": 13.077069282531738, "winnerGap": 3.120464324951172, "top": [ { "rank": 1, "tokenId": 63880, "text": "Lik", "score": 13.077069282531738 }, { "rank": 2, "tokenId": 65173, "text": "Certain", "score": 9.956604957580566 }, { "rank": 3, "tokenId": 69626, "text": "Possible", "score": 6.147104740142822 }, { "rank": 4, "tokenId": 19609, "text": "likely", "score": 4.779122829437256 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 4.724451541900635 }, { "rank": 6, "tokenId": 45566, "text": "Prob", "score": 3.6652841567993164 }, { "rank": 7, "tokenId": 2805, "text": "Un", "score": 2.407501459121704 }, { "rank": 8, "tokenId": 64889, "text": "Almost", "score": 1.6154810190200806 } ] }, "f32_reverse": { "winnerTokenId": 63880, "winnerText": "Lik", "winnerScore": 13.077069282531738, "winnerGap": 3.1204700469970703, "top": [ { "rank": 1, "tokenId": 63880, "text": "Lik", "score": 13.077069282531738 }, { "rank": 2, "tokenId": 65173, "text": "Certain", "score": 9.956599235534668 }, { "rank": 3, "tokenId": 69626, "text": "Possible", "score": 6.14711856842041 }, { "rank": 4, "tokenId": 19609, "text": "likely", "score": 4.779118537902832 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 4.724455833435059 }, { "rank": 6, "tokenId": 45566, "text": "Prob", "score": 3.6652822494506836 }, { "rank": 7, "tokenId": 2805, "text": "Un", "score": 2.4075026512145996 }, { "rank": 8, "tokenId": 64889, "text": "Almost", "score": 1.6154768466949463 } ] }, "f32_tree64": { "winnerTokenId": 63880, "winnerText": "Lik", "winnerScore": 13.077068328857422, "winnerGap": 3.1204729080200195, "top": [ { "rank": 1, "tokenId": 63880, "text": "Lik", "score": 13.077068328857422 }, { "rank": 2, "tokenId": 65173, "text": "Certain", "score": 9.956595420837402 }, { "rank": 3, "tokenId": 69626, "text": "Possible", "score": 6.147119998931885 }, { "rank": 4, "tokenId": 19609, "text": "likely", "score": 4.7791242599487305 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 4.724456787109375 }, { "rank": 6, "tokenId": 45566, "text": "Prob", "score": 3.6652839183807373 }, { "rank": 7, "tokenId": 2805, "text": "Un", "score": 2.4075002670288086 }, { "rank": 8, "tokenId": 64889, "text": "Almost", "score": 1.6154811382293701 } ] }, "f16_forward": { "winnerTokenId": 63880, "winnerText": "Lik", "winnerScore": 13.03125, "winnerGap": 2.9296875, "top": [ { "rank": 1, "tokenId": 63880, "text": "Lik", "score": 13.03125 }, { "rank": 2, "tokenId": 65173, "text": "Certain", "score": 10.1015625 }, { "rank": 3, "tokenId": 69626, "text": "Possible", "score": 6.1796875 }, { "rank": 4, "tokenId": 19609, "text": "likely", "score": 4.87890625 }, { "rank": 5, "tokenId": 41387, "text": "Unknown", "score": 4.74609375 }, { "rank": 6, "tokenId": 45566, "text": "Prob", "score": 3.6796875 }, { "rank": 7, "tokenId": 64889, "text": "Almost", "score": 1.66015625 }, { "rank": 8, "tokenId": 83172, "text": "Probably", "score": 1.53515625 } ] } }, "winnerTextsByMode": { "exact": "Lik", "f32_forward": "Lik", "f32_reverse": "Lik", "f32_tree64": "Lik", "f16_forward": "Lik" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Lik" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "sci-life-definition-oneword-b", "text": "Respond with one word only.\nA virus is alive or not alive. It is", "stableTopK": [ { "rank": 1, "tokenId": 95889, "text": "Neither", "logit": 8.563143730163574, "prob": 0.7586385607719421 }, { "rank": 2, "tokenId": 4348, "text": "Not", "logit": 7.232868671417236, "prob": 0.20058748126029968 }, { "rank": 3, "tokenId": 3771, "text": "No", "logit": 5.539609909057617, "prob": 0.0368918851017952 }, { "rank": 4, "tokenId": 14042, "text": "Non", "logit": 2.846729278564453, "prob": 0.002497052541002631 }, { "rank": 5, "tokenId": 90735, "text": "Alive", "logit": 2.1029293537139893, "prob": 0.00118685991037637 }, { "rank": 6, "tokenId": 198275, "text": "neither", "logit": -0.9110161662101746, "prob": 0.000058271947636967525 }, { "rank": 7, "tokenId": 236776, "text": "A", "logit": -2.502382278442383, "prob": 0.000011866920431202743 }, { "rank": 8, "tokenId": 9336, "text": "None", "logit": -2.848810911178589, "prob": 0.000008392396011913661 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 95889, "winnerText": "Neither", "winnerScore": 8.80777727324034, "winnerGap": 1.4296619503163797, "top": [ { "rank": 1, "tokenId": 95889, "text": "Neither", "score": 8.80777727324034 }, { "rank": 2, "tokenId": 4348, "text": "Not", "score": 7.378115322923961 }, { "rank": 3, "tokenId": 3771, "text": "No", "score": 5.603889934500687 }, { "rank": 4, "tokenId": 14042, "text": "Non", "score": 2.855318291408693 }, { "rank": 5, "tokenId": 90735, "text": "Alive", "score": 2.1063876361673834 }, { "rank": 6, "tokenId": 198275, "text": "neither", "score": -0.9112995106928565 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": -2.508210234001922 }, { "rank": 8, "tokenId": 9336, "text": "None", "score": -2.857419579833163 } ] }, "f32_forward": { "winnerTokenId": 95889, "winnerText": "Neither", "winnerScore": 8.807766914367676, "winnerGap": 1.429652214050293, "top": [ { "rank": 1, "tokenId": 95889, "text": "Neither", "score": 8.807766914367676 }, { "rank": 2, "tokenId": 4348, "text": "Not", "score": 7.378114700317383 }, { "rank": 3, "tokenId": 3771, "text": "No", "score": 5.603885173797607 }, { "rank": 4, "tokenId": 14042, "text": "Non", "score": 2.8553240299224854 }, { "rank": 5, "tokenId": 90735, "text": "Alive", "score": 2.106386423110962 }, { "rank": 6, "tokenId": 198275, "text": "neither", "score": -0.9112938642501831 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": -2.5082128047943115 }, { "rank": 8, "tokenId": 9336, "text": "None", "score": -2.857414484024048 } ] }, "f32_reverse": { "winnerTokenId": 95889, "winnerText": "Neither", "winnerScore": 8.807777404785156, "winnerGap": 1.4296684265136719, "top": [ { "rank": 1, "tokenId": 95889, "text": "Neither", "score": 8.807777404785156 }, { "rank": 2, "tokenId": 4348, "text": "Not", "score": 7.378108978271484 }, { "rank": 3, "tokenId": 3771, "text": "No", "score": 5.6038899421691895 }, { "rank": 4, "tokenId": 14042, "text": "Non", "score": 2.8553237915039062 }, { "rank": 5, "tokenId": 90735, "text": "Alive", "score": 2.1063857078552246 }, { "rank": 6, "tokenId": 198275, "text": "neither", "score": -0.9112887978553772 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": -2.508211612701416 }, { "rank": 8, "tokenId": 9336, "text": "None", "score": -2.857424736022949 } ] }, "f32_tree64": { "winnerTokenId": 95889, "winnerText": "Neither", "winnerScore": 8.80777645111084, "winnerGap": 1.4296598434448242, "top": [ { "rank": 1, "tokenId": 95889, "text": "Neither", "score": 8.80777645111084 }, { "rank": 2, "tokenId": 4348, "text": "Not", "score": 7.378116607666016 }, { "rank": 3, "tokenId": 3771, "text": "No", "score": 5.603889465332031 }, { "rank": 4, "tokenId": 14042, "text": "Non", "score": 2.855319023132324 }, { "rank": 5, "tokenId": 90735, "text": "Alive", "score": 2.1063880920410156 }, { "rank": 6, "tokenId": 198275, "text": "neither", "score": -0.9112997651100159 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": -2.5082099437713623 }, { "rank": 8, "tokenId": 9336, "text": "None", "score": -2.8574204444885254 } ] }, "f16_forward": { "winnerTokenId": 4348, "winnerText": "Not", "winnerScore": 7.34765625, "winnerGap": 1.6953125, "top": [ { "rank": 1, "tokenId": 4348, "text": "Not", "score": 7.34765625 }, { "rank": 2, "tokenId": 3771, "text": "No", "score": 5.65234375 }, { "rank": 3, "tokenId": 95889, "text": "Neither", "score": 4.83203125 }, { "rank": 4, "tokenId": 14042, "text": "Non", "score": 2.927734375 }, { "rank": 5, "tokenId": 90735, "text": "Alive", "score": 2.185546875 }, { "rank": 6, "tokenId": 198275, "text": "neither", "score": -0.8896484375 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": -2.482421875 }, { "rank": 8, "tokenId": 10784, "text": "Yes", "score": -2.783203125 } ] } }, "winnerTextsByMode": { "exact": "Neither", "f32_forward": "Neither", "f32_reverse": "Neither", "f32_tree64": "Neither", "f16_forward": "Not" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Neither", "Not" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 95889, 106 ], "decodedText": "Neither", "steps": [ { "step": 0, "tokenId": 95889, "text": "Neither" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 4348, 106 ], "decodedText": "Not", "steps": [ { "step": 0, "tokenId": 4348, "text": "Not" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "sci-multiverse-oneword-c", "text": "No explanation. One word only.\nParallel universes exist — true, false, or unfalsifiable?", "stableTopK": [ { "rank": 1, "tokenId": 2805, "text": "Un", "logit": 10.738414764404297, "prob": 0.5267385840415955 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "logit": 10.607572555541992, "prob": 0.462137371301651 }, { "rank": 3, "tokenId": 30340, "text": "Und", "logit": 6.539694786071777, "prob": 0.007908865809440613 }, { "rank": 4, "tokenId": 154745, "text": "Impossible", "logit": 4.652862548828125, "prob": 0.0011985974851995707 }, { "rank": 5, "tokenId": 236776, "text": "A", "logit": 4.054067611694336, "prob": 0.0006585973897017539 }, { "rank": 6, "tokenId": 69626, "text": "Possible", "logit": 3.2084624767303467, "prob": 0.00028273422503843904 }, { "rank": 7, "tokenId": 31900, "text": "Deb", "logit": 3.1194517612457275, "prob": 0.0002586553746368736 }, { "rank": 8, "tokenId": 4339, "text": "True", "logit": 2.819683074951172, "prob": 0.0001916609617182985 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 11.235885697488174, "winnerGap": 0.1498065071244863, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 11.235885697488174 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "score": 11.086079190363687 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 6.646342375480625 }, { "rank": 4, "tokenId": 154745, "text": "Impossible", "score": 4.690720172341125 }, { "rank": 5, "tokenId": 236776, "text": "A", "score": 4.079021883257663 }, { "rank": 6, "tokenId": 69626, "text": "Possible", "score": 3.2207806359448483 }, { "rank": 7, "tokenId": 31900, "text": "Deb", "score": 3.130771190337697 }, { "rank": 8, "tokenId": 4339, "text": "True", "score": 2.828029632650029 } ] }, "f32_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 11.2358980178833, "winnerGap": 0.14981746673583984, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 11.2358980178833 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "score": 11.086080551147461 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 6.646347999572754 }, { "rank": 4, "tokenId": 154745, "text": "Impossible", "score": 4.690717697143555 }, { "rank": 5, "tokenId": 236776, "text": "A", "score": 4.079021453857422 }, { "rank": 6, "tokenId": 69626, "text": "Possible", "score": 3.220780849456787 }, { "rank": 7, "tokenId": 31900, "text": "Deb", "score": 3.1307642459869385 }, { "rank": 8, "tokenId": 4339, "text": "True", "score": 2.8280301094055176 } ] }, "f32_reverse": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 11.23588752746582, "winnerGap": 0.14981460571289062, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 11.23588752746582 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "score": 11.08607292175293 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 6.646344184875488 }, { "rank": 4, "tokenId": 154745, "text": "Impossible", "score": 4.690720558166504 }, { "rank": 5, "tokenId": 236776, "text": "A", "score": 4.079019069671631 }, { "rank": 6, "tokenId": 69626, "text": "Possible", "score": 3.2207798957824707 }, { "rank": 7, "tokenId": 31900, "text": "Deb", "score": 3.1307694911956787 }, { "rank": 8, "tokenId": 4339, "text": "True", "score": 2.8280324935913086 } ] }, "f32_tree64": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 11.235886573791504, "winnerGap": 0.14980697631835938, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 11.235886573791504 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "score": 11.086079597473145 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 6.6463422775268555 }, { "rank": 4, "tokenId": 154745, "text": "Impossible", "score": 4.690720081329346 }, { "rank": 5, "tokenId": 236776, "text": "A", "score": 4.079021453857422 }, { "rank": 6, "tokenId": 69626, "text": "Possible", "score": 3.220781087875366 }, { "rank": 7, "tokenId": 31900, "text": "Deb", "score": 3.130770683288574 }, { "rank": 8, "tokenId": 4339, "text": "True", "score": 2.8280296325683594 } ] }, "f16_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 11.140625, "winnerGap": 4.1015625, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 11.140625 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "score": 7.0390625 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 6.61328125 }, { "rank": 4, "tokenId": 154745, "text": "Impossible", "score": 4.73828125 }, { "rank": 5, "tokenId": 236776, "text": "A", "score": 4.09375 }, { "rank": 6, "tokenId": 69626, "text": "Possible", "score": 3.236328125 }, { "rank": 7, "tokenId": 31900, "text": "Deb", "score": 3.1796875 }, { "rank": 8, "tokenId": 9798, "text": "Spec", "score": 2.61328125 } ] } }, "winnerTextsByMode": { "exact": "Un", "f32_forward": "Un", "f32_reverse": "Un", "f32_tree64": "Un", "f16_forward": "Un" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Un" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "sci-nature-nurture-oneword-d", "text": "Choose one word only.\nIntelligence is mostly nature or nurture. It is mostly", "stableTopK": [ { "rank": 1, "tokenId": 46797, "text": "Nature", "logit": 19.534587860107422, "prob": 0.9812985062599182 }, { "rank": 2, "tokenId": 236797, "text": "N", "logit": 14.715520858764648, "prob": 0.007923315279185772 }, { "rank": 3, "tokenId": 39822, "text": "nature", "logit": 14.394450187683105, "prob": 0.005747350864112377 }, { "rank": 4, "tokenId": 1018, "text": "**", "logit": 14.203896522521973, "prob": 0.004750193562358618 }, { "rank": 5, "tokenId": 236749, "text": "n", "logit": 11.183042526245117, "prob": 0.00023161733406595886 }, { "rank": 6, "tokenId": 120199, "text": "Inher", "logit": 7.789915084838867, "prob": 0.000007783134606142994 }, { "rank": 7, "tokenId": 26005, "text": "Environment", "logit": 7.650585651397705, "prob": 0.000006770871095795883 }, { "rank": 8, "tokenId": 949, "text": "her", "logit": 7.371426105499268, "prob": 0.000005121617050463101 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 46797, "winnerText": "Nature", "winnerScore": 23.318929328994614, "winnerGap": 7.216681969732605, "top": [ { "rank": 1, "tokenId": 46797, "text": "Nature", "score": 23.318929328994614 }, { "rank": 2, "tokenId": 236797, "text": "N", "score": 16.10224735926201 }, { "rank": 3, "tokenId": 39822, "text": "nature", "score": 15.682318980879131 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 15.435744764364188 }, { "rank": 5, "tokenId": 236749, "text": "n", "score": 11.74901971978926 }, { "rank": 6, "tokenId": 120199, "text": "Inher", "score": 7.972435987247806 }, { "rank": 7, "tokenId": 26005, "text": "Environment", "score": 7.823229594078612 }, { "rank": 8, "tokenId": 949, "text": "her", "score": 7.525392944235904 } ] }, "f32_forward": { "winnerTokenId": 46797, "winnerText": "Nature", "winnerScore": 23.318952560424805, "winnerGap": 7.216709136962891, "top": [ { "rank": 1, "tokenId": 46797, "text": "Nature", "score": 23.318952560424805 }, { "rank": 2, "tokenId": 236797, "text": "N", "score": 16.102243423461914 }, { "rank": 3, "tokenId": 39822, "text": "nature", "score": 15.6823148727417 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 15.435722351074219 }, { "rank": 5, "tokenId": 236749, "text": "n", "score": 11.7490234375 }, { "rank": 6, "tokenId": 120199, "text": "Inher", "score": 7.972428321838379 }, { "rank": 7, "tokenId": 26005, "text": "Environment", "score": 7.823241710662842 }, { "rank": 8, "tokenId": 949, "text": "her", "score": 7.525387763977051 } ] }, "f32_reverse": { "winnerTokenId": 46797, "winnerText": "Nature", "winnerScore": 23.318925857543945, "winnerGap": 7.216665267944336, "top": [ { "rank": 1, "tokenId": 46797, "text": "Nature", "score": 23.318925857543945 }, { "rank": 2, "tokenId": 236797, "text": "N", "score": 16.10226058959961 }, { "rank": 3, "tokenId": 39822, "text": "nature", "score": 15.682316780090332 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 15.435741424560547 }, { "rank": 5, "tokenId": 236749, "text": "n", "score": 11.749017715454102 }, { "rank": 6, "tokenId": 120199, "text": "Inher", "score": 7.972434043884277 }, { "rank": 7, "tokenId": 26005, "text": "Environment", "score": 7.823230743408203 }, { "rank": 8, "tokenId": 949, "text": "her", "score": 7.525394916534424 } ] }, "f32_tree64": { "winnerTokenId": 46797, "winnerText": "Nature", "winnerScore": 23.318927764892578, "winnerGap": 7.216678619384766, "top": [ { "rank": 1, "tokenId": 46797, "text": "Nature", "score": 23.318927764892578 }, { "rank": 2, "tokenId": 236797, "text": "N", "score": 16.102249145507812 }, { "rank": 3, "tokenId": 39822, "text": "nature", "score": 15.682317733764648 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 15.43574333190918 }, { "rank": 5, "tokenId": 236749, "text": "n", "score": 11.749019622802734 }, { "rank": 6, "tokenId": 120199, "text": "Inher", "score": 7.972436428070068 }, { "rank": 7, "tokenId": 26005, "text": "Environment", "score": 7.823230266571045 }, { "rank": 8, "tokenId": 949, "text": "her", "score": 7.525391578674316 } ] }, "f16_forward": { "winnerTokenId": 46797, "winnerText": "Nature", "winnerScore": 23.390625, "winnerGap": 7.59375, "top": [ { "rank": 1, "tokenId": 46797, "text": "Nature", "score": 23.390625 }, { "rank": 2, "tokenId": 39822, "text": "nature", "score": 15.796875 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 15.375 }, { "rank": 4, "tokenId": 236797, "text": "N", "score": 12.1171875 }, { "rank": 5, "tokenId": 120199, "text": "Inher", "score": 7.921875 }, { "rank": 6, "tokenId": 949, "text": "her", "score": 7.6171875 }, { "rank": 7, "tokenId": 50384, "text": "Shared", "score": 7.29296875 }, { "rank": 8, "tokenId": 16900, "text": "Her", "score": 7.22265625 } ] } }, "winnerTextsByMode": { "exact": "Nature", "f32_forward": "Nature", "f32_reverse": "Nature", "f32_tree64": "Nature", "f16_forward": "Nature" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Nature" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "sci-observer-effect-oneword-a", "text": "Answer with exactly one word only.\nObservation changes reality at the quantum level — literally or metaphorically?", "stableTopK": [ { "rank": 1, "tokenId": 22186, "text": "Both", "logit": 17.806686401367188, "prob": 0.4691128134727478 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "logit": 17.694931030273438, "prob": 0.41951021552085876 }, { "rank": 3, "tokenId": 55769, "text": "Literal", "logit": 16.109235763549805, "prob": 0.08591793477535248 }, { "rank": 4, "tokenId": 211284, "text": "Reality", "logit": 13.377696990966797, "prob": 0.005594891030341387 }, { "rank": 5, "tokenId": 51077, "text": "Indeed", "logit": 13.285672187805176, "prob": 0.005103002302348614 }, { "rank": 6, "tokenId": 4724, "text": "Ent", "logit": 13.003349304199219, "prob": 0.0038478176575154066 }, { "rank": 7, "tokenId": 120848, "text": "Observation", "logit": 12.520423889160156, "prob": 0.0023740106262266636 }, { "rank": 8, "tokenId": 102093, "text": "Quantum", "logit": 11.935295104980469, "prob": 0.0013224049471318722 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 22186, "winnerText": "Both", "winnerScore": 20.494169770440344, "winnerGap": 0.17196587887769255, "top": [ { "rank": 1, "tokenId": 22186, "text": "Both", "score": 20.494169770440344 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 20.32220389156265 }, { "rank": 3, "tokenId": 55769, "text": "Literal", "score": 17.99684025498871 }, { "rank": 4, "tokenId": 211284, "text": "Reality", "score": 14.38800040455024 }, { "rank": 5, "tokenId": 51077, "text": "Indeed", "score": 14.273330252421175 }, { "rank": 6, "tokenId": 4724, "text": "Ent", "score": 13.923927770760557 }, { "rank": 7, "tokenId": 120848, "text": "Observation", "score": 13.334270091621477 }, { "rank": 8, "tokenId": 102093, "text": "Quantum", "score": 12.632517422771713 } ] }, "f32_forward": { "winnerTokenId": 22186, "winnerText": "Both", "winnerScore": 20.494142532348633, "winnerGap": 0.17195510864257812, "top": [ { "rank": 1, "tokenId": 22186, "text": "Both", "score": 20.494142532348633 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 20.322187423706055 }, { "rank": 3, "tokenId": 55769, "text": "Literal", "score": 17.996850967407227 }, { "rank": 4, "tokenId": 211284, "text": "Reality", "score": 14.388008117675781 }, { "rank": 5, "tokenId": 51077, "text": "Indeed", "score": 14.273323059082031 }, { "rank": 6, "tokenId": 4724, "text": "Ent", "score": 13.923930168151855 }, { "rank": 7, "tokenId": 120848, "text": "Observation", "score": 13.334263801574707 }, { "rank": 8, "tokenId": 102093, "text": "Quantum", "score": 12.6325101852417 } ] }, "f32_reverse": { "winnerTokenId": 22186, "winnerText": "Both", "winnerScore": 20.494163513183594, "winnerGap": 0.1719646453857422, "top": [ { "rank": 1, "tokenId": 22186, "text": "Both", "score": 20.494163513183594 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 20.32219886779785 }, { "rank": 3, "tokenId": 55769, "text": "Literal", "score": 17.996835708618164 }, { "rank": 4, "tokenId": 211284, "text": "Reality", "score": 14.388001441955566 }, { "rank": 5, "tokenId": 51077, "text": "Indeed", "score": 14.273338317871094 }, { "rank": 6, "tokenId": 4724, "text": "Ent", "score": 13.923920631408691 }, { "rank": 7, "tokenId": 120848, "text": "Observation", "score": 13.334272384643555 }, { "rank": 8, "tokenId": 102093, "text": "Quantum", "score": 12.632519721984863 } ] }, "f32_tree64": { "winnerTokenId": 22186, "winnerText": "Both", "winnerScore": 20.494171142578125, "winnerGap": 0.171966552734375, "top": [ { "rank": 1, "tokenId": 22186, "text": "Both", "score": 20.494171142578125 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 20.32220458984375 }, { "rank": 3, "tokenId": 55769, "text": "Literal", "score": 17.996841430664062 }, { "rank": 4, "tokenId": 211284, "text": "Reality", "score": 14.38800048828125 }, { "rank": 5, "tokenId": 51077, "text": "Indeed", "score": 14.273329734802246 }, { "rank": 6, "tokenId": 4724, "text": "Ent", "score": 13.923929214477539 }, { "rank": 7, "tokenId": 120848, "text": "Observation", "score": 13.334270477294922 }, { "rank": 8, "tokenId": 102093, "text": "Quantum", "score": 12.632516860961914 } ] }, "f16_forward": { "winnerTokenId": 22186, "winnerText": "Both", "winnerScore": 20.453125, "winnerGap": 0.046875, "top": [ { "rank": 1, "tokenId": 22186, "text": "Both", "score": 20.453125 }, { "rank": 2, "tokenId": 10784, "text": "Yes", "score": 20.40625 }, { "rank": 3, "tokenId": 55769, "text": "Literal", "score": 18.015625 }, { "rank": 4, "tokenId": 51077, "text": "Indeed", "score": 14.4375 }, { "rank": 5, "tokenId": 4724, "text": "Ent", "score": 14.09375 }, { "rank": 6, "tokenId": 120848, "text": "Observation", "score": 13.2734375 }, { "rank": 7, "tokenId": 32592, "text": "Liter", "score": 12.40625 }, { "rank": 8, "tokenId": 16769, "text": "Super", "score": 11.625 } ] } }, "winnerTextsByMode": { "exact": "Both", "f32_forward": "Both", "f32_reverse": "Both", "f32_tree64": "Both", "f16_forward": "Both" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Both" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "sci-pluto-planet-oneword-b", "text": "Respond with one word only.\nPluto is a planet — true or false?", "stableTopK": [ { "rank": 1, "tokenId": 4339, "text": "True", "logit": 7.3246612548828125, "prob": 0.9997963309288025 }, { "rank": 2, "tokenId": 9277, "text": "False", "logit": -1.3113348484039307, "prob": 0.00017756038869265467 }, { "rank": 3, "tokenId": 3397, "text": "true", "logit": -3.365140438079834, "prob": 0.000022771395379095338 }, { "rank": 4, "tokenId": 22369, "text": "TRUE", "logit": -8.16228199005127, "prob": 1.8793927836213697e-7 }, { "rank": 5, "tokenId": 6288, "text": " True", "logit": -8.564424514770508, "prob": 1.2570983187742968e-7 }, { "rank": 6, "tokenId": 80071, "text": "Planet", "logit": -8.852518081665039, "prob": 9.42435889328408e-8 }, { "rank": 7, "tokenId": 10784, "text": "Yes", "logit": -9.269427299499512, "prob": 6.211413250412079e-8 }, { "rank": 8, "tokenId": 4530, "text": "false", "logit": -9.329448699951172, "prob": 5.84956403315573e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 7.4756456925250525, "winnerGap": 8.787815157851396, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 7.4756456925250525 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": -1.3121694653263432 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": -3.379360682608894 }, { "rank": 4, "tokenId": 22369, "text": "TRUE", "score": -8.373135700193032 }, { "rank": 5, "tokenId": 6288, "text": " True", "score": -8.809177029073588 }, { "rank": 6, "tokenId": 80071, "text": "Planet", "score": -9.123783124619958 }, { "rank": 7, "tokenId": 10784, "text": "Yes", "score": -9.582553205529504 }, { "rank": 8, "tokenId": 4530, "text": "false", "score": -9.648951335000984 } ] }, "f32_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 7.475650787353516, "winnerGap": 8.787819981575012, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 7.475650787353516 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": -1.3121691942214966 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": -3.3793606758117676 }, { "rank": 4, "tokenId": 22369, "text": "TRUE", "score": -8.373136520385742 }, { "rank": 5, "tokenId": 6288, "text": " True", "score": -8.809167861938477 }, { "rank": 6, "tokenId": 80071, "text": "Planet", "score": -9.12377643585205 }, { "rank": 7, "tokenId": 10784, "text": "Yes", "score": -9.582548141479492 }, { "rank": 8, "tokenId": 4530, "text": "false", "score": -9.648945808410645 } ] }, "f32_reverse": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 7.475644111633301, "winnerGap": 8.787814855575562, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 7.475644111633301 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": -1.3121707439422607 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": -3.379364490509033 }, { "rank": 4, "tokenId": 22369, "text": "TRUE", "score": -8.373133659362793 }, { "rank": 5, "tokenId": 6288, "text": " True", "score": -8.809174537658691 }, { "rank": 6, "tokenId": 80071, "text": "Planet", "score": -9.123773574829102 }, { "rank": 7, "tokenId": 10784, "text": "Yes", "score": -9.582551956176758 }, { "rank": 8, "tokenId": 4530, "text": "false", "score": -9.648945808410645 } ] }, "f32_tree64": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 7.475646018981934, "winnerGap": 8.78781533241272, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 7.475646018981934 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": -1.3121693134307861 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": -3.3793601989746094 }, { "rank": 4, "tokenId": 22369, "text": "TRUE", "score": -8.373135566711426 }, { "rank": 5, "tokenId": 6288, "text": " True", "score": -8.809176445007324 }, { "rank": 6, "tokenId": 80071, "text": "Planet", "score": -9.12378215789795 }, { "rank": 7, "tokenId": 10784, "text": "Yes", "score": -9.58255386352539 }, { "rank": 8, "tokenId": 4530, "text": "false", "score": -9.648950576782227 } ] }, "f16_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 3.46875, "winnerGap": 4.7763671875, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 3.46875 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": -1.3076171875 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": -3.400390625 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": -6.65234375 }, { "rank": 5, "tokenId": 2709, "text": "Ob", "score": -7.1484375 }, { "rank": 6, "tokenId": 236774, "text": "T", "score": -7.3203125 }, { "rank": 7, "tokenId": 19204, "text": "Variable", "score": -7.92578125 }, { "rank": 8, "tokenId": 31717, "text": "Still", "score": -8.109375 } ] } }, "winnerTextsByMode": { "exact": "True", "f32_forward": "True", "f32_reverse": "True", "f32_tree64": "True", "f16_forward": "True" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "True" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "sci-simulation-oneword-c", "text": "No explanation. One word only.\nWe live in a simulation — true, false, or untestable?", "stableTopK": [ { "rank": 1, "tokenId": 23450, "text": "Unt", "logit": 9.118850708007812, "prob": 0.41753461956977844 }, { "rank": 2, "tokenId": 2805, "text": "Un", "logit": 8.361808776855469, "prob": 0.19584539532661438 }, { "rank": 3, "tokenId": 30340, "text": "Und", "logit": 8.29711627960205, "prob": 0.1835767924785614 }, { "rank": 4, "tokenId": 41387, "text": "Unknown", "logit": 8.296228408813477, "prob": 0.18341386318206787 }, { "rank": 5, "tokenId": 154745, "text": "Impossible", "logit": 5.6473565101623535, "prob": 0.012973038479685783 }, { "rank": 6, "tokenId": 236776, "text": "A", "logit": 3.8487823009490967, "prob": 0.0021474885288625956 }, { "rank": 7, "tokenId": 11921, "text": "Met", "logit": 3.131517171859741, "prob": 0.0010481575736775994 }, { "rank": 8, "tokenId": 113054, "text": "Parad", "logit": 2.5244710445404053, "prob": 0.0005712021375074983 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 23450, "winnerText": "Unt", "winnerScore": 9.416361537050168, "winnerGap": 0.8273236900991527, "top": [ { "rank": 1, "tokenId": 23450, "text": "Unt", "score": 9.416361537050168 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 8.589037846951015 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 8.518944096117622 }, { "rank": 4, "tokenId": 41387, "text": "Unknown", "score": 8.51798148547538 }, { "rank": 5, "tokenId": 154745, "text": "Impossible", "score": 5.715518938982761 }, { "rank": 6, "tokenId": 236776, "text": "A", "score": 3.870104618658246 }, { "rank": 7, "tokenId": 11921, "text": "Met", "score": 3.142967194654304 }, { "rank": 8, "tokenId": 113054, "text": "Parad", "score": 2.5304565978779863 } ] }, "f32_forward": { "winnerTokenId": 23450, "winnerText": "Unt", "winnerScore": 9.416365623474121, "winnerGap": 0.8273286819458008, "top": [ { "rank": 1, "tokenId": 23450, "text": "Unt", "score": 9.416365623474121 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 8.58903694152832 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 8.51893424987793 }, { "rank": 4, "tokenId": 41387, "text": "Unknown", "score": 8.517986297607422 }, { "rank": 5, "tokenId": 154745, "text": "Impossible", "score": 5.715515613555908 }, { "rank": 6, "tokenId": 236776, "text": "A", "score": 3.870105028152466 }, { "rank": 7, "tokenId": 11921, "text": "Met", "score": 3.1429667472839355 }, { "rank": 8, "tokenId": 113054, "text": "Parad", "score": 2.530461549758911 } ] }, "f32_reverse": { "winnerTokenId": 23450, "winnerText": "Unt", "winnerScore": 9.416360855102539, "winnerGap": 0.8273143768310547, "top": [ { "rank": 1, "tokenId": 23450, "text": "Unt", "score": 9.416360855102539 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 8.589046478271484 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 8.518939018249512 }, { "rank": 4, "tokenId": 41387, "text": "Unknown", "score": 8.517991065979004 }, { "rank": 5, "tokenId": 154745, "text": "Impossible", "score": 5.715521812438965 }, { "rank": 6, "tokenId": 236776, "text": "A", "score": 3.870103597640991 }, { "rank": 7, "tokenId": 11921, "text": "Met", "score": 3.142970323562622 }, { "rank": 8, "tokenId": 113054, "text": "Parad", "score": 2.530453681945801 } ] }, "f32_tree64": { "winnerTokenId": 23450, "winnerText": "Unt", "winnerScore": 9.416360855102539, "winnerGap": 0.8273229598999023, "top": [ { "rank": 1, "tokenId": 23450, "text": "Unt", "score": 9.416360855102539 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 8.589037895202637 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 8.518942832946777 }, { "rank": 4, "tokenId": 41387, "text": "Unknown", "score": 8.517980575561523 }, { "rank": 5, "tokenId": 154745, "text": "Impossible", "score": 5.715518474578857 }, { "rank": 6, "tokenId": 236776, "text": "A", "score": 3.8701062202453613 }, { "rank": 7, "tokenId": 11921, "text": "Met", "score": 3.142967700958252 }, { "rank": 8, "tokenId": 113054, "text": "Parad", "score": 2.5304574966430664 } ] }, "f16_forward": { "winnerTokenId": 23450, "winnerText": "Unt", "winnerScore": 9.578125, "winnerGap": 1.0078125, "top": [ { "rank": 1, "tokenId": 23450, "text": "Unt", "score": 9.578125 }, { "rank": 2, "tokenId": 41387, "text": "Unknown", "score": 8.5703125 }, { "rank": 3, "tokenId": 154745, "text": "Impossible", "score": 5.76953125 }, { "rank": 4, "tokenId": 2805, "text": "Un", "score": 4.625 }, { "rank": 5, "tokenId": 30340, "text": "Und", "score": 4.55078125 }, { "rank": 6, "tokenId": 236776, "text": "A", "score": 3.859375 }, { "rank": 7, "tokenId": 11921, "text": "Met", "score": 3.15234375 }, { "rank": 8, "tokenId": 154915, "text": "Philos", "score": 2.544921875 } ] } }, "winnerTextsByMode": { "exact": "Unt", "f32_forward": "Unt", "f32_reverse": "Unt", "f32_tree64": "Unt", "f16_forward": "Unt" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Unt" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "sci-string-theory-oneword-d", "text": "Choose one word only.\nString theory is science or philosophy. It is", "stableTopK": [ { "rank": 1, "tokenId": 45946, "text": "Science", "logit": 12.13292407989502, "prob": 0.8859723210334778 }, { "rank": 2, "tokenId": 1018, "text": "**", "logit": 9.498384475708008, "prob": 0.0635702833533287 }, { "rank": 3, "tokenId": 64446, "text": "Physics", "logit": 8.93815803527832, "prob": 0.036303699016571045 }, { "rank": 4, "tokenId": 33856, "text": "science", "logit": 7.809326171875, "prob": 0.011741009540855885 }, { "rank": 5, "tokenId": 223328, "text": "Philosophy", "logit": 5.368057727813721, "prob": 0.0010220591211691499 }, { "rank": 6, "tokenId": 71827, "text": "physics", "logit": 4.993210315704346, "prob": 0.00070255744503811 }, { "rank": 7, "tokenId": 22186, "text": "Both", "logit": 3.8816354274749756, "prob": 0.00023116976080927998 }, { "rank": 8, "tokenId": 133539, "text": "Mathematics", "logit": 2.7081961631774902, "prob": 0.00007150111196096987 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 45946, "winnerText": "Science", "winnerScore": 12.868045713951066, "winnerGap": 3.031705846837651, "top": [ { "rank": 1, "tokenId": 45946, "text": "Science", "score": 12.868045713951066 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 9.836339867113415 }, { "rank": 3, "tokenId": 64446, "text": "Physics", "score": 9.217676324590059 }, { "rank": 4, "tokenId": 33856, "text": "science", "score": 7.993257750413143 }, { "rank": 5, "tokenId": 223328, "text": "Philosophy", "score": 5.426472309167937 }, { "rank": 6, "tokenId": 71827, "text": "physics", "score": 5.040094218234508 }, { "rank": 7, "tokenId": 22186, "text": "Both", "score": 3.9035133322578854 }, { "rank": 8, "tokenId": 133539, "text": "Mathematics", "score": 2.7155905376444593 } ] }, "f32_forward": { "winnerTokenId": 45946, "winnerText": "Science", "winnerScore": 12.868037223815918, "winnerGap": 3.031710624694824, "top": [ { "rank": 1, "tokenId": 45946, "text": "Science", "score": 12.868037223815918 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 9.836326599121094 }, { "rank": 3, "tokenId": 64446, "text": "Physics", "score": 9.217667579650879 }, { "rank": 4, "tokenId": 33856, "text": "science", "score": 7.993260860443115 }, { "rank": 5, "tokenId": 223328, "text": "Philosophy", "score": 5.4264750480651855 }, { "rank": 6, "tokenId": 71827, "text": "physics", "score": 5.040101051330566 }, { "rank": 7, "tokenId": 22186, "text": "Both", "score": 3.903512954711914 }, { "rank": 8, "tokenId": 133539, "text": "Mathematics", "score": 2.715597152709961 } ] }, "f32_reverse": { "winnerTokenId": 45946, "winnerText": "Science", "winnerScore": 12.868046760559082, "winnerGap": 3.031707763671875, "top": [ { "rank": 1, "tokenId": 45946, "text": "Science", "score": 12.868046760559082 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 9.836338996887207 }, { "rank": 3, "tokenId": 64446, "text": "Physics", "score": 9.217674255371094 }, { "rank": 4, "tokenId": 33856, "text": "science", "score": 7.993263244628906 }, { "rank": 5, "tokenId": 223328, "text": "Philosophy", "score": 5.4264678955078125 }, { "rank": 6, "tokenId": 71827, "text": "physics", "score": 5.040097236633301 }, { "rank": 7, "tokenId": 22186, "text": "Both", "score": 3.9035189151763916 }, { "rank": 8, "tokenId": 133539, "text": "Mathematics", "score": 2.715592384338379 } ] }, "f32_tree64": { "winnerTokenId": 45946, "winnerText": "Science", "winnerScore": 12.868045806884766, "winnerGap": 3.031705856323242, "top": [ { "rank": 1, "tokenId": 45946, "text": "Science", "score": 12.868045806884766 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 9.836339950561523 }, { "rank": 3, "tokenId": 64446, "text": "Physics", "score": 9.21767520904541 }, { "rank": 4, "tokenId": 33856, "text": "science", "score": 7.993257522583008 }, { "rank": 5, "tokenId": 223328, "text": "Philosophy", "score": 5.426472187042236 }, { "rank": 6, "tokenId": 71827, "text": "physics", "score": 5.040094375610352 }, { "rank": 7, "tokenId": 22186, "text": "Both", "score": 3.903513193130493 }, { "rank": 8, "tokenId": 133539, "text": "Mathematics", "score": 2.715590476989746 } ] }, "f16_forward": { "winnerTokenId": 45946, "winnerText": "Science", "winnerScore": 12.8046875, "winnerGap": 3.4609375, "top": [ { "rank": 1, "tokenId": 45946, "text": "Science", "score": 12.8046875 }, { "rank": 2, "tokenId": 64446, "text": "Physics", "score": 9.34375 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 8.7578125 }, { "rank": 4, "tokenId": 33856, "text": "science", "score": 7.984375 }, { "rank": 5, "tokenId": 223328, "text": "Philosophy", "score": 5.43359375 }, { "rank": 6, "tokenId": 71827, "text": "physics", "score": 5.015625 }, { "rank": 7, "tokenId": 133539, "text": "Mathematics", "score": 2.76171875 }, { "rank": 8, "tokenId": 109090, "text": "Theory", "score": 2.291015625 } ] } }, "winnerTextsByMode": { "exact": "Science", "f32_forward": "Science", "f32_reverse": "Science", "f32_tree64": "Science", "f16_forward": "Science" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Science" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "sci-universe-finite-oneword-a", "text": "Answer with exactly one word only.\nThe universe is finite or infinite. It is", "stableTopK": [ { "rank": 1, "tokenId": 41387, "text": "Unknown", "logit": 13.549544334411621, "prob": 0.9949126243591309 }, { "rank": 2, "tokenId": 20774, "text": "unknown", "logit": 7.158277988433838, "prob": 0.001667605247348547 }, { "rank": 3, "tokenId": 30340, "text": "Und", "logit": 6.787042617797852, "prob": 0.0011504500871524215 }, { "rank": 4, "tokenId": 132602, "text": "Infinite", "logit": 6.490559101104736, "prob": 0.0008552765939384699 }, { "rank": 5, "tokenId": 2704, "text": "Ind", "logit": 6.192474842071533, "prob": 0.0006348195020109415 }, { "rank": 6, "tokenId": 2805, "text": "Un", "logit": 5.679853916168213, "prob": 0.000380208482965827 }, { "rank": 7, "tokenId": 115587, "text": "Undefined", "logit": 5.374682903289795, "prob": 0.00028021264006383717 }, { "rank": 8, "tokenId": 7084, "text": "Open", "logit": 3.328131675720215, "prob": 0.00003619777271524072 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 41387, "winnerText": "Unknown", "winnerScore": 14.603187745763108, "winnerGap": 7.304221583324008, "top": [ { "rank": 1, "tokenId": 41387, "text": "Unknown", "score": 14.603187745763108 }, { "rank": 2, "tokenId": 20774, "text": "unknown", "score": 7.2989661624391005 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 6.906523745556855 }, { "rank": 4, "tokenId": 132602, "text": "Infinite", "score": 6.594774008055316 }, { "rank": 5, "tokenId": 2704, "text": "Ind", "score": 6.282744291473211 }, { "rank": 6, "tokenId": 2805, "text": "Un", "score": 5.749216320795227 }, { "rank": 7, "tokenId": 115587, "text": "Undefined", "score": 5.433316846059899 }, { "rank": 8, "tokenId": 7084, "text": "Open", "score": 3.3418843879379203 } ] }, "f32_forward": { "winnerTokenId": 41387, "winnerText": "Unknown", "winnerScore": 14.603194236755371, "winnerGap": 7.304228782653809, "top": [ { "rank": 1, "tokenId": 41387, "text": "Unknown", "score": 14.603194236755371 }, { "rank": 2, "tokenId": 20774, "text": "unknown", "score": 7.2989654541015625 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 6.906511306762695 }, { "rank": 4, "tokenId": 132602, "text": "Infinite", "score": 6.594768047332764 }, { "rank": 5, "tokenId": 2704, "text": "Ind", "score": 6.282754898071289 }, { "rank": 6, "tokenId": 2805, "text": "Un", "score": 5.749218463897705 }, { "rank": 7, "tokenId": 115587, "text": "Undefined", "score": 5.433320999145508 }, { "rank": 8, "tokenId": 7084, "text": "Open", "score": 3.3418853282928467 } ] }, "f32_reverse": { "winnerTokenId": 41387, "winnerText": "Unknown", "winnerScore": 14.603182792663574, "winnerGap": 7.304217338562012, "top": [ { "rank": 1, "tokenId": 41387, "text": "Unknown", "score": 14.603182792663574 }, { "rank": 2, "tokenId": 20774, "text": "unknown", "score": 7.2989654541015625 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 6.906522750854492 }, { "rank": 4, "tokenId": 132602, "text": "Infinite", "score": 6.594775199890137 }, { "rank": 5, "tokenId": 2704, "text": "Ind", "score": 6.28274393081665 }, { "rank": 6, "tokenId": 2805, "text": "Un", "score": 5.749215602874756 }, { "rank": 7, "tokenId": 115587, "text": "Undefined", "score": 5.433318614959717 }, { "rank": 8, "tokenId": 7084, "text": "Open", "score": 3.3418869972229004 } ] }, "f32_tree64": { "winnerTokenId": 41387, "winnerText": "Unknown", "winnerScore": 14.603189468383789, "winnerGap": 7.30422306060791, "top": [ { "rank": 1, "tokenId": 41387, "text": "Unknown", "score": 14.603189468383789 }, { "rank": 2, "tokenId": 20774, "text": "unknown", "score": 7.298966407775879 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 6.906523704528809 }, { "rank": 4, "tokenId": 132602, "text": "Infinite", "score": 6.59477424621582 }, { "rank": 5, "tokenId": 2704, "text": "Ind", "score": 6.282743453979492 }, { "rank": 6, "tokenId": 2805, "text": "Un", "score": 5.749216079711914 }, { "rank": 7, "tokenId": 115587, "text": "Undefined", "score": 5.433317184448242 }, { "rank": 8, "tokenId": 7084, "text": "Open", "score": 3.3418846130371094 } ] }, "f16_forward": { "winnerTokenId": 41387, "winnerText": "Unknown", "winnerScore": 14.5234375, "winnerGap": 7.1796875, "top": [ { "rank": 1, "tokenId": 41387, "text": "Unknown", "score": 14.5234375 }, { "rank": 2, "tokenId": 20774, "text": "unknown", "score": 7.34375 }, { "rank": 3, "tokenId": 30340, "text": "Und", "score": 6.9375 }, { "rank": 4, "tokenId": 132602, "text": "Infinite", "score": 6.65234375 }, { "rank": 5, "tokenId": 2704, "text": "Ind", "score": 6.3828125 }, { "rank": 6, "tokenId": 115587, "text": "Undefined", "score": 5.43359375 }, { "rank": 7, "tokenId": 7084, "text": "Open", "score": 3.3515625 }, { "rank": 8, "tokenId": 1136, "text": "und", "score": 1.8564453125 } ] } }, "winnerTextsByMode": { "exact": "Unknown", "f32_forward": "Unknown", "f32_reverse": "Unknown", "f32_tree64": "Unknown", "f16_forward": "Unknown" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Unknown" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "science-faith-both-oneword-b", "text": "Respond with one word only.\nScience, faith, or both: trusting that nature is intelligible is", "stableTopK": [ { "rank": 1, "tokenId": 135864, "text": "Rational", "logit": 1.2271703481674194, "prob": 0.5947375297546387 }, { "rank": 2, "tokenId": 236776, "text": "A", "logit": -0.44179272651672363, "prob": 0.11207374930381775 }, { "rank": 3, "tokenId": 47875, "text": "Reason", "logit": -0.7335044741630554, "prob": 0.08371727913618088 }, { "rank": 4, "tokenId": 39905, "text": "Natural", "logit": -0.7753747701644897, "prob": 0.08028437942266464 }, { "rank": 5, "tokenId": 46561, "text": "Optim", "logit": -1.0151447057724, "prob": 0.063168466091156 }, { "rank": 6, "tokenId": 37965, "text": "Emp", "logit": -2.0096325874328613, "prob": 0.02336682379245758 }, { "rank": 7, "tokenId": 27388, "text": "reason", "logit": -2.85971999168396, "prob": 0.009986457414925098 }, { "rank": 8, "tokenId": 223328, "text": "Philosophy", "logit": -3.198683261871338, "prob": 0.007115436717867851 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 135864, "winnerText": "Rational", "winnerScore": 1.2278549858371992, "winnerGap": 1.669680410017552, "top": [ { "rank": 1, "tokenId": 135864, "text": "Rational", "score": 1.2278549858371992 }, { "rank": 2, "tokenId": 236776, "text": "A", "score": -0.44182542418035275 }, { "rank": 3, "tokenId": 47875, "text": "Reason", "score": -0.7336557261554182 }, { "rank": 4, "tokenId": 39905, "text": "Natural", "score": -0.7755476867812092 }, { "rank": 5, "tokenId": 46561, "text": "Optim", "score": -1.015530921334971 }, { "rank": 6, "tokenId": 37965, "text": "Emp", "score": -2.0126480192994904 }, { "rank": 7, "tokenId": 27388, "text": "reason", "score": -2.8684288588614137 }, { "rank": 8, "tokenId": 223328, "text": "Philosophy", "score": -3.210883251443649 } ] }, "f32_forward": { "winnerTokenId": 135864, "winnerText": "Rational", "winnerScore": 1.2278566360473633, "winnerGap": 1.6696859002113342, "top": [ { "rank": 1, "tokenId": 135864, "text": "Rational", "score": 1.2278566360473633 }, { "rank": 2, "tokenId": 236776, "text": "A", "score": -0.44182926416397095 }, { "rank": 3, "tokenId": 47875, "text": "Reason", "score": -0.7336484789848328 }, { "rank": 4, "tokenId": 39905, "text": "Natural", "score": -0.7755448222160339 }, { "rank": 5, "tokenId": 46561, "text": "Optim", "score": -1.015529751777649 }, { "rank": 6, "tokenId": 37965, "text": "Emp", "score": -2.0126399993896484 }, { "rank": 7, "tokenId": 27388, "text": "reason", "score": -2.8684234619140625 }, { "rank": 8, "tokenId": 223328, "text": "Philosophy", "score": -3.210881233215332 } ] }, "f32_reverse": { "winnerTokenId": 135864, "winnerText": "Rational", "winnerScore": 1.2278565168380737, "winnerGap": 1.6696856021881104, "top": [ { "rank": 1, "tokenId": 135864, "text": "Rational", "score": 1.2278565168380737 }, { "rank": 2, "tokenId": 236776, "text": "A", "score": -0.4418290853500366 }, { "rank": 3, "tokenId": 47875, "text": "Reason", "score": -0.733651340007782 }, { "rank": 4, "tokenId": 39905, "text": "Natural", "score": -0.7755430936813354 }, { "rank": 5, "tokenId": 46561, "text": "Optim", "score": -1.0155305862426758 }, { "rank": 6, "tokenId": 37965, "text": "Emp", "score": -2.012639284133911 }, { "rank": 7, "tokenId": 27388, "text": "reason", "score": -2.8684334754943848 }, { "rank": 8, "tokenId": 223328, "text": "Philosophy", "score": -3.2108843326568604 } ] }, "f32_tree64": { "winnerTokenId": 135864, "winnerText": "Rational", "winnerScore": 1.2278549671173096, "winnerGap": 1.6696789264678955, "top": [ { "rank": 1, "tokenId": 135864, "text": "Rational", "score": 1.2278549671173096 }, { "rank": 2, "tokenId": 236776, "text": "A", "score": -0.44182395935058594 }, { "rank": 3, "tokenId": 47875, "text": "Reason", "score": -0.7336561679840088 }, { "rank": 4, "tokenId": 39905, "text": "Natural", "score": -0.7755472660064697 }, { "rank": 5, "tokenId": 46561, "text": "Optim", "score": -1.0155291557312012 }, { "rank": 6, "tokenId": 37965, "text": "Emp", "score": -2.0126471519470215 }, { "rank": 7, "tokenId": 27388, "text": "reason", "score": -2.8684282302856445 }, { "rank": 8, "tokenId": 223328, "text": "Philosophy", "score": -3.210883378982544 } ] }, "f16_forward": { "winnerTokenId": 135864, "winnerText": "Rational", "winnerScore": 1.220703125, "winnerGap": 1.55126953125, "top": [ { "rank": 1, "tokenId": 135864, "text": "Rational", "score": 1.220703125 }, { "rank": 2, "tokenId": 236776, "text": "A", "score": -0.33056640625 }, { "rank": 3, "tokenId": 47875, "text": "Reason", "score": -0.74853515625 }, { "rank": 4, "tokenId": 39905, "text": "Natural", "score": -0.79736328125 }, { "rank": 5, "tokenId": 46561, "text": "Optim", "score": -1.0078125 }, { "rank": 6, "tokenId": 37965, "text": "Emp", "score": -1.984375 }, { "rank": 7, "tokenId": 27388, "text": "reason", "score": -2.919921875 }, { "rank": 8, "tokenId": 11099, "text": "Found", "score": -3.16015625 } ] } }, "winnerTextsByMode": { "exact": "Rational", "f32_forward": "Rational", "f32_reverse": "Rational", "f32_tree64": "Rational", "f16_forward": "Rational" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Rational" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "science-model-open-answer-oneword-c", "text": "No explanation. One word only.\nAnswer with exactly one word: true or false. Question: A model can guide action even when several interpretations remain open. Answer:", "stableTopK": [ { "rank": 1, "tokenId": 4339, "text": "True", "logit": 13.751063346862793, "prob": 0.7976096272468567 }, { "rank": 2, "tokenId": 3397, "text": "true", "logit": 12.363572120666504, "prob": 0.19916389882564545 }, { "rank": 3, "tokenId": 9277, "text": "False", "logit": 8.034296035766602, "prob": 0.002624399261549115 }, { "rank": 4, "tokenId": 4530, "text": "false", "logit": 6.549438953399658, "prob": 0.0005945175653323531 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "logit": -0.6720000505447388, "prob": 4.3444376274237584e-7 }, { "rank": 6, "tokenId": 3771, "text": "No", "logit": -0.9484466910362244, "prob": 3.295143358172936e-7 }, { "rank": 7, "tokenId": 236776, "text": "A", "logit": -1.2985326051712036, "prob": 2.3218487399390142e-7 }, { "rank": 8, "tokenId": 1847, "text": " true", "logit": -1.3429594039916992, "prob": 2.2209542294149287e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 14.857326523766993, "winnerGap": 1.712494837187755, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 14.857326523766993 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 13.144831686579238 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": 8.2350879328379 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": 6.656566920432162 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": -0.672112391874947 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": -0.9487604084839667 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": -1.2993441540140225 }, { "rank": 8, "tokenId": 1847, "text": " true", "score": -1.3438595494039234 } ] }, "f32_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 14.857324600219727, "winnerGap": 1.7124958038330078, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 14.857324600219727 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 13.144828796386719 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": 8.235088348388672 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": 6.656565189361572 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": -0.6721141934394836 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": -0.9487643241882324 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": -1.2993428707122803 }, { "rank": 8, "tokenId": 1847, "text": " true", "score": -1.3438552618026733 } ] }, "f32_reverse": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 14.857314109802246, "winnerGap": 1.71246337890625, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 14.857314109802246 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 13.144850730895996 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": 8.235087394714355 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": 6.656569480895996 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": -0.6721112132072449 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": -0.9487659335136414 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": -1.2993419170379639 }, { "rank": 8, "tokenId": 1847, "text": " true", "score": -1.3438608646392822 } ] }, "f32_tree64": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 14.857325553894043, "winnerGap": 1.7124948501586914, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 14.857325553894043 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 13.144830703735352 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": 8.235088348388672 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": 6.656566143035889 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": -0.672112762928009 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": -0.9487603902816772 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": -1.2993444204330444 }, { "rank": 8, "tokenId": 1847, "text": " true", "score": -1.3438596725463867 } ] }, "f16_forward": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 13.125, "winnerGap": 4.9296875, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 13.125 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 8.1953125 }, { "rank": 3, "tokenId": 4339, "text": "True", "score": 6.80859375 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": 6.63671875 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": -0.9375 }, { "rank": 6, "tokenId": 236776, "text": "A", "score": -1.28515625 }, { "rank": 7, "tokenId": 1847, "text": " true", "score": -1.357421875 }, { "rank": 8, "tokenId": 10784, "text": "Yes", "score": -1.388671875 } ] } }, "winnerTextsByMode": { "exact": "True", "f32_forward": "True", "f32_reverse": "True", "f32_tree64": "True", "f16_forward": "true" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "True", "true" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 4339, 106 ], "decodedText": "True", "steps": [ { "step": 0, "tokenId": 4339, "text": "True" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 3397, 106 ], "decodedText": "true", "steps": [ { "step": 0, "tokenId": 3397, "text": "true" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "seatbelt-safe-unsafe-oneword-d", "text": "Choose one word only.\nDriving without a seatbelt is safe or unsafe. It is", "stableTopK": [ { "rank": 1, "tokenId": 2805, "text": "Un", "logit": 18.80887222290039, "prob": 0.9754263758659363 }, { "rank": 2, "tokenId": 1018, "text": "**", "logit": 15.0895414352417, "prob": 0.0236542746424675 }, { "rank": 3, "tokenId": 30908, "text": "unsafe", "logit": 11.840636253356934, "prob": 0.0009181805071420968 }, { "rank": 4, "tokenId": 236536, "text": "Dangerous", "logit": 3.6127660274505615, "prob": 2.4525024855392985e-7 }, { "rank": 5, "tokenId": 5213, "text": " **", "logit": 3.6000261306762695, "prob": 2.4214560312429967e-7 }, { "rank": 6, "tokenId": 38699, "text": "Una", "logit": 3.4175775051116943, "prob": 2.0176234727387055e-7 }, { "rank": 7, "tokenId": 198, "text": "", "logit": 3.185805559158325, "prob": 1.6002316272079042e-7 }, { "rank": 8, "tokenId": 236836, "text": "U", "logit": 2.28265118598938, "prob": 6.485565506864077e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 22.091865786762398, "winnerGap": 5.493052345209406, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 22.091865786762398 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 16.59881344155299 }, { "rank": 3, "tokenId": 30908, "text": "unsafe", "score": 12.520225701707062 }, { "rank": 4, "tokenId": 236536, "text": "Dangerous", "score": 3.6303844795935163 }, { "rank": 5, "tokenId": 5213, "text": " **", "score": 3.617458906950091 }, { "rank": 6, "tokenId": 38699, "text": "Una", "score": 3.4324799916816104 }, { "rank": 7, "tokenId": 198, "text": "", "score": 3.1978628895704446 }, { "rank": 8, "tokenId": 236836, "text": "U", "score": 2.287073755189537 } ] }, "f32_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 22.09189796447754, "winnerGap": 5.493091583251953, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 22.09189796447754 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 16.598806381225586 }, { "rank": 3, "tokenId": 30908, "text": "unsafe", "score": 12.520241737365723 }, { "rank": 4, "tokenId": 236536, "text": "Dangerous", "score": 3.63038969039917 }, { "rank": 5, "tokenId": 5213, "text": " **", "score": 3.6174585819244385 }, { "rank": 6, "tokenId": 38699, "text": "Una", "score": 3.4324798583984375 }, { "rank": 7, "tokenId": 198, "text": "", "score": 3.197866201400757 }, { "rank": 8, "tokenId": 236836, "text": "U", "score": 2.287073850631714 } ] }, "f32_reverse": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 22.09184455871582, "winnerGap": 5.493021011352539, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 22.09184455871582 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 16.59882354736328 }, { "rank": 3, "tokenId": 30908, "text": "unsafe", "score": 12.52022647857666 }, { "rank": 4, "tokenId": 236536, "text": "Dangerous", "score": 3.6303839683532715 }, { "rank": 5, "tokenId": 5213, "text": " **", "score": 3.6174514293670654 }, { "rank": 6, "tokenId": 38699, "text": "Una", "score": 3.4324755668640137 }, { "rank": 7, "tokenId": 198, "text": "", "score": 3.1978650093078613 }, { "rank": 8, "tokenId": 236836, "text": "U", "score": 2.2870728969573975 } ] }, "f32_tree64": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 22.09186553955078, "winnerGap": 5.493051528930664, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 22.09186553955078 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 16.598814010620117 }, { "rank": 3, "tokenId": 30908, "text": "unsafe", "score": 12.520224571228027 }, { "rank": 4, "tokenId": 236536, "text": "Dangerous", "score": 3.630385160446167 }, { "rank": 5, "tokenId": 5213, "text": " **", "score": 3.617459297180176 }, { "rank": 6, "tokenId": 38699, "text": "Una", "score": 3.4324793815612793 }, { "rank": 7, "tokenId": 198, "text": "", "score": 3.197862148284912 }, { "rank": 8, "tokenId": 236836, "text": "U", "score": 2.287074565887451 } ] }, "f16_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 21.984375, "winnerGap": 5.328125, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 21.984375 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 16.65625 }, { "rank": 3, "tokenId": 30908, "text": "unsafe", "score": 12.59375 }, { "rank": 4, "tokenId": 5213, "text": " **", "score": 3.61328125 }, { "rank": 5, "tokenId": 38699, "text": "Una", "score": 3.517578125 }, { "rank": 6, "tokenId": 236536, "text": "Dangerous", "score": 3.447265625 }, { "rank": 7, "tokenId": 198, "text": "", "score": 3.228515625 }, { "rank": 8, "tokenId": 236836, "text": "U", "score": 2.30078125 } ] } }, "winnerTextsByMode": { "exact": "Un", "f32_forward": "Un", "f32_reverse": "Un", "f32_tree64": "Un", "f16_forward": "Un" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Un" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "seatbelt-safe-unsafe-choice-oneword-a", "text": "Answer with exactly one word only.\nChoose exactly one word: safe or unsafe.\nDriving without a seatbelt is", "stableTopK": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "logit": 19.58502197265625, "prob": 0.9987403750419617 }, { "rank": 2, "tokenId": 2805, "text": "Un", "logit": 12.907626152038574, "prob": 0.0012574666179716587 }, { "rank": 3, "tokenId": 8069, "text": "uns", "logit": 6.380373001098633, "prob": 0.0000018396975747236866 }, { "rank": 4, "tokenId": 602, "text": "un", "logit": 4.5152058601379395, "prob": 2.849145630534622e-7 }, { "rank": 5, "tokenId": 25793, "text": " unsafe", "logit": 0.6276482939720154, "prob": 5.839421390874122e-9 }, { "rank": 6, "tokenId": 1018, "text": "**", "logit": -1.7324137687683105, "prob": 5.513252632205479e-10 }, { "rank": 7, "tokenId": 25083, "text": "unsigned", "logit": -2.309213876724243, "prob": 3.0967547970384146e-10 }, { "rank": 8, "tokenId": 236840, "text": "[", "logit": -2.687001943588257, "prob": 2.122440556640015e-10 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 23.406667328716992, "winnerGap": 9.600412899753401, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 23.406667328716992 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 13.806254428963591 }, { "rank": 3, "tokenId": 8069, "text": "uns", "score": 6.479270181411536 }, { "rank": 4, "tokenId": 602, "text": "un", "score": 4.54976580266068 }, { "rank": 5, "tokenId": 25793, "text": " unsafe", "score": 0.6277412667061881 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -1.7343458793846374 }, { "rank": 7, "tokenId": 25083, "text": "unsigned", "score": -2.3137911750434483 }, { "rank": 8, "tokenId": 236840, "text": "[", "score": -2.694221753686108 } ] }, "f32_forward": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 23.406673431396484, "winnerGap": 9.600384712219238, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 23.406673431396484 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 13.806288719177246 }, { "rank": 3, "tokenId": 8069, "text": "uns", "score": 6.479254245758057 }, { "rank": 4, "tokenId": 602, "text": "un", "score": 4.5497822761535645 }, { "rank": 5, "tokenId": 25793, "text": " unsafe", "score": 0.627741277217865 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -1.734346628189087 }, { "rank": 7, "tokenId": 25083, "text": "unsigned", "score": -2.3137919902801514 }, { "rank": 8, "tokenId": 236840, "text": "[", "score": -2.694223642349243 } ] }, "f32_reverse": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 23.406667709350586, "winnerGap": 9.600406646728516, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 23.406667709350586 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 13.80626106262207 }, { "rank": 3, "tokenId": 8069, "text": "uns", "score": 6.479274749755859 }, { "rank": 4, "tokenId": 602, "text": "un", "score": 4.5497660636901855 }, { "rank": 5, "tokenId": 25793, "text": " unsafe", "score": 0.6277393698692322 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -1.734347939491272 }, { "rank": 7, "tokenId": 25083, "text": "unsigned", "score": -2.3137996196746826 }, { "rank": 8, "tokenId": 236840, "text": "[", "score": -2.694220542907715 } ] }, "f32_tree64": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 23.406667709350586, "winnerGap": 9.600412368774414, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 23.406667709350586 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 13.806255340576172 }, { "rank": 3, "tokenId": 8069, "text": "uns", "score": 6.479271411895752 }, { "rank": 4, "tokenId": 602, "text": "un", "score": 4.549766540527344 }, { "rank": 5, "tokenId": 25793, "text": " unsafe", "score": 0.6277399063110352 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -1.7343459129333496 }, { "rank": 7, "tokenId": 25083, "text": "unsigned", "score": -2.3137922286987305 }, { "rank": 8, "tokenId": 236840, "text": "[", "score": -2.694221258163452 } ] }, "f16_forward": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 23.578125, "winnerGap": 9.671875, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 23.578125 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 13.90625 }, { "rank": 3, "tokenId": 8069, "text": "uns", "score": 6.55859375 }, { "rank": 4, "tokenId": 602, "text": "un", "score": 4.6015625 }, { "rank": 5, "tokenId": 25793, "text": " unsafe", "score": 0.6748046875 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -1.7451171875 }, { "rank": 7, "tokenId": 25083, "text": "unsigned", "score": -2.41015625 }, { "rank": 8, "tokenId": 3743, "text": "UN", "score": -2.65625 } ] } }, "winnerTextsByMode": { "exact": "unsafe", "f32_forward": "unsafe", "f32_reverse": "unsafe", "f32_tree64": "unsafe", "f16_forward": "unsafe" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "unsafe" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "seventeen-even-odd-oneword-b", "text": "Respond with one word only.\nSeventeen is even or odd. It is", "stableTopK": [ { "rank": 1, "tokenId": 54119, "text": "Odd", "logit": 6.233433723449707, "prob": 0.5704584717750549 }, { "rank": 2, "tokenId": 19808, "text": "odd", "logit": 5.949682235717773, "prob": 0.4295288622379303 }, { "rank": 3, "tokenId": 14986, "text": "Even", "logit": -4.562690734863281, "prob": 0.00001168225753644947 }, { "rank": 4, "tokenId": 20952, "text": "even", "logit": -7.489828109741211, "prob": 6.255862103898835e-7 }, { "rank": 5, "tokenId": 11049, "text": " odd", "logit": -9.03221321105957, "prob": 1.3379437291405338e-7 }, { "rank": 6, "tokenId": 75683, "text": "Seven", "logit": -10.43465805053711, "prob": 3.291271966077147e-8 }, { "rank": 7, "tokenId": 3253, "text": "OD", "logit": -10.606778144836426, "prob": 2.770849683031429e-8 }, { "rank": 8, "tokenId": 236806, "text": "O", "logit": -10.893694877624512, "prob": 2.0797283895035434e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 54119, "winnerText": "Odd", "winnerScore": 6.325535257036204, "winnerGap": 0.29595717692717827, "top": [ { "rank": 1, "tokenId": 54119, "text": "Odd", "score": 6.325535257036204 }, { "rank": 2, "tokenId": 19808, "text": "odd", "score": 6.029578080109026 }, { "rank": 3, "tokenId": 14986, "text": "Even", "score": -4.598369413998704 }, { "rank": 4, "tokenId": 20952, "text": "even", "score": -7.651532395675304 }, { "rank": 5, "tokenId": 11049, "text": " odd", "score": -9.320998623917014 }, { "rank": 6, "tokenId": 75683, "text": "Seven", "score": -10.888920413711276 }, { "rank": 7, "tokenId": 3253, "text": "OD", "score": -11.085169963187226 }, { "rank": 8, "tokenId": 236806, "text": "O", "score": -11.41436126252378 } ] }, "f32_forward": { "winnerTokenId": 54119, "winnerText": "Odd", "winnerScore": 6.325532913208008, "winnerGap": 0.2959561347961426, "top": [ { "rank": 1, "tokenId": 54119, "text": "Odd", "score": 6.325532913208008 }, { "rank": 2, "tokenId": 19808, "text": "odd", "score": 6.029576778411865 }, { "rank": 3, "tokenId": 14986, "text": "Even", "score": -4.598369121551514 }, { "rank": 4, "tokenId": 20952, "text": "even", "score": -7.6515278816223145 }, { "rank": 5, "tokenId": 11049, "text": " odd", "score": -9.321003913879395 }, { "rank": 6, "tokenId": 75683, "text": "Seven", "score": -10.88890552520752 }, { "rank": 7, "tokenId": 3253, "text": "OD", "score": -11.085172653198242 }, { "rank": 8, "tokenId": 236806, "text": "O", "score": -11.41435718536377 } ] }, "f32_reverse": { "winnerTokenId": 54119, "winnerText": "Odd", "winnerScore": 6.325531959533691, "winnerGap": 0.29595088958740234, "top": [ { "rank": 1, "tokenId": 54119, "text": "Odd", "score": 6.325531959533691 }, { "rank": 2, "tokenId": 19808, "text": "odd", "score": 6.029581069946289 }, { "rank": 3, "tokenId": 14986, "text": "Even", "score": -4.59836483001709 }, { "rank": 4, "tokenId": 20952, "text": "even", "score": -7.651536464691162 }, { "rank": 5, "tokenId": 11049, "text": " odd", "score": -9.320993423461914 }, { "rank": 6, "tokenId": 75683, "text": "Seven", "score": -10.888906478881836 }, { "rank": 7, "tokenId": 3253, "text": "OD", "score": -11.085180282592773 }, { "rank": 8, "tokenId": 236806, "text": "O", "score": -11.414361000061035 } ] }, "f32_tree64": { "winnerTokenId": 54119, "winnerText": "Odd", "winnerScore": 6.325535774230957, "winnerGap": 0.2959575653076172, "top": [ { "rank": 1, "tokenId": 54119, "text": "Odd", "score": 6.325535774230957 }, { "rank": 2, "tokenId": 19808, "text": "odd", "score": 6.02957820892334 }, { "rank": 3, "tokenId": 14986, "text": "Even", "score": -4.598367691040039 }, { "rank": 4, "tokenId": 20952, "text": "even", "score": -7.6515326499938965 }, { "rank": 5, "tokenId": 11049, "text": " odd", "score": -9.320998191833496 }, { "rank": 6, "tokenId": 75683, "text": "Seven", "score": -10.888920783996582 }, { "rank": 7, "tokenId": 3253, "text": "OD", "score": -11.08517074584961 }, { "rank": 8, "tokenId": 236806, "text": "O", "score": -11.414361953735352 } ] }, "f16_forward": { "winnerTokenId": 54119, "winnerText": "Odd", "winnerScore": 6.34765625, "winnerGap": 0.19921875, "top": [ { "rank": 1, "tokenId": 54119, "text": "Odd", "score": 6.34765625 }, { "rank": 2, "tokenId": 19808, "text": "odd", "score": 6.1484375 }, { "rank": 3, "tokenId": 20952, "text": "even", "score": -7.453125 }, { "rank": 4, "tokenId": 14986, "text": "Even", "score": -8.359375 }, { "rank": 5, "tokenId": 11049, "text": " odd", "score": -9.1484375 }, { "rank": 6, "tokenId": 3253, "text": "OD", "score": -11.0859375 }, { "rank": 7, "tokenId": 236806, "text": "O", "score": -11.4921875 }, { "rank": 8, "tokenId": 47526, "text": "seven", "score": -11.8359375 } ] } }, "winnerTextsByMode": { "exact": "Odd", "f32_forward": "Odd", "f32_reverse": "Odd", "f32_tree64": "Odd", "f16_forward": "Odd" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Odd" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "seventeen-parity-oneword-c", "text": "No explanation. One word only.\nSeventeen is an", "stableTopK": [ { "rank": 1, "tokenId": 21035, "text": "Age", "logit": -3.1353719234466553, "prob": 0.9965371489524841 }, { "rank": 2, "tokenId": 676, "text": "age", "logit": -9.697868347167969, "prob": 0.0014074646169319749 }, { "rank": 3, "tokenId": 4452, "text": "Number", "logit": -10.442989349365234, "prob": 0.0006680908845737576 }, { "rank": 4, "tokenId": 11980, "text": "Integer", "logit": -10.831707000732422, "prob": 0.0004529159632511437 }, { "rank": 5, "tokenId": 236776, "text": "A", "logit": -11.288780212402344, "prob": 0.00028675649082288146 }, { "rank": 6, "tokenId": 137072, "text": "Ordinal", "logit": -12.194952011108398, "prob": 0.00011586915206862614 }, { "rank": 7, "tokenId": 4036, "text": "Ad", "logit": -13.12108039855957, "prob": 0.0000458939430245664 }, { "rank": 8, "tokenId": 28018, "text": "integer", "logit": -13.208877563476562, "prob": 0.000042036404920509085 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 21035, "winnerText": "Age", "winnerScore": -3.146860043164864, "winnerGap": 6.91171551978038, "top": [ { "rank": 1, "tokenId": 21035, "text": "Age", "score": -3.146860043164864 }, { "rank": 2, "tokenId": 676, "text": "age", "score": -10.058575562945244 }, { "rank": 3, "tokenId": 4452, "text": "Number", "score": -10.898392971955637 }, { "rank": 4, "tokenId": 11980, "text": "Integer", "score": -11.34301771080409 }, { "rank": 5, "tokenId": 236776, "text": "A", "score": -11.872013736841032 }, { "rank": 6, "tokenId": 137072, "text": "Ordinal", "score": -12.942279800479804 }, { "rank": 7, "tokenId": 4036, "text": "Ad", "score": -14.069199381914615 }, { "rank": 8, "tokenId": 28018, "text": "integer", "score": -14.177933194068657 } ] }, "f32_forward": { "winnerTokenId": 21035, "winnerText": "Age", "winnerScore": -3.146855592727661, "winnerGap": 6.911715269088745, "top": [ { "rank": 1, "tokenId": 21035, "text": "Age", "score": -3.146855592727661 }, { "rank": 2, "tokenId": 676, "text": "age", "score": -10.058570861816406 }, { "rank": 3, "tokenId": 4452, "text": "Number", "score": -10.898398399353027 }, { "rank": 4, "tokenId": 11980, "text": "Integer", "score": -11.343011856079102 }, { "rank": 5, "tokenId": 236776, "text": "A", "score": -11.8720121383667 }, { "rank": 6, "tokenId": 137072, "text": "Ordinal", "score": -12.942282676696777 }, { "rank": 7, "tokenId": 4036, "text": "Ad", "score": -14.069202423095703 }, { "rank": 8, "tokenId": 28018, "text": "integer", "score": -14.177926063537598 } ] }, "f32_reverse": { "winnerTokenId": 21035, "winnerText": "Age", "winnerScore": -3.146857500076294, "winnerGap": 6.911707639694214, "top": [ { "rank": 1, "tokenId": 21035, "text": "Age", "score": -3.146857500076294 }, { "rank": 2, "tokenId": 676, "text": "age", "score": -10.058565139770508 }, { "rank": 3, "tokenId": 4452, "text": "Number", "score": -10.898396492004395 }, { "rank": 4, "tokenId": 11980, "text": "Integer", "score": -11.343023300170898 }, { "rank": 5, "tokenId": 236776, "text": "A", "score": -11.871999740600586 }, { "rank": 6, "tokenId": 137072, "text": "Ordinal", "score": -12.942262649536133 }, { "rank": 7, "tokenId": 4036, "text": "Ad", "score": -14.06919002532959 }, { "rank": 8, "tokenId": 28018, "text": "integer", "score": -14.177924156188965 } ] }, "f32_tree64": { "winnerTokenId": 21035, "winnerText": "Age", "winnerScore": -3.146859645843506, "winnerGap": 6.911716938018799, "top": [ { "rank": 1, "tokenId": 21035, "text": "Age", "score": -3.146859645843506 }, { "rank": 2, "tokenId": 676, "text": "age", "score": -10.058576583862305 }, { "rank": 3, "tokenId": 4452, "text": "Number", "score": -10.898392677307129 }, { "rank": 4, "tokenId": 11980, "text": "Integer", "score": -11.343017578125 }, { "rank": 5, "tokenId": 236776, "text": "A", "score": -11.8720121383667 }, { "rank": 6, "tokenId": 137072, "text": "Ordinal", "score": -12.942280769348145 }, { "rank": 7, "tokenId": 4036, "text": "Ad", "score": -14.069198608398438 }, { "rank": 8, "tokenId": 28018, "text": "integer", "score": -14.177933692932129 } ] }, "f16_forward": { "winnerTokenId": 21035, "winnerText": "Age", "winnerScore": -3.1484375, "winnerGap": 3.75390625, "top": [ { "rank": 1, "tokenId": 21035, "text": "Age", "score": -3.1484375 }, { "rank": 2, "tokenId": 4452, "text": "Number", "score": -6.90234375 }, { "rank": 3, "tokenId": 236776, "text": "A", "score": -7.9921875 }, { "rank": 4, "tokenId": 236788, "text": "E", "score": -8.6875 }, { "rank": 5, "tokenId": 676, "text": "age", "score": -10.0859375 }, { "rank": 6, "tokenId": 11980, "text": "Integer", "score": -11.296875 }, { "rank": 7, "tokenId": 137072, "text": "Ordinal", "score": -12.9140625 }, { "rank": 8, "tokenId": 4036, "text": "Ad", "score": -14.109375 } ] } }, "winnerTextsByMode": { "exact": "Age", "f32_forward": "Age", "f32_reverse": "Age", "f32_tree64": "Age", "f16_forward": "Age" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Age" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "ship-yes-no-both-oneword-d", "text": "Choose one word only.\nYes, no, or both: a ship rebuilt plank by plank over decades is still the same ship", "stableTopK": [ { "rank": 1, "tokenId": 3771, "text": "No", "logit": 15.687628746032715, "prob": 0.9167765378952026 }, { "rank": 2, "tokenId": 22186, "text": "Both", "logit": 12.68968677520752, "prob": 0.04573764652013779 }, { "rank": 3, "tokenId": 27413, "text": "both", "logit": 11.772346496582031, "prob": 0.01827586628496647 }, { "rank": 4, "tokenId": 1904, "text": "no", "logit": 11.586185455322266, "prob": 0.015171526931226254 }, { "rank": 5, "tokenId": 1018, "text": "**", "logit": 9.868634223937988, "prob": 0.002723367651924491 }, { "rank": 6, "tokenId": 10784, "text": "Yes", "logit": 8.372236251831055, "prob": 0.0006098581943660975 }, { "rank": 7, "tokenId": 31717, "text": "Still", "logit": 6.91640567779541, "prob": 0.00014222295430954546 }, { "rank": 8, "tokenId": 95889, "text": "Neither", "logit": 6.7212815284729, "prob": 0.00011701144103426486 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 17.410550770786266, "winnerGap": 3.8707283335631253, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 17.410550770786266 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 13.53982243722314 }, { "rank": 3, "tokenId": 27413, "text": "both", "score": 12.43941958396245 }, { "rank": 4, "tokenId": 1904, "text": "no", "score": 12.220004061706419 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 10.249664157322961 }, { "rank": 6, "tokenId": 10784, "text": "Yes", "score": 8.600346208406528 }, { "rank": 7, "tokenId": 31717, "text": "Still", "score": 7.043012219357976 }, { "rank": 8, "tokenId": 95889, "text": "Neither", "score": 6.837255257740111 } ] }, "f32_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 17.410533905029297, "winnerGap": 3.8707141876220703, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 17.410533905029297 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 13.539819717407227 }, { "rank": 3, "tokenId": 27413, "text": "both", "score": 12.439414024353027 }, { "rank": 4, "tokenId": 1904, "text": "no", "score": 12.220009803771973 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 10.24966049194336 }, { "rank": 6, "tokenId": 10784, "text": "Yes", "score": 8.600339889526367 }, { "rank": 7, "tokenId": 31717, "text": "Still", "score": 7.043008327484131 }, { "rank": 8, "tokenId": 95889, "text": "Neither", "score": 6.837247371673584 } ] }, "f32_reverse": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 17.410568237304688, "winnerGap": 3.8707523345947266, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 17.410568237304688 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 13.539815902709961 }, { "rank": 3, "tokenId": 27413, "text": "both", "score": 12.439428329467773 }, { "rank": 4, "tokenId": 1904, "text": "no", "score": 12.220003128051758 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 10.249677658081055 }, { "rank": 6, "tokenId": 10784, "text": "Yes", "score": 8.600342750549316 }, { "rank": 7, "tokenId": 31717, "text": "Still", "score": 7.043012619018555 }, { "rank": 8, "tokenId": 95889, "text": "Neither", "score": 6.837253093719482 } ] }, "f32_tree64": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 17.410551071166992, "winnerGap": 3.8707284927368164, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 17.410551071166992 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 13.539822578430176 }, { "rank": 3, "tokenId": 27413, "text": "both", "score": 12.43941879272461 }, { "rank": 4, "tokenId": 1904, "text": "no", "score": 12.220004081726074 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": 10.249664306640625 }, { "rank": 6, "tokenId": 10784, "text": "Yes", "score": 8.600346565246582 }, { "rank": 7, "tokenId": 31717, "text": "Still", "score": 7.043011665344238 }, { "rank": 8, "tokenId": 95889, "text": "Neither", "score": 6.837254524230957 } ] }, "f16_forward": { "winnerTokenId": 3771, "winnerText": "No", "winnerScore": 17.46875, "winnerGap": 3.9765625, "top": [ { "rank": 1, "tokenId": 3771, "text": "No", "score": 17.46875 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 13.4921875 }, { "rank": 3, "tokenId": 1904, "text": "no", "score": 12.2109375 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 10.3046875 }, { "rank": 5, "tokenId": 27413, "text": "both", "score": 8.53125 }, { "rank": 6, "tokenId": 95889, "text": "Neither", "score": 6.8515625 }, { "rank": 7, "tokenId": 113214, "text": "depends", "score": 6.37109375 }, { "rank": 8, "tokenId": 1525, "text": "amb", "score": 5.703125 } ] } }, "winnerTextsByMode": { "exact": "No", "f32_forward": "No", "f32_reverse": "No", "f32_tree64": "No", "f16_forward": "No" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "No" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "signal-noise-explicit-oneword-a", "text": "Answer with exactly one word only.\nAnswer with exactly one word: signal or noise. Question: Dreams are Answer:", "stableTopK": [ { "rank": 1, "tokenId": 33786, "text": "signal", "logit": 12.911979675292969, "prob": 0.9941202998161316 }, { "rank": 2, "tokenId": 33828, "text": "noise", "logit": 7.5405354499816895, "prob": 0.004620089195668697 }, { "rank": 3, "tokenId": 53159, "text": "Signal", "logit": 5.884564399719238, "prob": 0.0008820054936222732 }, { "rank": 4, "tokenId": 152593, "text": "signals", "logit": 4.330732345581055, "prob": 0.0001864878722699359 }, { "rank": 5, "tokenId": 73678, "text": "Noise", "logit": 2.4288203716278076, "prob": 0.000027839454560307786 }, { "rank": 6, "tokenId": 27413, "text": "both", "logit": 1.9014413356781006, "prob": 0.000016429446986876428 }, { "rank": 7, "tokenId": 23245, "text": "information", "logit": 1.621250867843628, "prob": 0.000012414742741384543 }, { "rank": 8, "tokenId": 198275, "text": "neither", "logit": 1.2801660299301147, "prob": 0.000008826865268929396 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 33786, "winnerText": "signal", "winnerScore": 13.811594188676905, "winnerGap": 6.1059606341961565, "top": [ { "rank": 1, "tokenId": 33786, "text": "signal", "score": 13.811594188676905 }, { "rank": 2, "tokenId": 33828, "text": "noise", "score": 7.705633554480748 }, { "rank": 3, "tokenId": 53159, "text": "Signal", "score": 5.961823448103015 }, { "rank": 4, "tokenId": 152593, "text": "signals", "score": 4.3611954111965145 }, { "rank": 5, "tokenId": 73678, "text": "Noise", "score": 2.434146065907033 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": 1.9039961331887056 }, { "rank": 7, "tokenId": 23245, "text": "information", "score": 1.622833152878334 }, { "rank": 8, "tokenId": 198275, "text": "neither", "score": 1.2809471617090633 } ] }, "f32_forward": { "winnerTokenId": 33786, "winnerText": "signal", "winnerScore": 13.81158447265625, "winnerGap": 6.10595178604126, "top": [ { "rank": 1, "tokenId": 33786, "text": "signal", "score": 13.81158447265625 }, { "rank": 2, "tokenId": 33828, "text": "noise", "score": 7.70563268661499 }, { "rank": 3, "tokenId": 53159, "text": "Signal", "score": 5.961822509765625 }, { "rank": 4, "tokenId": 152593, "text": "signals", "score": 4.361189365386963 }, { "rank": 5, "tokenId": 73678, "text": "Noise", "score": 2.4341392517089844 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": 1.9040011167526245 }, { "rank": 7, "tokenId": 23245, "text": "information", "score": 1.6228289604187012 }, { "rank": 8, "tokenId": 198275, "text": "neither", "score": 1.280947208404541 } ] }, "f32_reverse": { "winnerTokenId": 33786, "winnerText": "signal", "winnerScore": 13.811592102050781, "winnerGap": 6.10595703125, "top": [ { "rank": 1, "tokenId": 33786, "text": "signal", "score": 13.811592102050781 }, { "rank": 2, "tokenId": 33828, "text": "noise", "score": 7.705635070800781 }, { "rank": 3, "tokenId": 53159, "text": "Signal", "score": 5.961828231811523 }, { "rank": 4, "tokenId": 152593, "text": "signals", "score": 4.361192226409912 }, { "rank": 5, "tokenId": 73678, "text": "Noise", "score": 2.4341442584991455 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": 1.903997540473938 }, { "rank": 7, "tokenId": 23245, "text": "information", "score": 1.622833251953125 }, { "rank": 8, "tokenId": 198275, "text": "neither", "score": 1.2809457778930664 } ] }, "f32_tree64": { "winnerTokenId": 33786, "winnerText": "signal", "winnerScore": 13.811594009399414, "winnerGap": 6.105960845947266, "top": [ { "rank": 1, "tokenId": 33786, "text": "signal", "score": 13.811594009399414 }, { "rank": 2, "tokenId": 33828, "text": "noise", "score": 7.705633163452148 }, { "rank": 3, "tokenId": 53159, "text": "Signal", "score": 5.961823463439941 }, { "rank": 4, "tokenId": 152593, "text": "signals", "score": 4.361194610595703 }, { "rank": 5, "tokenId": 73678, "text": "Noise", "score": 2.43414568901062 }, { "rank": 6, "tokenId": 27413, "text": "both", "score": 1.9039958715438843 }, { "rank": 7, "tokenId": 23245, "text": "information", "score": 1.6228337287902832 }, { "rank": 8, "tokenId": 198275, "text": "neither", "score": 1.2809473276138306 } ] }, "f16_forward": { "winnerTokenId": 33786, "winnerText": "signal", "winnerScore": 13.96875, "winnerGap": 7.98828125, "top": [ { "rank": 1, "tokenId": 33786, "text": "signal", "score": 13.96875 }, { "rank": 2, "tokenId": 53159, "text": "Signal", "score": 5.98046875 }, { "rank": 3, "tokenId": 152593, "text": "signals", "score": 4.33984375 }, { "rank": 4, "tokenId": 33828, "text": "noise", "score": 3.779296875 }, { "rank": 5, "tokenId": 27413, "text": "both", "score": 1.9404296875 }, { "rank": 6, "tokenId": 23245, "text": "information", "score": 1.630859375 }, { "rank": 7, "tokenId": 73678, "text": "Noise", "score": 1.4638671875 }, { "rank": 8, "tokenId": 198275, "text": "neither", "score": 1.302734375 } ] } }, "winnerTextsByMode": { "exact": "signal", "f32_forward": "signal", "f32_reverse": "signal", "f32_tree64": "signal", "f16_forward": "signal" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "signal" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "sky-blue-green-oneword-b", "text": "Respond with one word only.\nThe sky on a clear day is blue or green. It is", "stableTopK": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "logit": 6.695894718170166, "prob": 0.9876295924186707 }, { "rank": 2, "tokenId": 9503, "text": "blue", "logit": 2.2623190879821777, "prob": 0.011725102551281452 }, { "rank": 3, "tokenId": 48818, "text": "Sky", "logit": -2.0734715461730957, "prob": 0.00015349940804298967 }, { "rank": 4, "tokenId": 77308, "text": "Azure", "logit": -3.002999782562256, "prob": 0.0000605923414696008 }, { "rank": 5, "tokenId": 236847, "text": "V", "logit": -4.318343639373779, "prob": 0.00001626189441594761 }, { "rank": 6, "tokenId": 15952, "text": "Beautiful", "logit": -4.3660054206848145, "prob": 0.00001550500383018516 }, { "rank": 7, "tokenId": 149876, "text": "蓝色", "logit": -4.527968883514404, "prob": 0.000013186576325097121 }, { "rank": 8, "tokenId": 3508, "text": "Bl", "logit": -4.62997579574585, "prob": 0.000011907785847142804 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 6.810529814090543, "winnerGap": 4.543903909130499, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 6.810529814090543 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": 2.266625904960044 }, { "rank": 3, "tokenId": 48818, "text": "Sky", "score": -2.0767784733122987 }, { "rank": 4, "tokenId": 77308, "text": "Azure", "score": -3.013088785048179 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": -4.348544888221944 }, { "rank": 6, "tokenId": 15952, "text": "Beautiful", "score": -4.39722907631505 }, { "rank": 7, "tokenId": 149876, "text": "蓝色", "score": -4.562825664352033 }, { "rank": 8, "tokenId": 3508, "text": "Bl", "score": -4.667271938791732 } ] }, "f32_forward": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 6.810532093048096, "winnerGap": 4.543903112411499, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 6.810532093048096 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": 2.2666289806365967 }, { "rank": 3, "tokenId": 48818, "text": "Sky", "score": -2.0767738819122314 }, { "rank": 4, "tokenId": 77308, "text": "Azure", "score": -3.0130882263183594 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": -4.348548889160156 }, { "rank": 6, "tokenId": 15952, "text": "Beautiful", "score": -4.397233486175537 }, { "rank": 7, "tokenId": 149876, "text": "蓝色", "score": -4.562830448150635 }, { "rank": 8, "tokenId": 3508, "text": "Bl", "score": -4.667274475097656 } ] }, "f32_reverse": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 6.810528755187988, "winnerGap": 4.543899297714233, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 6.810528755187988 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": 2.266629457473755 }, { "rank": 3, "tokenId": 48818, "text": "Sky", "score": -2.0767786502838135 }, { "rank": 4, "tokenId": 77308, "text": "Azure", "score": -3.0130844116210938 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": -4.348543167114258 }, { "rank": 6, "tokenId": 15952, "text": "Beautiful", "score": -4.397219181060791 }, { "rank": 7, "tokenId": 149876, "text": "蓝色", "score": -4.562821865081787 }, { "rank": 8, "tokenId": 3508, "text": "Bl", "score": -4.667271137237549 } ] }, "f32_tree64": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 6.810529708862305, "winnerGap": 4.543903827667236, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 6.810529708862305 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": 2.2666258811950684 }, { "rank": 3, "tokenId": 48818, "text": "Sky", "score": -2.076777696609497 }, { "rank": 4, "tokenId": 77308, "text": "Azure", "score": -3.013089179992676 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": -4.348544120788574 }, { "rank": 6, "tokenId": 15952, "text": "Beautiful", "score": -4.397229194641113 }, { "rank": 7, "tokenId": 149876, "text": "蓝色", "score": -4.562826633453369 }, { "rank": 8, "tokenId": 3508, "text": "Bl", "score": -4.667272567749023 } ] }, "f16_forward": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 6.79296875, "winnerGap": 4.5078125, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 6.79296875 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": 2.28515625 }, { "rank": 3, "tokenId": 48818, "text": "Sky", "score": -2.041015625 }, { "rank": 4, "tokenId": 77308, "text": "Azure", "score": -3.00390625 }, { "rank": 5, "tokenId": 128280, "text": "Rare", "score": -3.419921875 }, { "rank": 6, "tokenId": 165451, "text": "Atm", "score": -4.25390625 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": -4.29296875 }, { "rank": 8, "tokenId": 15952, "text": "Beautiful", "score": -4.37109375 } ] } }, "winnerTextsByMode": { "exact": "Blue", "f32_forward": "Blue", "f32_reverse": "Blue", "f32_tree64": "Blue", "f16_forward": "Blue" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Blue" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "sky-is-oneword-c", "text": "No explanation. One word only.\nThe sky is", "stableTopK": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "logit": 6.162533283233643, "prob": 0.9999456405639648 }, { "rank": 2, "tokenId": 9503, "text": "blue", "logit": -4.158493995666504, "prob": 0.00003293147528893314 }, { "rank": 3, "tokenId": 149876, "text": "蓝色", "logit": -5.131762504577637, "prob": 0.00001244302711711498 }, { "rank": 4, "tokenId": 236847, "text": "V", "logit": -5.80561637878418, "prob": 0.000006342712367768399 }, { "rank": 5, "tokenId": 77308, "text": "Azure", "logit": -6.939150810241699, "prob": 0.000002041678271780256 }, { "rank": 6, "tokenId": 240123, "text": "蓝", "logit": -9.262478828430176, "prob": 1.9997639810753753e-7 }, { "rank": 7, "tokenId": 132602, "text": "Infinite", "logit": -10.057082176208496, "prob": 9.03414090203114e-8 }, { "rank": 8, "tokenId": 3508, "text": "Bl", "logit": -10.435860633850098, "prob": 6.185651812984361e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 6.251472654847834, "winnerGap": 10.436910813234814, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 6.251472654847834 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": -4.1854381583869795 }, { "rank": 3, "tokenId": 149876, "text": "蓝色", "score": -5.182708557471521 }, { "rank": 4, "tokenId": 236847, "text": "V", "score": -5.879761662172282 }, { "rank": 5, "tokenId": 77308, "text": "Azure", "score": -7.067024770681112 }, { "rank": 6, "tokenId": 240123, "text": "蓝", "score": -9.574864772080854 }, { "rank": 7, "tokenId": 132602, "text": "Infinite", "score": -10.461465950414649 }, { "rank": 8, "tokenId": 3508, "text": "Bl", "score": -10.890282588242982 } ] }, "f32_forward": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 6.251471996307373, "winnerGap": 10.436909675598145, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 6.251471996307373 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": -4.1854376792907715 }, { "rank": 3, "tokenId": 149876, "text": "蓝色", "score": -5.182710647583008 }, { "rank": 4, "tokenId": 236847, "text": "V", "score": -5.879772186279297 }, { "rank": 5, "tokenId": 77308, "text": "Azure", "score": -7.067021369934082 }, { "rank": 6, "tokenId": 240123, "text": "蓝", "score": -9.574867248535156 }, { "rank": 7, "tokenId": 132602, "text": "Infinite", "score": -10.461467742919922 }, { "rank": 8, "tokenId": 3508, "text": "Bl", "score": -10.890281677246094 } ] }, "f32_reverse": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 6.251479625701904, "winnerGap": 10.436922073364258, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 6.251479625701904 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": -4.1854424476623535 }, { "rank": 3, "tokenId": 149876, "text": "蓝色", "score": -5.182699680328369 }, { "rank": 4, "tokenId": 236847, "text": "V", "score": -5.879751682281494 }, { "rank": 5, "tokenId": 77308, "text": "Azure", "score": -7.067015171051025 }, { "rank": 6, "tokenId": 240123, "text": "蓝", "score": -9.57485580444336 }, { "rank": 7, "tokenId": 132602, "text": "Infinite", "score": -10.461457252502441 }, { "rank": 8, "tokenId": 3508, "text": "Bl", "score": -10.89029598236084 } ] }, "f32_tree64": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 6.251471996307373, "winnerGap": 10.436907291412354, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 6.251471996307373 }, { "rank": 2, "tokenId": 9503, "text": "blue", "score": -4.1854352951049805 }, { "rank": 3, "tokenId": 149876, "text": "蓝色", "score": -5.182708740234375 }, { "rank": 4, "tokenId": 236847, "text": "V", "score": -5.879759788513184 }, { "rank": 5, "tokenId": 77308, "text": "Azure", "score": -7.067024230957031 }, { "rank": 6, "tokenId": 240123, "text": "蓝", "score": -9.574864387512207 }, { "rank": 7, "tokenId": 132602, "text": "Infinite", "score": -10.461464881896973 }, { "rank": 8, "tokenId": 3508, "text": "Bl", "score": -10.890283584594727 } ] }, "f16_forward": { "winnerTokenId": 16520, "winnerText": "Blue", "winnerScore": 6.32421875, "winnerGap": 9.3359375, "top": [ { "rank": 1, "tokenId": 16520, "text": "Blue", "score": 6.32421875 }, { "rank": 2, "tokenId": 77308, "text": "Azure", "score": -3.01171875 }, { "rank": 3, "tokenId": 9503, "text": "blue", "score": -4.18359375 }, { "rank": 4, "tokenId": 149876, "text": "蓝色", "score": -5.1953125 }, { "rank": 5, "tokenId": 236847, "text": "V", "score": -5.79296875 }, { "rank": 6, "tokenId": 9595, "text": " Blue", "score": -9.8984375 }, { "rank": 7, "tokenId": 132602, "text": "Infinite", "score": -10.3671875 }, { "rank": 8, "tokenId": 3508, "text": "Bl", "score": -10.9140625 } ] } }, "winnerTextsByMode": { "exact": "Blue", "f32_forward": "Blue", "f32_reverse": "Blue", "f32_tree64": "Blue", "f16_forward": "Blue" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Blue" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "snow-season-oneword-d", "text": "Choose one word only.\nSnow is most common in", "stableTopK": [ { "rank": 1, "tokenId": 43335, "text": "Winter", "logit": 11.171932220458984, "prob": 0.9774720668792725 }, { "rank": 2, "tokenId": 57750, "text": "winter", "logit": 7.215270519256592, "prob": 0.018695969134569168 }, { "rank": 3, "tokenId": 1018, "text": "**", "logit": 4.893152713775635, "prob": 0.0018334330525249243 }, { "rank": 4, "tokenId": 208571, "text": "Mountains", "logit": 4.643496036529541, "prob": 0.0014283694326877594 }, { "rank": 5, "tokenId": 66020, "text": "Cold", "logit": 3.1554670333862305, "prob": 0.00032255053520202637 }, { "rank": 6, "tokenId": 66471, "text": "cold", "logit": 1.4769984483718872, "prob": 0.000060207152273505926 }, { "rank": 7, "tokenId": 243152, "text": "❄", "logit": 1.084167242050171, "prob": 0.00004064842141815461 }, { "rank": 8, "tokenId": 150131, "text": "Arctic", "logit": 1.0442485809326172, "prob": 0.00003905775156454183 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 43335, "winnerText": "Winter", "winnerScore": 11.736115948370681, "winnerGap": 4.3766771162838225, "top": [ { "rank": 1, "tokenId": 43335, "text": "Winter", "score": 11.736115948370681 }, { "rank": 2, "tokenId": 57750, "text": "winter", "score": 7.359438832086859 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 4.937248960527317 }, { "rank": 4, "tokenId": 208571, "text": "Mountains", "score": 4.681122150924114 }, { "rank": 5, "tokenId": 66020, "text": "Cold", "score": 3.167181433402103 }, { "rank": 6, "tokenId": 66471, "text": "cold", "score": 1.4781921637751534 }, { "rank": 7, "tokenId": 243152, "text": "❄", "score": 1.0846374233581102 }, { "rank": 8, "tokenId": 150131, "text": "Arctic", "score": 1.044671652699492 } ] }, "f32_forward": { "winnerTokenId": 43335, "winnerText": "Winter", "winnerScore": 11.736115455627441, "winnerGap": 4.376676082611084, "top": [ { "rank": 1, "tokenId": 43335, "text": "Winter", "score": 11.736115455627441 }, { "rank": 2, "tokenId": 57750, "text": "winter", "score": 7.359439373016357 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 4.937255382537842 }, { "rank": 4, "tokenId": 208571, "text": "Mountains", "score": 4.681124210357666 }, { "rank": 5, "tokenId": 66020, "text": "Cold", "score": 3.1671831607818604 }, { "rank": 6, "tokenId": 66471, "text": "cold", "score": 1.4781910181045532 }, { "rank": 7, "tokenId": 243152, "text": "❄", "score": 1.0846278667449951 }, { "rank": 8, "tokenId": 150131, "text": "Arctic", "score": 1.0446710586547852 } ] }, "f32_reverse": { "winnerTokenId": 43335, "winnerText": "Winter", "winnerScore": 11.736109733581543, "winnerGap": 4.376674652099609, "top": [ { "rank": 1, "tokenId": 43335, "text": "Winter", "score": 11.736109733581543 }, { "rank": 2, "tokenId": 57750, "text": "winter", "score": 7.359435081481934 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 4.937253475189209 }, { "rank": 4, "tokenId": 208571, "text": "Mountains", "score": 4.681121349334717 }, { "rank": 5, "tokenId": 66020, "text": "Cold", "score": 3.1671810150146484 }, { "rank": 6, "tokenId": 66471, "text": "cold", "score": 1.478196382522583 }, { "rank": 7, "tokenId": 243152, "text": "❄", "score": 1.0846283435821533 }, { "rank": 8, "tokenId": 150131, "text": "Arctic", "score": 1.0446717739105225 } ] }, "f32_tree64": { "winnerTokenId": 43335, "winnerText": "Winter", "winnerScore": 11.736116409301758, "winnerGap": 4.376677989959717, "top": [ { "rank": 1, "tokenId": 43335, "text": "Winter", "score": 11.736116409301758 }, { "rank": 2, "tokenId": 57750, "text": "winter", "score": 7.359438419342041 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 4.937248229980469 }, { "rank": 4, "tokenId": 208571, "text": "Mountains", "score": 4.681121349334717 }, { "rank": 5, "tokenId": 66020, "text": "Cold", "score": 3.167182207107544 }, { "rank": 6, "tokenId": 66471, "text": "cold", "score": 1.4781928062438965 }, { "rank": 7, "tokenId": 243152, "text": "❄", "score": 1.0846364498138428 }, { "rank": 8, "tokenId": 150131, "text": "Arctic", "score": 1.0446707010269165 } ] }, "f16_forward": { "winnerTokenId": 57750, "winnerText": "winter", "winnerScore": 7.328125, "winnerGap": 0.53125, "top": [ { "rank": 1, "tokenId": 57750, "text": "winter", "score": 7.328125 }, { "rank": 2, "tokenId": 43335, "text": "Winter", "score": 6.796875 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 4.8984375 }, { "rank": 4, "tokenId": 66020, "text": "Cold", "score": 3.19921875 }, { "rank": 5, "tokenId": 243152, "text": "❄", "score": 2.1796875 }, { "rank": 6, "tokenId": 208571, "text": "Mountains", "score": 1.6142578125 }, { "rank": 7, "tokenId": 66471, "text": "cold", "score": 1.5009765625 }, { "rank": 8, "tokenId": 150131, "text": "Arctic", "score": 1.025390625 } ] } }, "winnerTextsByMode": { "exact": "Winter", "f32_forward": "Winter", "f32_reverse": "Winter", "f32_tree64": "Winter", "f16_forward": "winter" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Winter", "winter" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 43335, 106 ], "decodedText": "Winter", "steps": [ { "step": 0, "tokenId": 43335, "text": "Winter" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 57750, 106 ], "decodedText": "winter", "steps": [ { "step": 0, "tokenId": 57750, "text": "winter" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 0, "differingStepCount": 1, "healedAtStep": 1, "persistsThroughEnd": false } }, { "id": "snow-summer-winter-oneword-a", "text": "Answer with exactly one word only.\nSnow is most common in summer or winter. It is", "stableTopK": [ { "rank": 1, "tokenId": 43335, "text": "Winter", "logit": 11.40153694152832, "prob": 0.9958370327949524 }, { "rank": 2, "tokenId": 57750, "text": "winter", "logit": 5.856142997741699, "prob": 0.0038891462609171867 }, { "rank": 3, "tokenId": 19204, "text": "Variable", "logit": 1.1983399391174316, "prob": 0.000036897428799420595 }, { "rank": 4, "tokenId": 50483, "text": "Snow", "logit": 0.9870010614395142, "prob": 0.000029868459023418836 }, { "rank": 5, "tokenId": 35734, "text": "Summer", "logit": 0.8884670734405518, "prob": 0.00002706574923649896 }, { "rank": 6, "tokenId": 184402, "text": "Seasonal", "logit": 0.7279316186904907, "prob": 0.000023051563402987085 }, { "rank": 7, "tokenId": 49385, "text": "Season", "logit": 0.4341571033000946, "prob": 0.00001718366183922626 }, { "rank": 8, "tokenId": 26588, "text": "Weather", "logit": 0.3533409833908081, "prob": 0.00001584957863087766 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 43335, "winnerText": "Winter", "winnerScore": 12.00358299484612, "winnerGap": 6.071304661188044, "top": [ { "rank": 1, "tokenId": 43335, "text": "Winter", "score": 12.00358299484612 }, { "rank": 2, "tokenId": 57750, "text": "winter", "score": 5.9322783336580756 }, { "rank": 3, "tokenId": 19204, "text": "Variable", "score": 1.1989797813146836 }, { "rank": 4, "tokenId": 50483, "text": "Snow", "score": 0.987357238774786 }, { "rank": 5, "tokenId": 35734, "text": "Summer", "score": 0.8887287988585584 }, { "rank": 6, "tokenId": 184402, "text": "Seasonal", "score": 0.728074832680818 }, { "rank": 7, "tokenId": 49385, "text": "Season", "score": 0.4341871499538428 }, { "rank": 8, "tokenId": 26588, "text": "Weather", "score": 0.3533538236962812 } ] }, "f32_forward": { "winnerTokenId": 43335, "winnerText": "Winter", "winnerScore": 12.003581047058105, "winnerGap": 6.071292877197266, "top": [ { "rank": 1, "tokenId": 43335, "text": "Winter", "score": 12.003581047058105 }, { "rank": 2, "tokenId": 57750, "text": "winter", "score": 5.93228816986084 }, { "rank": 3, "tokenId": 19204, "text": "Variable", "score": 1.198974847793579 }, { "rank": 4, "tokenId": 50483, "text": "Snow", "score": 0.9873589277267456 }, { "rank": 5, "tokenId": 35734, "text": "Summer", "score": 0.8887293338775635 }, { "rank": 6, "tokenId": 184402, "text": "Seasonal", "score": 0.7280740141868591 }, { "rank": 7, "tokenId": 49385, "text": "Season", "score": 0.4341878294944763 }, { "rank": 8, "tokenId": 26588, "text": "Weather", "score": 0.35335394740104675 } ] }, "f32_reverse": { "winnerTokenId": 43335, "winnerText": "Winter", "winnerScore": 12.003584861755371, "winnerGap": 6.07130765914917, "top": [ { "rank": 1, "tokenId": 43335, "text": "Winter", "score": 12.003584861755371 }, { "rank": 2, "tokenId": 57750, "text": "winter", "score": 5.932277202606201 }, { "rank": 3, "tokenId": 19204, "text": "Variable", "score": 1.198976993560791 }, { "rank": 4, "tokenId": 50483, "text": "Snow", "score": 0.9873571395874023 }, { "rank": 5, "tokenId": 35734, "text": "Summer", "score": 0.8887265920639038 }, { "rank": 6, "tokenId": 184402, "text": "Seasonal", "score": 0.7280747294425964 }, { "rank": 7, "tokenId": 49385, "text": "Season", "score": 0.4341854453086853 }, { "rank": 8, "tokenId": 26588, "text": "Weather", "score": 0.3533536493778229 } ] }, "f32_tree64": { "winnerTokenId": 43335, "winnerText": "Winter", "winnerScore": 12.003582000732422, "winnerGap": 6.071303844451904, "top": [ { "rank": 1, "tokenId": 43335, "text": "Winter", "score": 12.003582000732422 }, { "rank": 2, "tokenId": 57750, "text": "winter", "score": 5.932278156280518 }, { "rank": 3, "tokenId": 19204, "text": "Variable", "score": 1.1989799737930298 }, { "rank": 4, "tokenId": 50483, "text": "Snow", "score": 0.9873571395874023 }, { "rank": 5, "tokenId": 35734, "text": "Summer", "score": 0.8887288570404053 }, { "rank": 6, "tokenId": 184402, "text": "Seasonal", "score": 0.7280745506286621 }, { "rank": 7, "tokenId": 49385, "text": "Season", "score": 0.4341864585876465 }, { "rank": 8, "tokenId": 26588, "text": "Weather", "score": 0.35335326194763184 } ] }, "f16_forward": { "winnerTokenId": 43335, "winnerText": "Winter", "winnerScore": 7.02734375, "winnerGap": 1.04296875, "top": [ { "rank": 1, "tokenId": 43335, "text": "Winter", "score": 7.02734375 }, { "rank": 2, "tokenId": 57750, "text": "winter", "score": 5.984375 }, { "rank": 3, "tokenId": 19204, "text": "Variable", "score": 1.2412109375 }, { "rank": 4, "tokenId": 50483, "text": "Snow", "score": 0.94873046875 }, { "rank": 5, "tokenId": 35734, "text": "Summer", "score": 0.89892578125 }, { "rank": 6, "tokenId": 49385, "text": "Season", "score": 0.42333984375 }, { "rank": 7, "tokenId": 26588, "text": "Weather", "score": 0.331298828125 }, { "rank": 8, "tokenId": 66020, "text": "Cold", "score": 0.192626953125 } ] } }, "winnerTextsByMode": { "exact": "Winter", "f32_forward": "Winter", "f32_reverse": "Winter", "f32_tree64": "Winter", "f16_forward": "Winter" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Winter" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "ssh-allow-block-choice-oneword-b", "text": "Respond with one word only.\nChoose exactly one word: allow or block.\nInbound SSH from the public internet should", "stableTopK": [ { "rank": 1, "tokenId": 5088, "text": "block", "logit": 9.71688461303711, "prob": 0.9174481630325317 }, { "rank": 2, "tokenId": 8021, "text": "allow", "logit": 7.272457122802734, "prob": 0.07961229979991913 }, { "rank": 3, "tokenId": 27122, "text": "Allow", "logit": 3.7575864791870117, "prob": 0.002368597313761711 }, { "rank": 4, "tokenId": 10809, "text": "Block", "logit": 2.3069875240325928, "prob": 0.0005552698858082294 }, { "rank": 5, "tokenId": 205380, "text": "deny", "logit": -1.376935362815857, "prob": 0.000013950723769085016 }, { "rank": 6, "tokenId": 1018, "text": "**", "logit": -5.3749871253967285, "prob": 2.560147152053105e-7 }, { "rank": 7, "tokenId": 70961, "text": " ब्लॉक", "logit": -5.7221832275390625, "prob": 1.809170839806029e-7 }, { "rank": 8, "tokenId": 112780, "text": "authorize", "logit": -5.8392815589904785, "prob": 1.609253246215303e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 5088, "winnerText": "block", "winnerScore": 10.07981056536073, "winnerGap": 2.659657493768165, "top": [ { "rank": 1, "tokenId": 5088, "text": "block", "score": 10.07981056536073 }, { "rank": 2, "tokenId": 8021, "text": "allow", "score": 7.420153071592566 }, { "rank": 3, "tokenId": 27122, "text": "Allow", "score": 3.777429587789314 }, { "rank": 4, "tokenId": 10809, "text": "Block", "score": 2.3115534757566394 }, { "rank": 5, "tokenId": 205380, "text": "deny", "score": -1.377901431275859 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -5.433635580429524 }, { "rank": 7, "tokenId": 70961, "text": " ब्लॉक", "score": -5.7931315444433125 }, { "rank": 8, "tokenId": 112780, "text": "authorize", "score": -5.914745739274046 } ] }, "f32_forward": { "winnerTokenId": 5088, "winnerText": "block", "winnerScore": 10.079815864562988, "winnerGap": 2.6596693992614746, "top": [ { "rank": 1, "tokenId": 5088, "text": "block", "score": 10.079815864562988 }, { "rank": 2, "tokenId": 8021, "text": "allow", "score": 7.420146465301514 }, { "rank": 3, "tokenId": 27122, "text": "Allow", "score": 3.777431011199951 }, { "rank": 4, "tokenId": 10809, "text": "Block", "score": 2.3115553855895996 }, { "rank": 5, "tokenId": 205380, "text": "deny", "score": -1.3779040575027466 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -5.433635711669922 }, { "rank": 7, "tokenId": 70961, "text": " ब्लॉक", "score": -5.793137550354004 }, { "rank": 8, "tokenId": 112780, "text": "authorize", "score": -5.914745807647705 } ] }, "f32_reverse": { "winnerTokenId": 5088, "winnerText": "block", "winnerScore": 10.07981014251709, "winnerGap": 2.6596546173095703, "top": [ { "rank": 1, "tokenId": 5088, "text": "block", "score": 10.07981014251709 }, { "rank": 2, "tokenId": 8021, "text": "allow", "score": 7.4201555252075195 }, { "rank": 3, "tokenId": 27122, "text": "Allow", "score": 3.777431011199951 }, { "rank": 4, "tokenId": 10809, "text": "Block", "score": 2.3115546703338623 }, { "rank": 5, "tokenId": 205380, "text": "deny", "score": -1.3779047727584839 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -5.433635234832764 }, { "rank": 7, "tokenId": 70961, "text": " ब्लॉक", "score": -5.793138027191162 }, { "rank": 8, "tokenId": 112780, "text": "authorize", "score": -5.914754390716553 } ] }, "f32_tree64": { "winnerTokenId": 5088, "winnerText": "block", "winnerScore": 10.079811096191406, "winnerGap": 2.6596570014953613, "top": [ { "rank": 1, "tokenId": 5088, "text": "block", "score": 10.079811096191406 }, { "rank": 2, "tokenId": 8021, "text": "allow", "score": 7.420154094696045 }, { "rank": 3, "tokenId": 27122, "text": "Allow", "score": 3.7774300575256348 }, { "rank": 4, "tokenId": 10809, "text": "Block", "score": 2.3115530014038086 }, { "rank": 5, "tokenId": 205380, "text": "deny", "score": -1.377901554107666 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -5.4336347579956055 }, { "rank": 7, "tokenId": 70961, "text": " ब्लॉक", "score": -5.793130874633789 }, { "rank": 8, "tokenId": 112780, "text": "authorize", "score": -5.91474723815918 } ] }, "f16_forward": { "winnerTokenId": 5088, "winnerText": "block", "winnerScore": 10.0703125, "winnerGap": 2.6015625, "top": [ { "rank": 1, "tokenId": 5088, "text": "block", "score": 10.0703125 }, { "rank": 2, "tokenId": 8021, "text": "allow", "score": 7.46875 }, { "rank": 3, "tokenId": 27122, "text": "Allow", "score": 4.03515625 }, { "rank": 4, "tokenId": 10809, "text": "Block", "score": 1.337890625 }, { "rank": 5, "tokenId": 124174, "text": "ブロック", "score": -3.115234375 }, { "rank": 6, "tokenId": 11106, "text": "require", "score": -3.875 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -4.4453125 }, { "rank": 8, "tokenId": 2174, "text": " allow", "score": -5.14453125 } ] } }, "winnerTextsByMode": { "exact": "block", "f32_forward": "block", "f32_reverse": "block", "f32_tree64": "block", "f16_forward": "block" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "block" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "ssh-public-private-oneword-c", "text": "No explanation. One word only.\nA private SSH key should be public or private. It should be", "stableTopK": [ { "rank": 1, "tokenId": 29335, "text": "Private", "logit": 11.708465576171875, "prob": 0.9744182229042053 }, { "rank": 2, "tokenId": 7128, "text": "private", "logit": 7.6985626220703125, "prob": 0.01767122559249401 }, { "rank": 3, "tokenId": 3503, "text": "Key", "logit": 5.271033763885498, "prob": 0.0015595678705722094 }, { "rank": 4, "tokenId": 22186, "text": "Both", "logit": 5.1273956298828125, "prob": 0.0013508994597941637 }, { "rank": 5, "tokenId": 19118, "text": "Public", "logit": 4.985110282897949, "prob": 0.0011717346496880054 }, { "rank": 6, "tokenId": 30730, "text": "Secret", "logit": 4.972662925720215, "prob": 0.0011572401272132993 }, { "rank": 7, "tokenId": 2205, "text": "As", "logit": 3.5697531700134277, "prob": 0.00028454273706302047 }, { "rank": 8, "tokenId": 179175, "text": "PrivateKey", "logit": 3.3027193546295166, "prob": 0.00021785934222862124 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 29335, "winnerText": "Private", "winnerScore": 12.363989676672432, "winnerGap": 4.489429599220571, "top": [ { "rank": 1, "tokenId": 29335, "text": "Private", "score": 12.363989676672432 }, { "rank": 2, "tokenId": 7128, "text": "private", "score": 7.874560077451861 }, { "rank": 3, "tokenId": 3503, "text": "Key", "score": 5.3263024767404294 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 5.178213096046097 }, { "rank": 5, "tokenId": 19118, "text": "Public", "score": 5.031772478718679 }, { "rank": 6, "tokenId": 30730, "text": "Secret", "score": 5.018969758467506 }, { "rank": 7, "tokenId": 2205, "text": "As", "score": 3.586747151450517 }, { "rank": 8, "tokenId": 179175, "text": "PrivateKey", "score": 3.316157025904104 } ] }, "f32_forward": { "winnerTokenId": 29335, "winnerText": "Private", "winnerScore": 12.363993644714355, "winnerGap": 4.489431858062744, "top": [ { "rank": 1, "tokenId": 29335, "text": "Private", "score": 12.363993644714355 }, { "rank": 2, "tokenId": 7128, "text": "private", "score": 7.874561786651611 }, { "rank": 3, "tokenId": 3503, "text": "Key", "score": 5.32630729675293 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 5.178207874298096 }, { "rank": 5, "tokenId": 19118, "text": "Public", "score": 5.031772613525391 }, { "rank": 6, "tokenId": 30730, "text": "Secret", "score": 5.018962860107422 }, { "rank": 7, "tokenId": 2205, "text": "As", "score": 3.586747646331787 }, { "rank": 8, "tokenId": 179175, "text": "PrivateKey", "score": 3.3161563873291016 } ] }, "f32_reverse": { "winnerTokenId": 29335, "winnerText": "Private", "winnerScore": 12.36398696899414, "winnerGap": 4.489429950714111, "top": [ { "rank": 1, "tokenId": 29335, "text": "Private", "score": 12.36398696899414 }, { "rank": 2, "tokenId": 7128, "text": "private", "score": 7.874557018280029 }, { "rank": 3, "tokenId": 3503, "text": "Key", "score": 5.326302528381348 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 5.178211212158203 }, { "rank": 5, "tokenId": 19118, "text": "Public", "score": 5.031773567199707 }, { "rank": 6, "tokenId": 30730, "text": "Secret", "score": 5.0189714431762695 }, { "rank": 7, "tokenId": 2205, "text": "As", "score": 3.5867459774017334 }, { "rank": 8, "tokenId": 179175, "text": "PrivateKey", "score": 3.3161532878875732 } ] }, "f32_tree64": { "winnerTokenId": 29335, "winnerText": "Private", "winnerScore": 12.36398983001709, "winnerGap": 4.489429473876953, "top": [ { "rank": 1, "tokenId": 29335, "text": "Private", "score": 12.36398983001709 }, { "rank": 2, "tokenId": 7128, "text": "private", "score": 7.874560356140137 }, { "rank": 3, "tokenId": 3503, "text": "Key", "score": 5.326302528381348 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 5.178213596343994 }, { "rank": 5, "tokenId": 19118, "text": "Public", "score": 5.031772136688232 }, { "rank": 6, "tokenId": 30730, "text": "Secret", "score": 5.018970966339111 }, { "rank": 7, "tokenId": 2205, "text": "As", "score": 3.586747407913208 }, { "rank": 8, "tokenId": 179175, "text": "PrivateKey", "score": 3.3161568641662598 } ] }, "f16_forward": { "winnerTokenId": 29335, "winnerText": "Private", "winnerScore": 8.4140625, "winnerGap": 0.8359375, "top": [ { "rank": 1, "tokenId": 29335, "text": "Private", "score": 8.4140625 }, { "rank": 2, "tokenId": 7128, "text": "private", "score": 7.578125 }, { "rank": 3, "tokenId": 3503, "text": "Key", "score": 5.35546875 }, { "rank": 4, "tokenId": 22186, "text": "Both", "score": 5.17578125 }, { "rank": 5, "tokenId": 19118, "text": "Public", "score": 5.07421875 }, { "rank": 6, "tokenId": 2205, "text": "As", "score": 4.0078125 }, { "rank": 7, "tokenId": 179175, "text": "PrivateKey", "score": 3.345703125 }, { "rank": 8, "tokenId": 75064, "text": "Either", "score": 3.05859375 } ] } }, "winnerTextsByMode": { "exact": "Private", "f32_forward": "Private", "f32_reverse": "Private", "f32_tree64": "Private", "f16_forward": "Private" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Private" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "ssh-safe-unsafe-choice-oneword-d", "text": "Choose one word only.\nChoose exactly one word: safe or unsafe.\nSharing a private SSH key publicly is", "stableTopK": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "logit": 22.222652435302734, "prob": 0.9923855662345886 }, { "rank": 2, "tokenId": 1018, "text": "**", "logit": 17.264175415039062, "prob": 0.00697013596072793 }, { "rank": 3, "tokenId": 2805, "text": "Un", "logit": 14.50869369506836, "prob": 0.0004431499692145735 }, { "rank": 4, "tokenId": 8069, "text": "uns", "logit": 13.709501266479492, "prob": 0.00019928098481614143 }, { "rank": 5, "tokenId": 602, "text": "un", "logit": 8.821038246154785, "prob": 0.0000015011819414212368 }, { "rank": 6, "tokenId": 5213, "text": " **", "logit": 5.89174747467041, "prob": 8.021554265269515e-8 }, { "rank": 7, "tokenId": 193883, "text": "dangerous", "logit": 5.579184532165527, "prob": 5.8683252035507394e-8 }, { "rank": 8, "tokenId": 902, "text": "In", "logit": 4.858151435852051, "prob": 2.8534710594385615e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 28.56451346185966, "winnerGap": 8.894507533479008, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 28.56451346185966 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 19.670005928380654 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": 15.831084965204358 }, { "rank": 4, "tokenId": 8069, "text": "uns", "score": 14.804750401161542 }, { "rank": 5, "tokenId": 602, "text": "un", "score": 9.089307974976848 }, { "rank": 6, "tokenId": 5213, "text": " **", "score": 5.969296180854606 }, { "rank": 7, "tokenId": 193883, "text": "dangerous", "score": 5.644871479794499 }, { "rank": 8, "tokenId": 902, "text": "In", "score": 4.9012981669758915 } ] }, "f32_forward": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 28.564537048339844, "winnerGap": 8.894533157348633, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 28.564537048339844 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 19.67000389099121 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": 15.831088066101074 }, { "rank": 4, "tokenId": 8069, "text": "uns", "score": 14.804758071899414 }, { "rank": 5, "tokenId": 602, "text": "un", "score": 9.089317321777344 }, { "rank": 6, "tokenId": 5213, "text": " **", "score": 5.969296455383301 }, { "rank": 7, "tokenId": 193883, "text": "dangerous", "score": 5.644871234893799 }, { "rank": 8, "tokenId": 902, "text": "In", "score": 4.901298522949219 } ] }, "f32_reverse": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 28.564523696899414, "winnerGap": 8.894502639770508, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 28.564523696899414 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 19.670021057128906 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": 15.831076622009277 }, { "rank": 4, "tokenId": 8069, "text": "uns", "score": 14.804756164550781 }, { "rank": 5, "tokenId": 602, "text": "un", "score": 9.089310646057129 }, { "rank": 6, "tokenId": 5213, "text": " **", "score": 5.9692912101745605 }, { "rank": 7, "tokenId": 193883, "text": "dangerous", "score": 5.644868850708008 }, { "rank": 8, "tokenId": 902, "text": "In", "score": 4.901295185089111 } ] }, "f32_tree64": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 28.56451416015625, "winnerGap": 8.894508361816406, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 28.56451416015625 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 19.670005798339844 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": 15.831086158752441 }, { "rank": 4, "tokenId": 8069, "text": "uns", "score": 14.804750442504883 }, { "rank": 5, "tokenId": 602, "text": "un", "score": 9.089308738708496 }, { "rank": 6, "tokenId": 5213, "text": " **", "score": 5.969297409057617 }, { "rank": 7, "tokenId": 193883, "text": "dangerous", "score": 5.644871711730957 }, { "rank": 8, "tokenId": 902, "text": "In", "score": 4.901297569274902 } ] }, "f16_forward": { "winnerTokenId": 30908, "winnerText": "unsafe", "winnerScore": 28.609375, "winnerGap": 8.953125, "top": [ { "rank": 1, "tokenId": 30908, "text": "unsafe", "score": 28.609375 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 19.65625 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": 15.671875 }, { "rank": 4, "tokenId": 8069, "text": "uns", "score": 14.890625 }, { "rank": 5, "tokenId": 602, "text": "un", "score": 9.1015625 }, { "rank": 6, "tokenId": 5213, "text": " **", "score": 5.9453125 }, { "rank": 7, "tokenId": 193883, "text": "dangerous", "score": 5.671875 }, { "rank": 8, "tokenId": 902, "text": "In", "score": 4.94140625 } ] } }, "winnerTextsByMode": { "exact": "unsafe", "f32_forward": "unsafe", "f32_reverse": "unsafe", "f32_tree64": "unsafe", "f16_forward": "unsafe" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "unsafe" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "story-open-oneword-a", "text": "Answer with exactly one word only.\nOnce upon a", "stableTopK": [ { "rank": 1, "tokenId": 32214, "text": "Story", "logit": -9.255258560180664, "prob": 0.527366042137146 }, { "rank": 2, "tokenId": 50483, "text": "Snow", "logit": -9.878247261047363, "prob": 0.2828470468521118 }, { "rank": 3, "tokenId": 127134, "text": "Happ", "logit": -11.969647407531738, "prob": 0.034935589879751205 }, { "rank": 4, "tokenId": 165453, "text": "Novel", "logit": -12.531494140625, "prob": 0.01991870626807213 }, { "rank": 5, "tokenId": 236811, "text": "F", "logit": -13.09473991394043, "prob": 0.01134087610989809 }, { "rank": 6, "tokenId": 138409, "text": "Rabbit", "logit": -13.276202201843262, "prob": 0.009458854794502258 }, { "rank": 7, "tokenId": 236774, "text": "T", "logit": -13.343713760375977, "prob": 0.008841350674629211 }, { "rank": 8, "tokenId": 27426, "text": "Begin", "logit": -13.530706405639648, "prob": 0.007333457004278898 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 32214, "winnerText": "Story", "winnerScore": -9.566889335783351, "winnerGap": 0.6935602802342942, "top": [ { "rank": 1, "tokenId": 32214, "text": "Story", "score": -9.566889335783351 }, { "rank": 2, "tokenId": 50483, "text": "Snow", "score": -10.260449616017645 }, { "rank": 3, "tokenId": 127134, "text": "Happ", "score": -12.673350992664233 }, { "rank": 4, "tokenId": 165453, "text": "Novel", "score": -13.34767917596757 }, { "rank": 5, "tokenId": 236811, "text": "F", "score": -14.036643235583767 }, { "rank": 6, "tokenId": 138409, "text": "Rabbit", "score": -14.26155031288458 }, { "rank": 7, "tokenId": 236774, "text": "T", "score": -14.345609892135085 }, { "rank": 8, "tokenId": 27426, "text": "Begin", "score": -14.57953444314694 } ] }, "f32_forward": { "winnerTokenId": 32214, "winnerText": "Story", "winnerScore": -9.566888809204102, "winnerGap": 0.6935586929321289, "top": [ { "rank": 1, "tokenId": 32214, "text": "Story", "score": -9.566888809204102 }, { "rank": 2, "tokenId": 50483, "text": "Snow", "score": -10.26044750213623 }, { "rank": 3, "tokenId": 127134, "text": "Happ", "score": -12.673348426818848 }, { "rank": 4, "tokenId": 165453, "text": "Novel", "score": -13.34766960144043 }, { "rank": 5, "tokenId": 236811, "text": "F", "score": -14.036643981933594 }, { "rank": 6, "tokenId": 138409, "text": "Rabbit", "score": -14.261557579040527 }, { "rank": 7, "tokenId": 236774, "text": "T", "score": -14.345597267150879 }, { "rank": 8, "tokenId": 27426, "text": "Begin", "score": -14.579534530639648 } ] }, "f32_reverse": { "winnerTokenId": 32214, "winnerText": "Story", "winnerScore": -9.566880226135254, "winnerGap": 0.6935577392578125, "top": [ { "rank": 1, "tokenId": 32214, "text": "Story", "score": -9.566880226135254 }, { "rank": 2, "tokenId": 50483, "text": "Snow", "score": -10.260437965393066 }, { "rank": 3, "tokenId": 127134, "text": "Happ", "score": -12.673340797424316 }, { "rank": 4, "tokenId": 165453, "text": "Novel", "score": -13.347699165344238 }, { "rank": 5, "tokenId": 236811, "text": "F", "score": -14.03663158416748 }, { "rank": 6, "tokenId": 138409, "text": "Rabbit", "score": -14.261560440063477 }, { "rank": 7, "tokenId": 236774, "text": "T", "score": -14.345609664916992 }, { "rank": 8, "tokenId": 27426, "text": "Begin", "score": -14.579529762268066 } ] }, "f32_tree64": { "winnerTokenId": 32214, "winnerText": "Story", "winnerScore": -9.566887855529785, "winnerGap": 0.6935606002807617, "top": [ { "rank": 1, "tokenId": 32214, "text": "Story", "score": -9.566887855529785 }, { "rank": 2, "tokenId": 50483, "text": "Snow", "score": -10.260448455810547 }, { "rank": 3, "tokenId": 127134, "text": "Happ", "score": -12.673352241516113 }, { "rank": 4, "tokenId": 165453, "text": "Novel", "score": -13.347678184509277 }, { "rank": 5, "tokenId": 236811, "text": "F", "score": -14.036643981933594 }, { "rank": 6, "tokenId": 138409, "text": "Rabbit", "score": -14.26154899597168 }, { "rank": 7, "tokenId": 236774, "text": "T", "score": -14.345610618591309 }, { "rank": 8, "tokenId": 27426, "text": "Begin", "score": -14.579532623291016 } ] }, "f16_forward": { "winnerTokenId": 32214, "winnerText": "Story", "winnerScore": -9.5859375, "winnerGap": 0.7734375, "top": [ { "rank": 1, "tokenId": 32214, "text": "Story", "score": -9.5859375 }, { "rank": 2, "tokenId": 50483, "text": "Snow", "score": -10.359375 }, { "rank": 3, "tokenId": 76795, "text": "Wonder", "score": -11.984375 }, { "rank": 4, "tokenId": 127134, "text": "Happ", "score": -12.640625 }, { "rank": 5, "tokenId": 236811, "text": "F", "score": -14.078125 }, { "rank": 6, "tokenId": 236774, "text": "T", "score": -14.25 }, { "rank": 7, "tokenId": 27426, "text": "Begin", "score": -14.421875 }, { "rank": 8, "tokenId": 165453, "text": "Novel", "score": -14.421875 } ] } }, "winnerTextsByMode": { "exact": "Story", "f32_forward": "Story", "f32_reverse": "Story", "f32_tree64": "Story", "f16_forward": "Story" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Story" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "su-mushroom-oneword-b", "text": "Respond with one word only.\nAnswer with exactly one word: safe or unsafe. Question: Eating a wild mushroom you cannot identify is Answer:", "stableTopK": [ { "rank": 1, "tokenId": 2805, "text": "Un", "logit": 12.777615547180176, "prob": 0.8979595899581909 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "logit": 10.60285472869873, "prob": 0.10203993320465088 }, { "rank": 3, "tokenId": 602, "text": "un", "logit": -2.0242156982421875, "prob": 3.34890728481696e-7 }, { "rank": 4, "tokenId": 38699, "text": "Una", "logit": -3.2182493209838867, "prob": 1.0147076068278693e-7 }, { "rank": 5, "tokenId": 39316, "text": "Safe", "logit": -4.9110589027404785, "prob": 1.8670807122589395e-8 }, { "rank": 6, "tokenId": 1018, "text": "**", "logit": -5.325978755950928, "prob": 1.233007385081919e-8 }, { "rank": 7, "tokenId": 46308, "text": "Une", "logit": -6.107019424438477, "prob": 5.646301204365045e-9 }, { "rank": 8, "tokenId": 8069, "text": "uns", "logit": -6.152733325958252, "prob": 5.39399769294846e-9 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 13.647075501226176, "winnerGap": 2.566401227944265, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 13.647075501226176 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": 11.080674273281911 }, { "rank": 3, "tokenId": 602, "text": "un", "score": -2.027296593924717 }, { "rank": 4, "tokenId": 38699, "text": "Una", "score": -3.2306788420170154 }, { "rank": 5, "tokenId": 39316, "text": "Safe", "score": -4.955646944348018 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -5.383013428332223 }, { "rank": 7, "tokenId": 46308, "text": "Une", "score": -6.193538012849743 }, { "rank": 8, "tokenId": 8069, "text": "uns", "score": -6.2412428738999175 } ] }, "f32_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 13.647087097167969, "winnerGap": 2.566408157348633, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 13.647087097167969 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": 11.080678939819336 }, { "rank": 3, "tokenId": 602, "text": "un", "score": -2.027296304702759 }, { "rank": 4, "tokenId": 38699, "text": "Una", "score": -3.230687379837036 }, { "rank": 5, "tokenId": 39316, "text": "Safe", "score": -4.955643653869629 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -5.3830156326293945 }, { "rank": 7, "tokenId": 46308, "text": "Une", "score": -6.193537712097168 }, { "rank": 8, "tokenId": 8069, "text": "uns", "score": -6.241252422332764 } ] }, "f32_reverse": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 13.647072792053223, "winnerGap": 2.566399574279785, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 13.647072792053223 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": 11.080673217773438 }, { "rank": 3, "tokenId": 602, "text": "un", "score": -2.027299165725708 }, { "rank": 4, "tokenId": 38699, "text": "Una", "score": -3.2306835651397705 }, { "rank": 5, "tokenId": 39316, "text": "Safe", "score": -4.955641269683838 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -5.383021354675293 }, { "rank": 7, "tokenId": 46308, "text": "Une", "score": -6.19354772567749 }, { "rank": 8, "tokenId": 8069, "text": "uns", "score": -6.241238594055176 } ] }, "f32_tree64": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 13.647074699401855, "winnerGap": 2.5664024353027344, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 13.647074699401855 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": 11.080672264099121 }, { "rank": 3, "tokenId": 602, "text": "un", "score": -2.0272984504699707 }, { "rank": 4, "tokenId": 38699, "text": "Una", "score": -3.230679988861084 }, { "rank": 5, "tokenId": 39316, "text": "Safe", "score": -4.955647945404053 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": -5.3830132484436035 }, { "rank": 7, "tokenId": 46308, "text": "Une", "score": -6.193536758422852 }, { "rank": 8, "tokenId": 8069, "text": "uns", "score": -6.241243362426758 } ] }, "f16_forward": { "winnerTokenId": 2805, "winnerText": "Un", "winnerScore": 13.6328125, "winnerGap": 2.609375, "top": [ { "rank": 1, "tokenId": 2805, "text": "Un", "score": 13.6328125 }, { "rank": 2, "tokenId": 30908, "text": "unsafe", "score": 11.0234375 }, { "rank": 3, "tokenId": 602, "text": "un", "score": -1.9990234375 }, { "rank": 4, "tokenId": 38699, "text": "Una", "score": -3.162109375 }, { "rank": 5, "tokenId": 1018, "text": "**", "score": -4.36328125 }, { "rank": 6, "tokenId": 39316, "text": "Safe", "score": -4.98046875 }, { "rank": 7, "tokenId": 46308, "text": "Une", "score": -6.140625 }, { "rank": 8, "tokenId": 25793, "text": " unsafe", "score": -6.5390625 } ] } }, "winnerTextsByMode": { "exact": "Un", "f32_forward": "Un", "f32_reverse": "Un", "f32_tree64": "Un", "f16_forward": "Un" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Un" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tech-ai-jobs-oneword-c", "text": "No explanation. One word only.\nAI will create more jobs than it destroys — agree or disagree?", "stableTopK": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "logit": 10.644285202026367, "prob": 0.9998353123664856 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "logit": 1.2203545570373535, "prob": 0.00008075462392298505 }, { "rank": 3, "tokenId": 233950, "text": "Disagree", "logit": -0.2831723093986511, "prob": 0.000017955353541765362 }, { "rank": 4, "tokenId": 43868, "text": "Nu", "logit": -0.521476686000824, "prob": 0.000014148151421977673 }, { "rank": 5, "tokenId": 11585, "text": "Ag", "logit": -1.2788223028182983, "prob": 0.000006634201781707816 }, { "rank": 6, "tokenId": 131999, "text": "Agreement", "logit": -1.552899718284607, "prob": 0.000005043805686000269 }, { "rank": 7, "tokenId": 83172, "text": "Probably", "logit": -1.7312391996383667, "prob": 0.000004219942184136016 }, { "rank": 8, "tokenId": 115863, "text": "Analyze", "logit": -1.800592303276062, "prob": 0.00000393719437852269 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 208903, "winnerText": "Agree", "winnerScore": 11.128058417158854, "winnerGap": 9.907030591061464, "top": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "score": 11.128058417158854 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 1.2210278260973908 }, { "rank": 3, "tokenId": 233950, "text": "Disagree", "score": -0.2831844446319334 }, { "rank": 4, "tokenId": 43868, "text": "Nu", "score": -0.5215281289320055 }, { "rank": 5, "tokenId": 11585, "text": "Ag", "score": -1.2795966679663522 }, { "rank": 6, "tokenId": 131999, "text": "Agreement", "score": -1.554290124887654 }, { "rank": 7, "tokenId": 83172, "text": "Probably", "score": -1.7331658105322099 }, { "rank": 8, "tokenId": 115863, "text": "Analyze", "score": -1.802760571958859 } ] }, "f32_forward": { "winnerTokenId": 208903, "winnerText": "Agree", "winnerScore": 11.128060340881348, "winnerGap": 9.90703547000885, "top": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "score": 11.128060340881348 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 1.2210248708724976 }, { "rank": 3, "tokenId": 233950, "text": "Disagree", "score": -0.283185213804245 }, { "rank": 4, "tokenId": 43868, "text": "Nu", "score": -0.5215258002281189 }, { "rank": 5, "tokenId": 11585, "text": "Ag", "score": -1.2795957326889038 }, { "rank": 6, "tokenId": 131999, "text": "Agreement", "score": -1.554287314414978 }, { "rank": 7, "tokenId": 83172, "text": "Probably", "score": -1.733169674873352 }, { "rank": 8, "tokenId": 115863, "text": "Analyze", "score": -1.8027589321136475 } ] }, "f32_reverse": { "winnerTokenId": 208903, "winnerText": "Agree", "winnerScore": 11.128060340881348, "winnerGap": 9.907031416893005, "top": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "score": 11.128060340881348 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 1.2210289239883423 }, { "rank": 3, "tokenId": 233950, "text": "Disagree", "score": -0.2831845283508301 }, { "rank": 4, "tokenId": 43868, "text": "Nu", "score": -0.5215225219726562 }, { "rank": 5, "tokenId": 11585, "text": "Ag", "score": -1.2795933485031128 }, { "rank": 6, "tokenId": 131999, "text": "Agreement", "score": -1.5542864799499512 }, { "rank": 7, "tokenId": 83172, "text": "Probably", "score": -1.7331688404083252 }, { "rank": 8, "tokenId": 115863, "text": "Analyze", "score": -1.8027595281600952 } ] }, "f32_tree64": { "winnerTokenId": 208903, "winnerText": "Agree", "winnerScore": 11.128058433532715, "winnerGap": 9.907030940055847, "top": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "score": 11.128058433532715 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 1.2210274934768677 }, { "rank": 3, "tokenId": 233950, "text": "Disagree", "score": -0.28318357467651367 }, { "rank": 4, "tokenId": 43868, "text": "Nu", "score": -0.5215277671813965 }, { "rank": 5, "tokenId": 11585, "text": "Ag", "score": -1.2795968055725098 }, { "rank": 6, "tokenId": 131999, "text": "Agreement", "score": -1.5542911291122437 }, { "rank": 7, "tokenId": 83172, "text": "Probably", "score": -1.7331650257110596 }, { "rank": 8, "tokenId": 115863, "text": "Analyze", "score": -1.8027606010437012 } ] }, "f16_forward": { "winnerTokenId": 208903, "winnerText": "Agree", "winnerScore": 7.09375, "winnerGap": 5.859375, "top": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "score": 7.09375 }, { "rank": 2, "tokenId": 179306, "text": "Depends", "score": 1.234375 }, { "rank": 3, "tokenId": 233950, "text": "Disagree", "score": -0.290771484375 }, { "rank": 4, "tokenId": 43868, "text": "Nu", "score": -0.5078125 }, { "rank": 5, "tokenId": 69626, "text": "Possible", "score": -0.83251953125 }, { "rank": 6, "tokenId": 11585, "text": "Ag", "score": -1.2841796875 }, { "rank": 7, "tokenId": 131999, "text": "Agreement", "score": -1.5107421875 }, { "rank": 8, "tokenId": 30092, "text": "Could", "score": -1.685546875 } ] } }, "winnerTextsByMode": { "exact": "Agree", "f32_forward": "Agree", "f32_reverse": "Agree", "f32_tree64": "Agree", "f16_forward": "Agree" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Agree" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tech-crypto-future-oneword-d", "text": "Choose one word only.\nCryptocurrency is the future of money or a speculative bubble. It is", "stableTopK": [ { "rank": 1, "tokenId": 1018, "text": "**", "logit": 15.761319160461426, "prob": 0.6806269884109497 }, { "rank": 2, "tokenId": 36177, "text": "Future", "logit": 14.269914627075195, "prob": 0.15317942202091217 }, { "rank": 3, "tokenId": 9798, "text": "Spec", "logit": 13.72491455078125, "prob": 0.08881981670856476 }, { "rank": 4, "tokenId": 2805, "text": "Un", "logit": 13.008390426635742, "prob": 0.04338378459215164 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "logit": 11.58228874206543, "prob": 0.010422677733004093 }, { "rank": 6, "tokenId": 13811, "text": "Vol", "logit": 11.14636516571045, "prob": 0.006740003824234009 }, { "rank": 7, "tokenId": 5690, "text": "spec", "logit": 10.57159423828125, "prob": 0.00379350152797997 }, { "rank": 8, "tokenId": 1584, "text": "Cont", "logit": 10.125003814697266, "prob": 0.0024271048605442047 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 1018, "winnerText": "**", "winnerScore": 17.512153390904544, "winnerGap": 1.991203342995071, "top": [ { "rank": 1, "tokenId": 1018, "text": "**", "score": 17.512153390904544 }, { "rank": 2, "tokenId": 36177, "text": "Future", "score": 15.520950047909473 }, { "rank": 3, "tokenId": 9798, "text": "Spec", "score": 14.824240203053064 }, { "rank": 4, "tokenId": 2805, "text": "Un", "score": 13.93013278664824 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 12.215426644333874 }, { "rank": 6, "tokenId": 13811, "text": "Vol", "score": 11.706448428628066 }, { "rank": 7, "tokenId": 5690, "text": "spec", "score": 11.044980440696108 }, { "rank": 8, "tokenId": 1584, "text": "Cont", "score": 10.538056447209895 } ] }, "f32_forward": { "winnerTokenId": 1018, "winnerText": "**", "winnerScore": 17.512165069580078, "winnerGap": 1.9912176132202148, "top": [ { "rank": 1, "tokenId": 1018, "text": "**", "score": 17.512165069580078 }, { "rank": 2, "tokenId": 36177, "text": "Future", "score": 15.520947456359863 }, { "rank": 3, "tokenId": 9798, "text": "Spec", "score": 14.82424545288086 }, { "rank": 4, "tokenId": 2805, "text": "Un", "score": 13.930130958557129 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 12.215415954589844 }, { "rank": 6, "tokenId": 13811, "text": "Vol", "score": 11.706453323364258 }, { "rank": 7, "tokenId": 5690, "text": "spec", "score": 11.044988632202148 }, { "rank": 8, "tokenId": 1584, "text": "Cont", "score": 10.538052558898926 } ] }, "f32_reverse": { "winnerTokenId": 1018, "winnerText": "**", "winnerScore": 17.51218032836914, "winnerGap": 1.9912481307983398, "top": [ { "rank": 1, "tokenId": 1018, "text": "**", "score": 17.51218032836914 }, { "rank": 2, "tokenId": 36177, "text": "Future", "score": 15.5209321975708 }, { "rank": 3, "tokenId": 9798, "text": "Spec", "score": 14.824234008789062 }, { "rank": 4, "tokenId": 2805, "text": "Un", "score": 13.930130004882812 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 12.21543025970459 }, { "rank": 6, "tokenId": 13811, "text": "Vol", "score": 11.706448554992676 }, { "rank": 7, "tokenId": 5690, "text": "spec", "score": 11.044983863830566 }, { "rank": 8, "tokenId": 1584, "text": "Cont", "score": 10.538053512573242 } ] }, "f32_tree64": { "winnerTokenId": 1018, "winnerText": "**", "winnerScore": 17.51215362548828, "winnerGap": 1.9912023544311523, "top": [ { "rank": 1, "tokenId": 1018, "text": "**", "score": 17.51215362548828 }, { "rank": 2, "tokenId": 36177, "text": "Future", "score": 15.520951271057129 }, { "rank": 3, "tokenId": 9798, "text": "Spec", "score": 14.824239730834961 }, { "rank": 4, "tokenId": 2805, "text": "Un", "score": 13.930133819580078 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 12.215426445007324 }, { "rank": 6, "tokenId": 13811, "text": "Vol", "score": 11.70644760131836 }, { "rank": 7, "tokenId": 5690, "text": "spec", "score": 11.0449800491333 }, { "rank": 8, "tokenId": 1584, "text": "Cont", "score": 10.538056373596191 } ] }, "f16_forward": { "winnerTokenId": 1018, "winnerText": "**", "winnerScore": 17.53125, "winnerGap": 1.9765625, "top": [ { "rank": 1, "tokenId": 1018, "text": "**", "score": 17.53125 }, { "rank": 2, "tokenId": 36177, "text": "Future", "score": 15.5546875 }, { "rank": 3, "tokenId": 9798, "text": "Spec", "score": 14.9296875 }, { "rank": 4, "tokenId": 2805, "text": "Un", "score": 13.9609375 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 12.28125 }, { "rank": 6, "tokenId": 13811, "text": "Vol", "score": 11.8203125 }, { "rank": 7, "tokenId": 5690, "text": "spec", "score": 11.203125 }, { "rank": 8, "tokenId": 39979, "text": "future", "score": 10.546875 } ] } }, "winnerTextsByMode": { "exact": "**", "f32_forward": "**", "f32_reverse": "**", "f32_tree64": "**", "f16_forward": "**" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "**" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tech-gmo-safe-oneword-a", "text": "Answer with exactly one word only.\nGMO food is safe for human consumption — true, false, or debated?", "stableTopK": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "logit": 10.468185424804688, "prob": 0.9998965263366699 }, { "rank": 2, "tokenId": 4339, "text": "True", "logit": 1.116407036781311, "prob": 0.00008680191240273416 }, { "rank": 3, "tokenId": 53417, "text": "deb", "logit": -0.6500428318977356, "prob": 0.000014837813068879768 }, { "rank": 4, "tokenId": 61569, "text": "Safety", "logit": -4.871776580810547, "prob": 2.1771792546587676e-7 }, { "rank": 5, "tokenId": 3771, "text": "No", "logit": -5.0692667961120605, "prob": 1.7870030433186912e-7 }, { "rank": 6, "tokenId": 10784, "text": "Yes", "logit": -5.666095733642578, "prob": 9.838429804176485e-8 }, { "rank": 7, "tokenId": 43868, "text": "Nu", "logit": -6.213751792907715, "prob": 5.689600612868162e-8 }, { "rank": 8, "tokenId": 9277, "text": "False", "logit": -6.232957363128662, "prob": 5.5813710986285514e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 10.927068767457824, "winnerGap": 9.810146545905907, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 10.927068767457824 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 1.116922221551917 }, { "rank": 3, "tokenId": 53417, "text": "deb", "score": -0.650139592026683 }, { "rank": 4, "tokenId": 61569, "text": "Safety", "score": -4.915294242544789 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": -5.118359659263021 }, { "rank": 6, "tokenId": 10784, "text": "Yes", "score": -5.734947413786273 }, { "rank": 7, "tokenId": 43868, "text": "Nu", "score": -6.304970218396704 }, { "rank": 8, "tokenId": 9277, "text": "False", "score": -6.325036888202973 } ] }, "f32_forward": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 10.927068710327148, "winnerGap": 9.810142040252686, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 10.927068710327148 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 1.116926670074463 }, { "rank": 3, "tokenId": 53417, "text": "deb", "score": -0.6501346230506897 }, { "rank": 4, "tokenId": 61569, "text": "Safety", "score": -4.915294647216797 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": -5.118360996246338 }, { "rank": 6, "tokenId": 10784, "text": "Yes", "score": -5.734946250915527 }, { "rank": 7, "tokenId": 43868, "text": "Nu", "score": -6.304969310760498 }, { "rank": 8, "tokenId": 9277, "text": "False", "score": -6.325044631958008 } ] }, "f32_reverse": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 10.927071571350098, "winnerGap": 9.81014633178711, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 10.927071571350098 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 1.1169252395629883 }, { "rank": 3, "tokenId": 53417, "text": "deb", "score": -0.6501380205154419 }, { "rank": 4, "tokenId": 61569, "text": "Safety", "score": -4.915279865264893 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": -5.118358135223389 }, { "rank": 6, "tokenId": 10784, "text": "Yes", "score": -5.734957695007324 }, { "rank": 7, "tokenId": 43868, "text": "Nu", "score": -6.3049750328063965 }, { "rank": 8, "tokenId": 9277, "text": "False", "score": -6.325049877166748 } ] }, "f32_tree64": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 10.927069664001465, "winnerGap": 9.810146570205688, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 10.927069664001465 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 1.1169230937957764 }, { "rank": 3, "tokenId": 53417, "text": "deb", "score": -0.6501390933990479 }, { "rank": 4, "tokenId": 61569, "text": "Safety", "score": -4.915294647216797 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": -5.118358135223389 }, { "rank": 6, "tokenId": 10784, "text": "Yes", "score": -5.734947681427002 }, { "rank": 7, "tokenId": 43868, "text": "Nu", "score": -6.304970741271973 }, { "rank": 8, "tokenId": 9277, "text": "False", "score": -6.325037002563477 } ] }, "f16_forward": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 10.7421875, "winnerGap": 10.64874267578125, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 10.7421875 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 0.09344482421875 }, { "rank": 3, "tokenId": 53417, "text": "deb", "score": -0.60546875 }, { "rank": 4, "tokenId": 118971, "text": "Proven", "score": -3.921875 }, { "rank": 5, "tokenId": 114214, "text": "Resolved", "score": -3.92578125 }, { "rank": 6, "tokenId": 4348, "text": "Not", "score": -4.3984375 }, { "rank": 7, "tokenId": 61569, "text": "Safety", "score": -4.91015625 }, { "rank": 8, "tokenId": 3771, "text": "No", "score": -5.16015625 } ] } }, "winnerTextsByMode": { "exact": "Deb", "f32_forward": "Deb", "f32_reverse": "Deb", "f32_tree64": "Deb", "f16_forward": "Deb" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Deb" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tech-nuclear-energy-oneword-b", "text": "Respond with one word only.\nNuclear energy is the solution to climate change — agree or disagree?", "stableTopK": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "logit": 3.7028331756591797, "prob": 0.865144670009613 }, { "rank": 2, "tokenId": 2805, "text": "Un", "logit": 1.167593002319336, "prob": 0.06855638325214386 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "logit": -0.2856840491294861, "prob": 0.01602867804467678 }, { "rank": 4, "tokenId": 83042, "text": "Partial", "logit": -0.3088461756706238, "prob": 0.015661686658859253 }, { "rank": 5, "tokenId": 212551, "text": "Insufficient", "logit": -0.6391831040382385, "prob": 0.011255765333771706 }, { "rank": 6, "tokenId": 3771, "text": "No", "logit": -1.5315600633621216, "prob": 0.004611270967870951 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "logit": -1.779433012008667, "prob": 0.0035989084281027317 }, { "rank": 8, "tokenId": 179306, "text": "Depends", "logit": -1.8002508878707886, "prob": 0.0035247611813247204 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 3.7218113504729358, "winnerGap": 2.553627262279657, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 3.7218113504729358 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 1.1681840881932786 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": -0.2856953981236998 }, { "rank": 4, "tokenId": 83042, "text": "Partial", "score": -0.30885881378816293 }, { "rank": 5, "tokenId": 212551, "text": "Insufficient", "score": -0.6392796228812223 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": -1.5328933885520755 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "score": -1.781521171644434 }, { "rank": 8, "tokenId": 179306, "text": "Depends", "score": -1.802417495312401 } ] }, "f32_forward": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 3.721808671951294, "winnerGap": 2.5536218881607056, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 3.721808671951294 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 1.1681867837905884 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": -0.2856943607330322 }, { "rank": 4, "tokenId": 83042, "text": "Partial", "score": -0.30885836482048035 }, { "rank": 5, "tokenId": 212551, "text": "Insufficient", "score": -0.6392788887023926 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": -1.5328941345214844 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "score": -1.7815200090408325 }, { "rank": 8, "tokenId": 179306, "text": "Depends", "score": -1.8024202585220337 } ] }, "f32_reverse": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 3.7218127250671387, "winnerGap": 2.553627133369446, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 3.7218127250671387 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 1.1681855916976929 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": -0.2856948971748352 }, { "rank": 4, "tokenId": 83042, "text": "Partial", "score": -0.30886051058769226 }, { "rank": 5, "tokenId": 212551, "text": "Insufficient", "score": -0.6392799019813538 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": -1.5328935384750366 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "score": -1.781523585319519 }, { "rank": 8, "tokenId": 179306, "text": "Depends", "score": -1.8024154901504517 } ] }, "f32_tree64": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 3.721810817718506, "winnerGap": 2.5536270141601562, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 3.721810817718506 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 1.1681838035583496 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": -0.28569579124450684 }, { "rank": 4, "tokenId": 83042, "text": "Partial", "score": -0.3088587522506714 }, { "rank": 5, "tokenId": 212551, "text": "Insufficient", "score": -0.6392790079116821 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": -1.532893180847168 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "score": -1.7815214395523071 }, { "rank": 8, "tokenId": 179306, "text": "Depends", "score": -1.8024177551269531 } ] }, "f16_forward": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 3.71875, "winnerGap": 2.5341796875, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 3.71875 }, { "rank": 2, "tokenId": 2805, "text": "Un", "score": 1.1845703125 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": -0.2958984375 }, { "rank": 4, "tokenId": 212551, "text": "Insufficient", "score": -0.66552734375 }, { "rank": 5, "tokenId": 55144, "text": "Amb", "score": -0.80419921875 }, { "rank": 6, "tokenId": 83042, "text": "Partial", "score": -1.2861328125 }, { "rank": 7, "tokenId": 3771, "text": "No", "score": -1.5341796875 }, { "rank": 8, "tokenId": 179306, "text": "Depends", "score": -1.8212890625 } ] } }, "winnerTextsByMode": { "exact": "Deb", "f32_forward": "Deb", "f32_reverse": "Deb", "f32_tree64": "Deb", "f16_forward": "Deb" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Deb" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tech-quantum-hype-oneword-c", "text": "No explanation. One word only.\nQuantum computing is overhyped or underestimated. It is", "stableTopK": [ { "rank": 1, "tokenId": 18385, "text": "Transform", "logit": 11.247525215148926, "prob": 0.7159176468849182 }, { "rank": 2, "tokenId": 20235, "text": "Real", "logit": 9.66016674041748, "prob": 0.14638009667396545 }, { "rank": 3, "tokenId": 163243, "text": "Revolution", "logit": 8.8014554977417, "prob": 0.06202239170670509 }, { "rank": 4, "tokenId": 211284, "text": "Reality", "logit": 8.168282508850098, "prob": 0.032927967607975006 }, { "rank": 5, "tokenId": 61117, "text": "Emer", "logit": 6.922287940979004, "prob": 0.009471883997321129 }, { "rank": 6, "tokenId": 104884, "text": "Potential", "logit": 6.350091457366943, "prob": 0.005344838835299015 }, { "rank": 7, "tokenId": 71869, "text": "Cru", "logit": 5.872003078460693, "prob": 0.003313625929877162 }, { "rank": 8, "tokenId": 33071, "text": "Pot", "logit": 5.732580184936523, "prob": 0.0028823907487094402 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 18385, "winnerText": "Transform", "winnerScore": 11.823980301654858, "winnerGap": 1.8074852504532277, "top": [ { "rank": 1, "tokenId": 18385, "text": "Transform", "score": 11.823980301654858 }, { "rank": 2, "tokenId": 20235, "text": "Real", "score": 10.01649505120163 }, { "rank": 3, "tokenId": 163243, "text": "Revolution", "score": 9.067879110157328 }, { "rank": 4, "tokenId": 211284, "text": "Reality", "score": 8.379621871124968 }, { "rank": 5, "tokenId": 61117, "text": "Emer", "score": 7.049218943840685 }, { "rank": 6, "tokenId": 104884, "text": "Potential", "score": 6.4475625045012634 }, { "rank": 7, "tokenId": 71869, "text": "Cru", "score": 5.948762694489419 }, { "rank": 8, "tokenId": 33071, "text": "Pot", "score": 5.803922212572516 } ] }, "f32_forward": { "winnerTokenId": 18385, "winnerText": "Transform", "winnerScore": 11.823991775512695, "winnerGap": 1.8074951171875, "top": [ { "rank": 1, "tokenId": 18385, "text": "Transform", "score": 11.823991775512695 }, { "rank": 2, "tokenId": 20235, "text": "Real", "score": 10.016496658325195 }, { "rank": 3, "tokenId": 163243, "text": "Revolution", "score": 9.06788158416748 }, { "rank": 4, "tokenId": 211284, "text": "Reality", "score": 8.379617691040039 }, { "rank": 5, "tokenId": 61117, "text": "Emer", "score": 7.049213886260986 }, { "rank": 6, "tokenId": 104884, "text": "Potential", "score": 6.447560787200928 }, { "rank": 7, "tokenId": 71869, "text": "Cru", "score": 5.948757171630859 }, { "rank": 8, "tokenId": 33071, "text": "Pot", "score": 5.803921222686768 } ] }, "f32_reverse": { "winnerTokenId": 18385, "winnerText": "Transform", "winnerScore": 11.823980331420898, "winnerGap": 1.8074884414672852, "top": [ { "rank": 1, "tokenId": 18385, "text": "Transform", "score": 11.823980331420898 }, { "rank": 2, "tokenId": 20235, "text": "Real", "score": 10.016491889953613 }, { "rank": 3, "tokenId": 163243, "text": "Revolution", "score": 9.067882537841797 }, { "rank": 4, "tokenId": 211284, "text": "Reality", "score": 8.379612922668457 }, { "rank": 5, "tokenId": 61117, "text": "Emer", "score": 7.049220085144043 }, { "rank": 6, "tokenId": 104884, "text": "Potential", "score": 6.447562217712402 }, { "rank": 7, "tokenId": 71869, "text": "Cru", "score": 5.948761940002441 }, { "rank": 8, "tokenId": 33071, "text": "Pot", "score": 5.803926467895508 } ] }, "f32_tree64": { "winnerTokenId": 18385, "winnerText": "Transform", "winnerScore": 11.823980331420898, "winnerGap": 1.807485580444336, "top": [ { "rank": 1, "tokenId": 18385, "text": "Transform", "score": 11.823980331420898 }, { "rank": 2, "tokenId": 20235, "text": "Real", "score": 10.016494750976562 }, { "rank": 3, "tokenId": 163243, "text": "Revolution", "score": 9.067878723144531 }, { "rank": 4, "tokenId": 211284, "text": "Reality", "score": 8.379621505737305 }, { "rank": 5, "tokenId": 61117, "text": "Emer", "score": 7.049219608306885 }, { "rank": 6, "tokenId": 104884, "text": "Potential", "score": 6.447562217712402 }, { "rank": 7, "tokenId": 71869, "text": "Cru", "score": 5.948762893676758 }, { "rank": 8, "tokenId": 33071, "text": "Pot", "score": 5.803922653198242 } ] }, "f16_forward": { "winnerTokenId": 18385, "winnerText": "Transform", "winnerScore": 11.8984375, "winnerGap": 1.8984375, "top": [ { "rank": 1, "tokenId": 18385, "text": "Transform", "score": 11.8984375 }, { "rank": 2, "tokenId": 20235, "text": "Real", "score": 10 }, { "rank": 3, "tokenId": 163243, "text": "Revolution", "score": 9.078125 }, { "rank": 4, "tokenId": 211284, "text": "Reality", "score": 8.5546875 }, { "rank": 5, "tokenId": 61117, "text": "Emer", "score": 6.9921875 }, { "rank": 6, "tokenId": 104884, "text": "Potential", "score": 6.453125 }, { "rank": 7, "tokenId": 71869, "text": "Cru", "score": 6.01171875 }, { "rank": 8, "tokenId": 33071, "text": "Pot", "score": 5.94140625 } ] } }, "winnerTextsByMode": { "exact": "Transform", "f32_forward": "Transform", "f32_reverse": "Transform", "f32_tree64": "Transform", "f16_forward": "Transform" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Transform" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tech-remote-work-oneword-d", "text": "Choose one word only.\nRemote work is more or less productive than office work. It is", "stableTopK": [ { "rank": 1, "tokenId": 19204, "text": "Variable", "logit": 7.859825611114502, "prob": 0.4585784673690796 }, { "rank": 2, "tokenId": 21288, "text": "Subject", "logit": 7.575803756713867, "prob": 0.3451950252056122 }, { "rank": 3, "tokenId": 3637, "text": "Context", "logit": 6.002678871154785, "prob": 0.07159209251403809 }, { "rank": 4, "tokenId": 1018, "text": "**", "logit": 5.8605546951293945, "prob": 0.062107108533382416 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "logit": 5.489229679107666, "prob": 0.04284270852804184 }, { "rank": 6, "tokenId": 236847, "text": "V", "logit": 3.429424524307251, "prob": 0.005461537279188633 }, { "rank": 7, "tokenId": 105367, "text": "Mixed", "logit": 2.934657573699951, "prob": 0.0033299701754003763 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "logit": 2.8827598094940186, "prob": 0.003161560045555234 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 19204, "winnerText": "Variable", "winnerScore": 8.04745334778029, "winnerGap": 0.3041567501251592, "top": [ { "rank": 1, "tokenId": 19204, "text": "Variable", "score": 8.04745334778029 }, { "rank": 2, "tokenId": 21288, "text": "Subject", "score": 7.743296597655132 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 6.0847675162577275 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 5.936860965260742 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 5.5517514917906325 }, { "rank": 6, "tokenId": 236847, "text": "V", "score": 3.4444795507225194 }, { "rank": 7, "tokenId": 105367, "text": "Mixed", "score": 2.944071149275942 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 2.8916817463197475 } ] }, "f32_forward": { "winnerTokenId": 19204, "winnerText": "Variable", "winnerScore": 8.047454833984375, "winnerGap": 0.304168701171875, "top": [ { "rank": 1, "tokenId": 19204, "text": "Variable", "score": 8.047454833984375 }, { "rank": 2, "tokenId": 21288, "text": "Subject", "score": 7.7432861328125 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 6.084771633148193 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 5.936864376068115 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 5.551758289337158 }, { "rank": 6, "tokenId": 236847, "text": "V", "score": 3.444474220275879 }, { "rank": 7, "tokenId": 105367, "text": "Mixed", "score": 2.944059133529663 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 2.891676902770996 } ] }, "f32_reverse": { "winnerTokenId": 19204, "winnerText": "Variable", "winnerScore": 8.04745101928711, "winnerGap": 0.3041539192199707, "top": [ { "rank": 1, "tokenId": 19204, "text": "Variable", "score": 8.04745101928711 }, { "rank": 2, "tokenId": 21288, "text": "Subject", "score": 7.743297100067139 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 6.084767818450928 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 5.936858177185059 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 5.551746368408203 }, { "rank": 6, "tokenId": 236847, "text": "V", "score": 3.4444799423217773 }, { "rank": 7, "tokenId": 105367, "text": "Mixed", "score": 2.9440712928771973 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 2.891679286956787 } ] }, "f32_tree64": { "winnerTokenId": 19204, "winnerText": "Variable", "winnerScore": 8.047454833984375, "winnerGap": 0.30415868759155273, "top": [ { "rank": 1, "tokenId": 19204, "text": "Variable", "score": 8.047454833984375 }, { "rank": 2, "tokenId": 21288, "text": "Subject", "score": 7.743296146392822 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 6.084768295288086 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 5.936863422393799 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 5.551751136779785 }, { "rank": 6, "tokenId": 236847, "text": "V", "score": 3.4444799423217773 }, { "rank": 7, "tokenId": 105367, "text": "Mixed", "score": 2.944070816040039 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 2.891681432723999 } ] }, "f16_forward": { "winnerTokenId": 19204, "winnerText": "Variable", "winnerScore": 8.1171875, "winnerGap": 0.4296875, "top": [ { "rank": 1, "tokenId": 19204, "text": "Variable", "score": 8.1171875 }, { "rank": 2, "tokenId": 21288, "text": "Subject", "score": 7.6875 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 6.11328125 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 6.02734375 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 5.640625 }, { "rank": 6, "tokenId": 236847, "text": "V", "score": 3.380859375 }, { "rank": 7, "tokenId": 105367, "text": "Mixed", "score": 3.048828125 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 2.875 } ] } }, "winnerTextsByMode": { "exact": "Variable", "f32_forward": "Variable", "f32_reverse": "Variable", "f32_tree64": "Variable", "f16_forward": "Variable" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Variable" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tech-self-driving-ready-oneword-a", "text": "Answer with exactly one word only.\nSelf-driving cars are ready for public roads — true or false?", "stableTopK": [ { "rank": 1, "tokenId": 9277, "text": "False", "logit": 8.231173515319824, "prob": 0.9999982118606567 }, { "rank": 2, "tokenId": 3771, "text": "No", "logit": -5.7567338943481445, "prob": 8.416436116931436e-7 }, { "rank": 3, "tokenId": 2805, "text": "Un", "logit": -6.817439556121826, "prob": 2.9138664103811607e-7 }, { "rank": 4, "tokenId": 4530, "text": "false", "logit": -7.047021389007568, "prob": 2.3161329920640128e-7 }, { "rank": 5, "tokenId": 4339, "text": "True", "logit": -7.671164035797119, "prob": 1.2407998895014316e-7 }, { "rank": 6, "tokenId": 4348, "text": "Not", "logit": -8.057828903198242, "prob": 8.428986575381714e-8 }, { "rank": 7, "tokenId": 41387, "text": "Unknown", "logit": -8.791936874389648, "prob": 4.045352142156844e-8 }, { "rank": 8, "tokenId": 8450, "text": " False", "logit": -9.10837173461914, "prob": 2.948019783843847e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 8.447583229534619, "winnerGap": 14.276574971955199, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 8.447583229534619 }, { "rank": 2, "tokenId": 3771, "text": "No", "score": -5.82899174242058 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": -6.938572960216076 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": -7.1811064908363065 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -7.8452397698727765 }, { "rank": 6, "tokenId": 4348, "text": "Not", "score": -8.260445746794229 }, { "rank": 7, "tokenId": 41387, "text": "Unknown", "score": -9.057464981009026 }, { "rank": 8, "tokenId": 8450, "text": " False", "score": -9.404822377009793 } ] }, "f32_forward": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 8.44758129119873, "winnerGap": 14.27657175064087, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 8.44758129119873 }, { "rank": 2, "tokenId": 3771, "text": "No", "score": -5.828990459442139 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": -6.938574314117432 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": -7.1811017990112305 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -7.845234394073486 }, { "rank": 6, "tokenId": 4348, "text": "Not", "score": -8.260440826416016 }, { "rank": 7, "tokenId": 41387, "text": "Unknown", "score": -9.057476043701172 }, { "rank": 8, "tokenId": 8450, "text": " False", "score": -9.404830932617188 } ] }, "f32_reverse": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 8.447574615478516, "winnerGap": 14.276566982269287, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 8.447574615478516 }, { "rank": 2, "tokenId": 3771, "text": "No", "score": -5.8289923667907715 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": -6.938575267791748 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": -7.1811089515686035 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -7.845239639282227 }, { "rank": 6, "tokenId": 4348, "text": "Not", "score": -8.260446548461914 }, { "rank": 7, "tokenId": 41387, "text": "Unknown", "score": -9.057454109191895 }, { "rank": 8, "tokenId": 8450, "text": " False", "score": -9.40482234954834 } ] }, "f32_tree64": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 8.447583198547363, "winnerGap": 14.276575565338135, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 8.447583198547363 }, { "rank": 2, "tokenId": 3771, "text": "No", "score": -5.8289923667907715 }, { "rank": 3, "tokenId": 2805, "text": "Un", "score": -6.938574314117432 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": -7.181107044219971 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -7.845240592956543 }, { "rank": 6, "tokenId": 4348, "text": "Not", "score": -8.260445594787598 }, { "rank": 7, "tokenId": 41387, "text": "Unknown", "score": -9.057464599609375 }, { "rank": 8, "tokenId": 8450, "text": " False", "score": -9.404824256896973 } ] }, "f16_forward": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 8.421875, "winnerGap": 14.30078125, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 8.421875 }, { "rank": 2, "tokenId": 3771, "text": "No", "score": -5.87890625 }, { "rank": 3, "tokenId": 48333, "text": "Complex", "score": -6.99609375 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": -7.1484375 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": -7.87890625 }, { "rank": 6, "tokenId": 4339, "text": "True", "score": -7.9140625 }, { "rank": 7, "tokenId": 4348, "text": "Not", "score": -8.3203125 }, { "rank": 8, "tokenId": 41387, "text": "Unknown", "score": -9.1328125 } ] } }, "winnerTextsByMode": { "exact": "False", "f32_forward": "False", "f32_reverse": "False", "f32_tree64": "False", "f16_forward": "False" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "False" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tech-social-media-harm-oneword-b", "text": "Respond with one word only.\nSocial media does more harm than good — true or false?", "stableTopK": [ { "rank": 1, "tokenId": 9277, "text": "False", "logit": 2.8485286235809326, "prob": 0.9373427033424377 }, { "rank": 2, "tokenId": 95889, "text": "Neither", "logit": -0.6358524560928345, "prob": 0.028750866651535034 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "logit": -1.2960901260375977, "prob": 0.014856391586363316 }, { "rank": 4, "tokenId": 30340, "text": "Und", "logit": -1.3569785356521606, "prob": 0.013978798873722553 }, { "rank": 5, "tokenId": 4339, "text": "True", "logit": -3.59914231300354, "prob": 0.0014849455328658223 }, { "rank": 6, "tokenId": 43868, "text": "Nu", "logit": -3.802126407623291, "prob": 0.0012121479958295822 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "logit": -4.067185878753662, "prob": 0.0009299118537455797 }, { "rank": 8, "tokenId": 2805, "text": "Un", "logit": -4.886242389678955, "prob": 0.0004099491925444454 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 2.8571384852444286, "winnerGap": 3.4930840359327338, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 2.8571384852444286 }, { "rank": 2, "tokenId": 95889, "text": "Neither", "score": -0.6359455506883052 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": -1.2968988173166451 }, { "rank": 4, "tokenId": 30340, "text": "Und", "score": -1.357904201766729 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -3.616561829007381 }, { "rank": 6, "tokenId": 43868, "text": "Nu", "score": -3.822682696207238 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "score": -4.09238371836577 }, { "rank": 8, "tokenId": 2805, "text": "Un", "score": -4.930148361140777 } ] }, "f32_forward": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 2.8571372032165527, "winnerGap": 3.493085503578186, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 2.8571372032165527 }, { "rank": 2, "tokenId": 95889, "text": "Neither", "score": -0.6359483003616333 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": -1.2968982458114624 }, { "rank": 4, "tokenId": 30340, "text": "Und", "score": -1.357905387878418 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -3.6165611743927 }, { "rank": 6, "tokenId": 43868, "text": "Nu", "score": -3.8226828575134277 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "score": -4.092380523681641 }, { "rank": 8, "tokenId": 2805, "text": "Un", "score": -4.930149078369141 } ] }, "f32_reverse": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 2.857137680053711, "winnerGap": 3.493089020252228, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 2.857137680053711 }, { "rank": 2, "tokenId": 95889, "text": "Neither", "score": -0.6359513401985168 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": -1.2968991994857788 }, { "rank": 4, "tokenId": 30340, "text": "Und", "score": -1.3579055070877075 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -3.616558074951172 }, { "rank": 6, "tokenId": 43868, "text": "Nu", "score": -3.8226804733276367 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "score": -4.09237813949585 }, { "rank": 8, "tokenId": 2805, "text": "Un", "score": -4.930149555206299 } ] }, "f32_tree64": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 2.857138156890869, "winnerGap": 3.4930858612060547, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 2.857138156890869 }, { "rank": 2, "tokenId": 95889, "text": "Neither", "score": -0.6359477043151855 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": -1.296898365020752 }, { "rank": 4, "tokenId": 30340, "text": "Und", "score": -1.357903003692627 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -3.6165623664855957 }, { "rank": 6, "tokenId": 43868, "text": "Nu", "score": -3.822681427001953 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "score": -4.092383861541748 }, { "rank": 8, "tokenId": 2805, "text": "Un", "score": -4.93014669418335 } ] }, "f16_forward": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 2.888671875, "winnerGap": 3.509765625, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 2.888671875 }, { "rank": 2, "tokenId": 95889, "text": "Neither", "score": -0.62109375 }, { "rank": 3, "tokenId": 31900, "text": "Deb", "score": -1.2548828125 }, { "rank": 4, "tokenId": 30340, "text": "Und", "score": -1.43359375 }, { "rank": 5, "tokenId": 154572, "text": "Discuss", "score": -3.5703125 }, { "rank": 6, "tokenId": 4339, "text": "True", "score": -3.607421875 }, { "rank": 7, "tokenId": 43868, "text": "Nu", "score": -3.826171875 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": -4.08203125 } ] } }, "winnerTextsByMode": { "exact": "False", "f32_forward": "False", "f32_reverse": "False", "f32_tree64": "False", "f16_forward": "False" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "False" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tech-space-colonize-oneword-c", "text": "No explanation. One word only.\nColonizing Mars is humanity's priority or a distraction. It is", "stableTopK": [ { "rank": 1, "tokenId": 65221, "text": "Exist", "logit": 13.682193756103516, "prob": 0.23609186708927155 }, { "rank": 2, "tokenId": 93294, "text": "Fut", "logit": 12.944238662719727, "prob": 0.11287329345941544 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "logit": 12.802826881408691, "prob": 0.09798889607191086 }, { "rank": 4, "tokenId": 236796, "text": "D", "logit": 12.796838760375977, "prob": 0.0974038764834404 }, { "rank": 5, "tokenId": 71869, "text": "Cru", "logit": 12.353935241699219, "prob": 0.06254976242780685 }, { "rank": 6, "tokenId": 14977, "text": "Question", "logit": 12.106404304504395, "prob": 0.04883423447608948 }, { "rank": 7, "tokenId": 181220, "text": "Survival", "logit": 11.734912872314453, "prob": 0.03368121013045311 }, { "rank": 8, "tokenId": 71398, "text": "Risk", "logit": 11.635430335998535, "prob": 0.030491793528199196 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 65221, "winnerText": "Exist", "winnerScore": 14.77025700867025, "winnerGap": 0.9190416424958681, "top": [ { "rank": 1, "tokenId": 65221, "text": "Exist", "score": 14.77025700867025 }, { "rank": 2, "tokenId": 93294, "text": "Fut", "score": 13.851215366174381 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "score": 13.677887569168533 }, { "rank": 4, "tokenId": 236796, "text": "D", "score": 13.670568402844971 }, { "rank": 5, "tokenId": 71869, "text": "Cru", "score": 13.133230702680695 }, { "rank": 6, "tokenId": 14977, "text": "Question", "score": 12.836359470080792 }, { "rank": 7, "tokenId": 181220, "text": "Survival", "score": 12.39519452055042 }, { "rank": 8, "tokenId": 71398, "text": "Risk", "score": 12.277923466682495 } ] }, "f32_forward": { "winnerTokenId": 65221, "winnerText": "Exist", "winnerScore": 14.770256996154785, "winnerGap": 0.9190397262573242, "top": [ { "rank": 1, "tokenId": 65221, "text": "Exist", "score": 14.770256996154785 }, { "rank": 2, "tokenId": 93294, "text": "Fut", "score": 13.851217269897461 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "score": 13.677885055541992 }, { "rank": 4, "tokenId": 236796, "text": "D", "score": 13.67055606842041 }, { "rank": 5, "tokenId": 71869, "text": "Cru", "score": 13.133244514465332 }, { "rank": 6, "tokenId": 14977, "text": "Question", "score": 12.83635139465332 }, { "rank": 7, "tokenId": 181220, "text": "Survival", "score": 12.395179748535156 }, { "rank": 8, "tokenId": 71398, "text": "Risk", "score": 12.27794361114502 } ] }, "f32_reverse": { "winnerTokenId": 65221, "winnerText": "Exist", "winnerScore": 14.770259857177734, "winnerGap": 0.9190425872802734, "top": [ { "rank": 1, "tokenId": 65221, "text": "Exist", "score": 14.770259857177734 }, { "rank": 2, "tokenId": 93294, "text": "Fut", "score": 13.851217269897461 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "score": 13.677881240844727 }, { "rank": 4, "tokenId": 236796, "text": "D", "score": 13.670585632324219 }, { "rank": 5, "tokenId": 71869, "text": "Cru", "score": 13.133233070373535 }, { "rank": 6, "tokenId": 14977, "text": "Question", "score": 12.836355209350586 }, { "rank": 7, "tokenId": 181220, "text": "Survival", "score": 12.395188331604004 }, { "rank": 8, "tokenId": 71398, "text": "Risk", "score": 12.277920722961426 } ] }, "f32_tree64": { "winnerTokenId": 65221, "winnerText": "Exist", "winnerScore": 14.770257949829102, "winnerGap": 0.919041633605957, "top": [ { "rank": 1, "tokenId": 65221, "text": "Exist", "score": 14.770257949829102 }, { "rank": 2, "tokenId": 93294, "text": "Fut", "score": 13.851216316223145 }, { "rank": 3, "tokenId": 55144, "text": "Amb", "score": 13.677886962890625 }, { "rank": 4, "tokenId": 236796, "text": "D", "score": 13.670570373535156 }, { "rank": 5, "tokenId": 71869, "text": "Cru", "score": 13.133232116699219 }, { "rank": 6, "tokenId": 14977, "text": "Question", "score": 12.836359024047852 }, { "rank": 7, "tokenId": 181220, "text": "Survival", "score": 12.395195007324219 }, { "rank": 8, "tokenId": 71398, "text": "Risk", "score": 12.277923583984375 } ] }, "f16_forward": { "winnerTokenId": 93294, "winnerText": "Fut", "winnerScore": 13.84375, "winnerGap": 0.03125, "top": [ { "rank": 1, "tokenId": 93294, "text": "Fut", "score": 13.84375 }, { "rank": 2, "tokenId": 236796, "text": "D", "score": 13.8125 }, { "rank": 3, "tokenId": 181220, "text": "Survival", "score": 12.3984375 }, { "rank": 4, "tokenId": 71398, "text": "Risk", "score": 12.375 }, { "rank": 5, "tokenId": 66733, "text": "Challenge", "score": 12.2890625 }, { "rank": 6, "tokenId": 31900, "text": "Deb", "score": 11.75 }, { "rank": 7, "tokenId": 53058, "text": "Priority", "score": 11.625 }, { "rank": 8, "tokenId": 38370, "text": "Ur", "score": 11.453125 } ] } }, "winnerTextsByMode": { "exact": "Exist", "f32_forward": "Exist", "f32_reverse": "Exist", "f32_tree64": "Exist", "f16_forward": "Fut" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Exist", "Fut" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 65221, 1915 ], "decodedText": "Existential", "steps": [ { "step": 0, "tokenId": 65221, "text": "Exist" }, { "step": 1, "tokenId": 1915, "text": "ential" } ] }, "f16_forward": { "tokenIds": [ 93294, 755 ], "decodedText": "Futile", "steps": [ { "step": 0, "tokenId": 93294, "text": "Fut" }, { "step": 1, "tokenId": 755, "text": "ile" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 1, "differingStepCount": 2, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "tf-animals-moral-oneword-d", "text": "Choose one word only.\nAnswer with exactly one word: true or false. Question: Animals have moral rights. Answer:", "stableTopK": [ { "rank": 1, "tokenId": 9277, "text": "False", "logit": 3.895927667617798, "prob": 0.980189859867096 }, { "rank": 2, "tokenId": 4530, "text": "false", "logit": -0.32952460646629333, "prob": 0.014329122379422188 }, { "rank": 3, "tokenId": 4339, "text": "True", "logit": -1.3359798192977905, "prob": 0.005237470846623182 }, { "rank": 4, "tokenId": 3397, "text": "true", "logit": -4.557961940765381, "prob": 0.00020884908735752106 }, { "rank": 5, "tokenId": 25874, "text": "FALSE", "logit": -7.326341152191162, "prob": 0.000013108131497574504 }, { "rank": 6, "tokenId": 2805, "text": "Un", "logit": -8.25378704071045, "prob": 0.000005185087957215728 }, { "rank": 7, "tokenId": 1136, "text": "und", "logit": -9.15304946899414, "prob": 0.0000021096548152854666 }, { "rank": 8, "tokenId": 31900, "text": "Deb", "logit": -9.264389038085938, "prob": 0.000001887371013253869 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 3.918059460437288, "winnerGap": 4.247593674787105, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 3.918059460437288 }, { "rank": 2, "tokenId": 4530, "text": "false", "score": -0.3295342143498168 }, { "rank": 3, "tokenId": 4339, "text": "True", "score": -1.3368641022161345 }, { "rank": 4, "tokenId": 3397, "text": "true", "score": -4.593530038961859 }, { "rank": 5, "tokenId": 25874, "text": "FALSE", "score": -7.477424435548585 }, { "rank": 6, "tokenId": 2805, "text": "Un", "score": -8.47204584680447 }, { "rank": 7, "tokenId": 1136, "text": "und", "score": -9.454057705415712 }, { "rank": 8, "tokenId": 31900, "text": "Deb", "score": -9.576983990500107 } ] }, "f32_forward": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 3.9180469512939453, "winnerGap": 4.247580975294113, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 3.9180469512939453 }, { "rank": 2, "tokenId": 4530, "text": "false", "score": -0.32953402400016785 }, { "rank": 3, "tokenId": 4339, "text": "True", "score": -1.336864709854126 }, { "rank": 4, "tokenId": 3397, "text": "true", "score": -4.593525409698486 }, { "rank": 5, "tokenId": 25874, "text": "FALSE", "score": -7.4774169921875 }, { "rank": 6, "tokenId": 2805, "text": "Un", "score": -8.47204303741455 }, { "rank": 7, "tokenId": 1136, "text": "und", "score": -9.454056739807129 }, { "rank": 8, "tokenId": 31900, "text": "Deb", "score": -9.576981544494629 } ] }, "f32_reverse": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 3.918060779571533, "winnerGap": 4.247595012187958, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 3.918060779571533 }, { "rank": 2, "tokenId": 4530, "text": "false", "score": -0.32953423261642456 }, { "rank": 3, "tokenId": 4339, "text": "True", "score": -1.3368642330169678 }, { "rank": 4, "tokenId": 3397, "text": "true", "score": -4.593527793884277 }, { "rank": 5, "tokenId": 25874, "text": "FALSE", "score": -7.477433204650879 }, { "rank": 6, "tokenId": 2805, "text": "Un", "score": -8.472057342529297 }, { "rank": 7, "tokenId": 1136, "text": "und", "score": -9.454038619995117 }, { "rank": 8, "tokenId": 31900, "text": "Deb", "score": -9.576982498168945 } ] }, "f32_tree64": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 3.9180588722229004, "winnerGap": 4.247592926025391, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 3.9180588722229004 }, { "rank": 2, "tokenId": 4530, "text": "false", "score": -0.32953405380249023 }, { "rank": 3, "tokenId": 4339, "text": "True", "score": -1.3368643522262573 }, { "rank": 4, "tokenId": 3397, "text": "true", "score": -4.59352970123291 }, { "rank": 5, "tokenId": 25874, "text": "FALSE", "score": -7.477424621582031 }, { "rank": 6, "tokenId": 2805, "text": "Un", "score": -8.4720458984375 }, { "rank": 7, "tokenId": 1136, "text": "und", "score": -9.454057693481445 }, { "rank": 8, "tokenId": 31900, "text": "Deb", "score": -9.576983451843262 } ] }, "f16_forward": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 3.939453125, "winnerGap": 4.224609375, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 3.939453125 }, { "rank": 2, "tokenId": 4530, "text": "false", "score": -0.28515625 }, { "rank": 3, "tokenId": 4339, "text": "True", "score": -1.0693359375 }, { "rank": 4, "tokenId": 25874, "text": "FALSE", "score": -6.5546875 }, { "rank": 5, "tokenId": 3397, "text": "true", "score": -6.59765625 }, { "rank": 6, "tokenId": 2709, "text": "Ob", "score": -8.140625 }, { "rank": 7, "tokenId": 2805, "text": "Un", "score": -8.7109375 }, { "rank": 8, "tokenId": 14713, "text": "Under", "score": -8.78125 } ] } }, "winnerTextsByMode": { "exact": "False", "f32_forward": "False", "f32_reverse": "False", "f32_tree64": "False", "f16_forward": "False" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "False" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tf-death-bad-oneword-a", "text": "Answer with exactly one word only.\nAnswer with exactly one word: true or false. Question: Death is bad. Answer:", "stableTopK": [ { "rank": 1, "tokenId": 9277, "text": "False", "logit": 4.244988918304443, "prob": 0.7544596195220947 }, { "rank": 2, "tokenId": 4339, "text": "True", "logit": 2.599461317062378, "prob": 0.14554338157176971 }, { "rank": 3, "tokenId": 21288, "text": "Subject", "logit": 1.7766449451446533, "prob": 0.06392163038253784 }, { "rank": 4, "tokenId": 13552, "text": "subject", "logit": 0.5794658064842224, "prob": 0.0193072110414505 }, { "rank": 5, "tokenId": 4530, "text": "false", "logit": -0.40053752064704895, "prob": 0.007246186025440693 }, { "rank": 6, "tokenId": 3397, "text": "true", "logit": -1.2851712703704834, "prob": 0.002991699380800128 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "logit": -1.9713449478149414, "prob": 0.0015063174068927765 }, { "rank": 8, "tokenId": 1136, "text": "und", "logit": -2.5845015048980713, "prob": 0.0008158794371411204 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 4.273665033024528, "winnerGap": 1.6676679101612706, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 4.273665033024528 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 2.6059971228632577 }, { "rank": 3, "tokenId": 21288, "text": "Subject", "score": 1.7787306280656594 }, { "rank": 4, "tokenId": 13552, "text": "subject", "score": 0.579533306212511 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": -0.40056369691121674 }, { "rank": 6, "tokenId": 3397, "text": "true", "score": -1.2859603811630222 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "score": -1.9741869054824406 }, { "rank": 8, "tokenId": 1136, "text": "und", "score": -2.590924545246672 } ] }, "f32_forward": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 4.273665904998779, "winnerGap": 1.6676743030548096, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 4.273665904998779 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 2.6059916019439697 }, { "rank": 3, "tokenId": 21288, "text": "Subject", "score": 1.7787343263626099 }, { "rank": 4, "tokenId": 13552, "text": "subject", "score": 0.579535186290741 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": -0.4005592167377472 }, { "rank": 6, "tokenId": 3397, "text": "true", "score": -1.2859596014022827 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "score": -1.9741860628128052 }, { "rank": 8, "tokenId": 1136, "text": "und", "score": -2.590925455093384 } ] }, "f32_reverse": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 4.273664951324463, "winnerGap": 1.6676714420318604, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 4.273664951324463 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 2.6059935092926025 }, { "rank": 3, "tokenId": 21288, "text": "Subject", "score": 1.7787327766418457 }, { "rank": 4, "tokenId": 13552, "text": "subject", "score": 0.5795336961746216 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": -0.40055978298187256 }, { "rank": 6, "tokenId": 3397, "text": "true", "score": -1.2859601974487305 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "score": -1.97419011592865 }, { "rank": 8, "tokenId": 1136, "text": "und", "score": -2.5909252166748047 } ] }, "f32_tree64": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 4.273664474487305, "winnerGap": 1.6676671504974365, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 4.273664474487305 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 2.605997323989868 }, { "rank": 3, "tokenId": 21288, "text": "Subject", "score": 1.778731346130371 }, { "rank": 4, "tokenId": 13552, "text": "subject", "score": 0.5795323848724365 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": -0.40056419372558594 }, { "rank": 6, "tokenId": 3397, "text": "true", "score": -1.2859606742858887 }, { "rank": 7, "tokenId": 55144, "text": "Amb", "score": -1.9741865396499634 }, { "rank": 8, "tokenId": 1136, "text": "und", "score": -2.5909242630004883 } ] }, "f16_forward": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 4.3671875, "winnerGap": 1.748046875, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 4.3671875 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 2.619140625 }, { "rank": 3, "tokenId": 13552, "text": "subject", "score": 0.60595703125 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": -0.4326171875 }, { "rank": 5, "tokenId": 3397, "text": "true", "score": -1.27734375 }, { "rank": 6, "tokenId": 55144, "text": "Amb", "score": -1.9326171875 }, { "rank": 7, "tokenId": 21288, "text": "Subject", "score": -2.2109375 }, { "rank": 8, "tokenId": 1136, "text": "und", "score": -2.515625 } ] } }, "winnerTextsByMode": { "exact": "False", "f32_forward": "False", "f32_reverse": "False", "f32_tree64": "False", "f16_forward": "False" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "False" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tf-democracy-best-oneword-b", "text": "Respond with one word only.\nAnswer with exactly one word: true or false. Question: Democracy is the best form of government. Answer:", "stableTopK": [ { "rank": 1, "tokenId": 4530, "text": "false", "logit": 11.607423782348633, "prob": 0.9809413552284241 }, { "rank": 2, "tokenId": 9277, "text": "False", "logit": 7.666389465332031, "prob": 0.01905783638358116 }, { "rank": 3, "tokenId": 3397, "text": "true", "logit": -2.6621861457824707, "prob": 6.229170139704365e-7 }, { "rank": 4, "tokenId": 25874, "text": "FALSE", "logit": -5.203859806060791, "prob": 4.9045070227293763e-8 }, { "rank": 5, "tokenId": 4339, "text": "True", "logit": -6.359395503997803, "prob": 1.5443744061371945e-8 }, { "rank": 6, "tokenId": 2416, "text": " false", "logit": -6.535686016082764, "prob": 1.2947638516891402e-8 }, { "rank": 7, "tokenId": 893, "text": "de", "logit": -6.576037883758545, "prob": 1.2435578788938528e-8 }, { "rank": 8, "tokenId": 2805, "text": "Un", "logit": -6.926059246063232, "prob": 8.763016978718952e-9 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 12.244973987414475, "winnerGap": 4.404842450192049, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 12.244973987414475 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 7.840131537222426 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": -2.6692069208130533 }, { "rank": 4, "tokenId": 25874, "text": "FALSE", "score": -5.257012002909541 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -6.457303152665659 }, { "rank": 6, "tokenId": 2416, "text": " false", "score": -6.642133896250472 }, { "rank": 7, "tokenId": 893, "text": "de", "score": -6.68450796405863 }, { "rank": 8, "tokenId": 2805, "text": "Un", "score": -7.053202546380128 } ] }, "f32_forward": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 12.244976043701172, "winnerGap": 4.404848575592041, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 12.244976043701172 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 7.840127468109131 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": -2.669210433959961 }, { "rank": 4, "tokenId": 25874, "text": "FALSE", "score": -5.257014274597168 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -6.457306385040283 }, { "rank": 6, "tokenId": 2416, "text": " false", "score": -6.642131328582764 }, { "rank": 7, "tokenId": 893, "text": "de", "score": -6.684510231018066 }, { "rank": 8, "tokenId": 2805, "text": "Un", "score": -7.05319881439209 } ] }, "f32_reverse": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 12.244982719421387, "winnerGap": 4.404852867126465, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 12.244982719421387 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 7.840129852294922 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": -2.669205904006958 }, { "rank": 4, "tokenId": 25874, "text": "FALSE", "score": -5.257016181945801 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -6.45729398727417 }, { "rank": 6, "tokenId": 2416, "text": " false", "score": -6.64212703704834 }, { "rank": 7, "tokenId": 893, "text": "de", "score": -6.684519290924072 }, { "rank": 8, "tokenId": 2805, "text": "Un", "score": -7.053208351135254 } ] }, "f32_tree64": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 12.244973182678223, "winnerGap": 4.404843330383301, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 12.244973182678223 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 7.840129852294922 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": -2.6692075729370117 }, { "rank": 4, "tokenId": 25874, "text": "FALSE", "score": -5.257010459899902 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -6.457304000854492 }, { "rank": 6, "tokenId": 2416, "text": " false", "score": -6.642134666442871 }, { "rank": 7, "tokenId": 893, "text": "de", "score": -6.68450927734375 }, { "rank": 8, "tokenId": 2805, "text": "Un", "score": -7.05320405960083 } ] }, "f16_forward": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 8.328125, "winnerGap": 0.5078125, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 8.328125 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 7.8203125 }, { "rank": 3, "tokenId": 25874, "text": "FALSE", "score": -1.23046875 }, { "rank": 4, "tokenId": 3397, "text": "true", "score": -2.685546875 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -3.474609375 }, { "rank": 6, "tokenId": 198275, "text": "neither", "score": -4.11328125 }, { "rank": 7, "tokenId": 13552, "text": "subject", "score": -4.6953125 }, { "rank": 8, "tokenId": 236797, "text": "N", "score": -4.71484375 } ] } }, "winnerTextsByMode": { "exact": "false", "f32_forward": "false", "f32_reverse": "false", "f32_tree64": "false", "f16_forward": "false" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "false" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tf-math-invented-oneword-c", "text": "No explanation. One word only.\nAnswer with exactly one word: true or false. Question: Mathematics was invented, not discovered. Answer:", "stableTopK": [ { "rank": 1, "tokenId": 4339, "text": "True", "logit": 15.215332984924316, "prob": 0.7513692378997803 }, { "rank": 2, "tokenId": 3397, "text": "true", "logit": 14.034361839294434, "prob": 0.23065567016601562 }, { "rank": 3, "tokenId": 9277, "text": "False", "logit": 11.254536628723145, "prob": 0.014312036335468292 }, { "rank": 4, "tokenId": 4530, "text": "false", "logit": 9.887396812438965, "prob": 0.003647204488515854 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "logit": 3.106713056564331, "prob": 0.000004141394583712099 }, { "rank": 6, "tokenId": 1018, "text": "**", "logit": 0.8090237379074097, "prob": 4.161719289186294e-7 }, { "rank": 7, "tokenId": 236811, "text": "F", "logit": 0.7447982430458069, "prob": 3.902833327629196e-7 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "logit": 0.6463772058486938, "prob": 3.537010115906014e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 16.767687700113967, "winnerGap": 1.5497141499537435, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 16.767687700113967 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 15.217973550160224 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": 11.832141008460358 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": 10.270706025354476 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": 3.117892265401398 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 0.8092194129217418 }, { "rank": 7, "tokenId": 236811, "text": "F", "score": 0.7449516233712536 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "score": 0.646477826167704 } ] }, "f32_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 16.767690658569336, "winnerGap": 1.5497064590454102, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 16.767690658569336 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 15.217984199523926 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": 11.832141876220703 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": 10.270711898803711 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": 3.117898941040039 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 0.8092224597930908 }, { "rank": 7, "tokenId": 236811, "text": "F", "score": 0.7449526190757751 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "score": 0.6464828848838806 } ] }, "f32_reverse": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 16.767688751220703, "winnerGap": 1.5497150421142578, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 16.767688751220703 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 15.217973709106445 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": 11.83214282989502 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": 10.270705223083496 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": 3.1178910732269287 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 0.809217631816864 }, { "rank": 7, "tokenId": 236811, "text": "F", "score": 0.7449501156806946 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "score": 0.6464822888374329 } ] }, "f32_tree64": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 16.767688751220703, "winnerGap": 1.5497150421142578, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 16.767688751220703 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 15.217973709106445 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": 11.83213996887207 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": 10.27070426940918 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": 3.117891788482666 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 0.8092191219329834 }, { "rank": 7, "tokenId": 236811, "text": "F", "score": 0.7449514865875244 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "score": 0.6464772820472717 } ] }, "f16_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 16.8125, "winnerGap": 5.640625, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 16.8125 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 11.171875 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": 10.2734375 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": 7.77734375 }, { "rank": 5, "tokenId": 22369, "text": "TRUE", "score": 2.1484375 }, { "rank": 6, "tokenId": 1018, "text": "**", "score": 0.82470703125 }, { "rank": 7, "tokenId": 236811, "text": "F", "score": 0.78466796875 }, { "rank": 8, "tokenId": 3771, "text": "No", "score": 0.50634765625 } ] } }, "winnerTextsByMode": { "exact": "True", "f32_forward": "True", "f32_reverse": "True", "f32_tree64": "True", "f16_forward": "True" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "True" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tf-money-happiness-oneword-d", "text": "Choose one word only.\nAnswer with exactly one word: true or false. Question: Money buys happiness. Answer:", "stableTopK": [ { "rank": 1, "tokenId": 9277, "text": "False", "logit": 12.743847846984863, "prob": 0.9948822855949402 }, { "rank": 2, "tokenId": 4530, "text": "false", "logit": 7.4738450050354, "prob": 0.005117272492498159 }, { "rank": 3, "tokenId": 25874, "text": "FALSE", "logit": -2.566314220428467, "prob": 2.2317873060728743e-7 }, { "rank": 4, "tokenId": 4339, "text": "True", "logit": -2.7048416137695312, "prob": 1.943081855415585e-7 }, { "rank": 5, "tokenId": 8450, "text": " False", "logit": -6.062648296356201, "prob": 6.764164783845672e-9 }, { "rank": 6, "tokenId": 2416, "text": " false", "logit": -6.476770401000977, "prob": 4.470573689019375e-9 }, { "rank": 7, "tokenId": 1018, "text": "**", "logit": -6.615593910217285, "prob": 3.89110521581415e-9 }, { "rank": 8, "tokenId": 3397, "text": "true", "logit": -6.810885429382324, "prob": 3.200802956015991e-9 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 13.605852901553353, "winnerGap": 5.971364369367629, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 13.605852901553353 }, { "rank": 2, "tokenId": 4530, "text": "false", "score": 7.634488532185724 }, { "rank": 3, "tokenId": 25874, "text": "FALSE", "score": -2.5726013132450056 }, { "rank": 4, "tokenId": 4339, "text": "True", "score": -2.7122063917344548 }, { "rank": 5, "tokenId": 8450, "text": " False", "score": -6.147253249107365 }, { "rank": 6, "tokenId": 2416, "text": " false", "score": -6.580309680446703 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -6.726071399921945 }, { "rank": 8, "tokenId": 3397, "text": "true", "score": -6.931663006476159 } ] }, "f32_forward": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 13.605854988098145, "winnerGap": 5.971366882324219, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 13.605854988098145 }, { "rank": 2, "tokenId": 4530, "text": "false", "score": 7.634488105773926 }, { "rank": 3, "tokenId": 25874, "text": "FALSE", "score": -2.572601556777954 }, { "rank": 4, "tokenId": 4339, "text": "True", "score": -2.7122135162353516 }, { "rank": 5, "tokenId": 8450, "text": " False", "score": -6.147256374359131 }, { "rank": 6, "tokenId": 2416, "text": " false", "score": -6.580305576324463 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -6.726068496704102 }, { "rank": 8, "tokenId": 3397, "text": "true", "score": -6.9316606521606445 } ] }, "f32_reverse": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 13.605856895446777, "winnerGap": 5.971358776092529, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 13.605856895446777 }, { "rank": 2, "tokenId": 4530, "text": "false", "score": 7.634498119354248 }, { "rank": 3, "tokenId": 25874, "text": "FALSE", "score": -2.5726006031036377 }, { "rank": 4, "tokenId": 4339, "text": "True", "score": -2.7122108936309814 }, { "rank": 5, "tokenId": 8450, "text": " False", "score": -6.14725923538208 }, { "rank": 6, "tokenId": 2416, "text": " false", "score": -6.580312252044678 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -6.726071834564209 }, { "rank": 8, "tokenId": 3397, "text": "true", "score": -6.931670188903809 } ] }, "f32_tree64": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 13.605852127075195, "winnerGap": 5.9713640213012695, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 13.605852127075195 }, { "rank": 2, "tokenId": 4530, "text": "false", "score": 7.634488105773926 }, { "rank": 3, "tokenId": 25874, "text": "FALSE", "score": -2.572601318359375 }, { "rank": 4, "tokenId": 4339, "text": "True", "score": -2.7122058868408203 }, { "rank": 5, "tokenId": 8450, "text": " False", "score": -6.147254943847656 }, { "rank": 6, "tokenId": 2416, "text": " false", "score": -6.58030891418457 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -6.726070404052734 }, { "rank": 8, "tokenId": 3397, "text": "true", "score": -6.931662082672119 } ] }, "f16_forward": { "winnerTokenId": 9277, "winnerText": "False", "winnerScore": 13.6875, "winnerGap": 5.984375, "top": [ { "rank": 1, "tokenId": 9277, "text": "False", "score": 13.6875 }, { "rank": 2, "tokenId": 4530, "text": "false", "score": 7.703125 }, { "rank": 3, "tokenId": 25874, "text": "FALSE", "score": -2.59375 }, { "rank": 4, "tokenId": 4339, "text": "True", "score": -2.6953125 }, { "rank": 5, "tokenId": 2318, "text": "\\[", "score": -5.796875 }, { "rank": 6, "tokenId": 8450, "text": " False", "score": -6.10546875 }, { "rank": 7, "tokenId": 2416, "text": " false", "score": -6.55859375 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": -6.7578125 } ] } }, "winnerTextsByMode": { "exact": "False", "f32_forward": "False", "f32_reverse": "False", "f32_tree64": "False", "f16_forward": "False" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "False" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tf-privacy-dead-oneword-a", "text": "Answer with exactly one word only.\nAnswer with exactly one word: true or false. Question: Privacy is dead in the digital age. Answer:", "stableTopK": [ { "rank": 1, "tokenId": 4530, "text": "false", "logit": 4.992819309234619, "prob": 0.8725006580352783 }, { "rank": 2, "tokenId": 9277, "text": "False", "logit": 2.8903536796569824, "prob": 0.1065802052617073 }, { "rank": 3, "tokenId": 3397, "text": "true", "logit": 1.2077335119247437, "prob": 0.01981179788708687 }, { "rank": 4, "tokenId": 4339, "text": "True", "logit": -1.6780012845993042, "prob": 0.0011057709343731403 }, { "rank": 5, "tokenId": 25874, "text": "FALSE", "logit": -9.768062591552734, "prob": 3.3899726759045734e-7 }, { "rank": 6, "tokenId": 2416, "text": " false", "logit": -10.001161575317383, "prob": 2.685113145162177e-7 }, { "rank": 7, "tokenId": 8450, "text": " False", "logit": -11.854372024536133, "prob": 4.208462911492461e-8 }, { "rank": 8, "tokenId": 6005, "text": "first", "logit": -12.77404499053955, "prob": 1.6777013556179554e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 5.039696933164683, "winnerGap": 2.140349568635509, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 5.039696933164683 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 2.8993473645291736 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": 1.208386393902714 }, { "rank": 4, "tokenId": 4339, "text": "True", "score": -1.6797528189075024 }, { "rank": 5, "tokenId": 25874, "text": "FALSE", "score": -10.137018181262896 }, { "rank": 6, "tokenId": 2416, "text": " false", "score": -10.398509819243287 }, { "rank": 7, "tokenId": 8450, "text": " False", "score": -12.536496512251972 }, { "rank": 8, "tokenId": 6005, "text": "first", "score": -13.642711464940614 } ] }, "f32_forward": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 5.039693355560303, "winnerGap": 2.1403439044952393, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 5.039693355560303 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 2.8993494510650635 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": 1.208383560180664 }, { "rank": 4, "tokenId": 4339, "text": "True", "score": -1.6797549724578857 }, { "rank": 5, "tokenId": 25874, "text": "FALSE", "score": -10.137020111083984 }, { "rank": 6, "tokenId": 2416, "text": " false", "score": -10.398516654968262 }, { "rank": 7, "tokenId": 8450, "text": " False", "score": -12.536482810974121 }, { "rank": 8, "tokenId": 6005, "text": "first", "score": -13.642718315124512 } ] }, "f32_reverse": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 5.039701461791992, "winnerGap": 2.140355110168457, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 5.039701461791992 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 2.899346351623535 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": 1.2083871364593506 }, { "rank": 4, "tokenId": 4339, "text": "True", "score": -1.6797499656677246 }, { "rank": 5, "tokenId": 25874, "text": "FALSE", "score": -10.137001991271973 }, { "rank": 6, "tokenId": 2416, "text": " false", "score": -10.398508071899414 }, { "rank": 7, "tokenId": 8450, "text": " False", "score": -12.536492347717285 }, { "rank": 8, "tokenId": 6005, "text": "first", "score": -13.64272403717041 } ] }, "f32_tree64": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 5.039696216583252, "winnerGap": 2.1403489112854004, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 5.039696216583252 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 2.8993473052978516 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": 1.208385944366455 }, { "rank": 4, "tokenId": 4339, "text": "True", "score": -1.6797535419464111 }, { "rank": 5, "tokenId": 25874, "text": "FALSE", "score": -10.137019157409668 }, { "rank": 6, "tokenId": 2416, "text": " false", "score": -10.39850902557373 }, { "rank": 7, "tokenId": 8450, "text": " False", "score": -12.536497116088867 }, { "rank": 8, "tokenId": 6005, "text": "first", "score": -13.642709732055664 } ] }, "f16_forward": { "winnerTokenId": 4530, "winnerText": "false", "winnerScore": 5.0546875, "winnerGap": 2.38671875, "top": [ { "rank": 1, "tokenId": 4530, "text": "false", "score": 5.0546875 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 2.66796875 }, { "rank": 3, "tokenId": 3397, "text": "true", "score": 2.216796875 }, { "rank": 4, "tokenId": 4339, "text": "True", "score": -1.685546875 }, { "rank": 5, "tokenId": 25874, "text": "FALSE", "score": -10.1484375 }, { "rank": 6, "tokenId": 2416, "text": " false", "score": -10.4921875 }, { "rank": 7, "tokenId": 74860, "text": "often", "score": -11.0390625 }, { "rank": 8, "tokenId": 19609, "text": "likely", "score": -11.96875 } ] } }, "winnerTextsByMode": { "exact": "false", "f32_forward": "false", "f32_reverse": "false", "f32_tree64": "false", "f16_forward": "false" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "false" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tf-time-illusion-oneword-b", "text": "Respond with one word only.\nAnswer with exactly one word: true or false. Question: Time is an illusion. Answer:", "stableTopK": [ { "rank": 1, "tokenId": 3397, "text": "true", "logit": 3.2907426357269287, "prob": 0.7443019151687622 }, { "rank": 2, "tokenId": 4339, "text": "True", "logit": 2.0710556507110596, "prob": 0.2198091745376587 }, { "rank": 3, "tokenId": 9277, "text": "False", "logit": -0.39583340287208557, "prob": 0.01865045726299286 }, { "rank": 4, "tokenId": 4530, "text": "false", "logit": -0.47629883885383606, "prob": 0.017208531498908997 }, { "rank": 5, "tokenId": 2805, "text": "Un", "logit": -8.386589050292969, "prob": 0.000006314637175819371 }, { "rank": 6, "tokenId": 2709, "text": "Ob", "logit": -9.6170654296875, "prob": 0.000001844842586251616 }, { "rank": 7, "tokenId": 13552, "text": "subject", "logit": -10.257329940795898, "prob": 9.725142717798008e-7 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "logit": -10.32734489440918, "prob": 9.067527457773394e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 3.3040361669146314, "winnerGap": 1.229683125798715, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 3.3040361669146314 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 2.0743530411159163 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": -0.3958513966983024 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": -0.4763363844385857 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": -8.615912710350834 }, { "rank": 6, "tokenId": 2709, "text": "Ob", "score": -9.968425458916009 }, { "rank": 7, "tokenId": 13552, "text": "subject", "score": -10.687642253091623 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "score": -10.767000829347708 } ] }, "f32_forward": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 3.3040432929992676, "winnerGap": 1.2296860218048096, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 3.3040432929992676 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 2.074357271194458 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": -0.3958527743816376 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": -0.4763363301753998 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": -8.615915298461914 }, { "rank": 6, "tokenId": 2709, "text": "Ob", "score": -9.968426704406738 }, { "rank": 7, "tokenId": 13552, "text": "subject", "score": -10.687644004821777 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "score": -10.76700496673584 } ] }, "f32_reverse": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 3.3040342330932617, "winnerGap": 1.2296819686889648, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 3.3040342330932617 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 2.074352264404297 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": -0.39585334062576294 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": -0.47633475065231323 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": -8.615920066833496 }, { "rank": 6, "tokenId": 2709, "text": "Ob", "score": -9.968420028686523 }, { "rank": 7, "tokenId": 13552, "text": "subject", "score": -10.687642097473145 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "score": -10.766987800598145 } ] }, "f32_tree64": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 3.3040359020233154, "winnerGap": 1.2296829223632812, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 3.3040359020233154 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 2.074352979660034 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": -0.39585208892822266 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": -0.4763369560241699 }, { "rank": 5, "tokenId": 2805, "text": "Un", "score": -8.615913391113281 }, { "rank": 6, "tokenId": 2709, "text": "Ob", "score": -9.968424797058105 }, { "rank": 7, "tokenId": 13552, "text": "subject", "score": -10.687643051147461 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "score": -10.767001152038574 } ] }, "f16_forward": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 3.34765625, "winnerGap": 1.24609375, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 3.34765625 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 2.1015625 }, { "rank": 3, "tokenId": 9277, "text": "False", "score": -0.388916015625 }, { "rank": 4, "tokenId": 4530, "text": "false", "score": -0.421630859375 }, { "rank": 5, "tokenId": 3321, "text": "De", "score": -5.87109375 }, { "rank": 6, "tokenId": 602, "text": "un", "score": -7.3125 }, { "rank": 7, "tokenId": 193634, "text": "philosoph", "score": -7.46484375 }, { "rank": 8, "tokenId": 2805, "text": "Un", "score": -8.5625 } ] } }, "winnerTextsByMode": { "exact": "true", "f32_forward": "true", "f32_reverse": "true", "f32_tree64": "true", "f16_forward": "true" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "true" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tf-torture-justified-oneword-c", "text": "No explanation. One word only.\nAnswer with exactly one word: true or false. Question: Torture is never justified. Answer:", "stableTopK": [ { "rank": 1, "tokenId": 3397, "text": "true", "logit": 13.91090202331543, "prob": 0.9873555302619934 }, { "rank": 2, "tokenId": 4339, "text": "True", "logit": 9.550576210021973, "prob": 0.012612702324986458 }, { "rank": 3, "tokenId": 4530, "text": "false", "logit": 3.478571653366089, "prob": 0.000029091765100019984 }, { "rank": 4, "tokenId": 9277, "text": "False", "logit": 0.6763270497322083, "prob": 0.0000017651054804446176 }, { "rank": 5, "tokenId": 57985, "text": "truth", "logit": -1.9196953773498535, "prob": 1.316232101089554e-7 }, { "rank": 6, "tokenId": 22369, "text": "TRUE", "logit": -2.443571090698242, "prob": 7.79500055614335e-8 }, { "rank": 7, "tokenId": 1847, "text": " true", "logit": -2.58013916015625, "prob": 6.799944429758398e-8 }, { "rank": 8, "tokenId": 106, "text": "", "logit": -2.9231514930725098, "prob": 4.825441024536303e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 15.06031102571442, "winnerGap": 5.165929122939822, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 15.06031102571442 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 9.894381902774597 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": 3.4942885602006264 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": 0.6764396490404643 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -1.92232010775332 }, { "rank": 6, "tokenId": 22369, "text": "TRUE", "score": -2.449000655110762 }, { "rank": 7, "tokenId": 1847, "text": " true", "score": -2.5865304674323824 }, { "rank": 8, "tokenId": 106, "text": "", "score": -2.932457629787873 } ] }, "f32_forward": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 15.060303688049316, "winnerGap": 5.1659345626831055, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 15.060303688049316 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 9.894369125366211 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": 3.4942948818206787 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": 0.6764428615570068 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -1.9223159551620483 }, { "rank": 6, "tokenId": 22369, "text": "TRUE", "score": -2.449000597000122 }, { "rank": 7, "tokenId": 1847, "text": " true", "score": -2.586531400680542 }, { "rank": 8, "tokenId": 106, "text": "", "score": -2.9324610233306885 } ] }, "f32_reverse": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 15.060303688049316, "winnerGap": 5.165925025939941, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 15.060303688049316 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 9.894378662109375 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": 3.4942898750305176 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": 0.6764418482780457 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -1.9223120212554932 }, { "rank": 6, "tokenId": 22369, "text": "TRUE", "score": -2.4490010738372803 }, { "rank": 7, "tokenId": 1847, "text": " true", "score": -2.586531400680542 }, { "rank": 8, "tokenId": 106, "text": "", "score": -2.932452917098999 } ] }, "f32_tree64": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 15.060311317443848, "winnerGap": 5.165929794311523, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 15.060311317443848 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 9.894381523132324 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": 3.4942879676818848 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": 0.6764392852783203 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -1.9223215579986572 }, { "rank": 6, "tokenId": 22369, "text": "TRUE", "score": -2.4489998817443848 }, { "rank": 7, "tokenId": 1847, "text": " true", "score": -2.5865306854248047 }, { "rank": 8, "tokenId": 106, "text": "", "score": -2.93245792388916 } ] }, "f16_forward": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 11.1015625, "winnerGap": 1.09375, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 11.1015625 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 10.0078125 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": 3.53515625 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": 0.63232421875 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -1.947265625 }, { "rank": 6, "tokenId": 1904, "text": "no", "score": -2.322265625 }, { "rank": 7, "tokenId": 197039, "text": " verdadeira", "score": -2.41015625 }, { "rank": 8, "tokenId": 22369, "text": "TRUE", "score": -2.44921875 } ] } }, "winnerTextsByMode": { "exact": "true", "f32_forward": "true", "f32_reverse": "true", "f32_tree64": "true", "f16_forward": "true" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "true" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tf-war-necessary-oneword-d", "text": "Choose one word only.\nAnswer with exactly one word: true or false. Question: War is sometimes necessary. Answer:", "stableTopK": [ { "rank": 1, "tokenId": 4339, "text": "True", "logit": 12.309347152709961, "prob": 0.7294605374336243 }, { "rank": 2, "tokenId": 3397, "text": "true", "logit": 11.317416191101074, "prob": 0.2705276608467102 }, { "rank": 3, "tokenId": 22369, "text": "TRUE", "logit": 1.1727253198623657, "prob": 0.000010627433766785543 }, { "rank": 4, "tokenId": 9277, "text": "False", "logit": -1.527626395225525, "prob": 7.139709055081767e-7 }, { "rank": 5, "tokenId": 4530, "text": "false", "logit": -3.158106565475464, "prob": 1.398208695491121e-7 }, { "rank": 6, "tokenId": 1847, "text": " true", "logit": -3.7379937171936035, "prob": 7.829431325490077e-8 }, { "rank": 7, "tokenId": 1018, "text": "**", "logit": -4.2013020515441895, "prob": 4.9262670387406615e-8 }, { "rank": 8, "tokenId": 6288, "text": " True", "logit": -4.588537693023682, "prob": 3.3445957114963676e-8 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.079581258859555, "winnerGap": 1.1741948475118065, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.079581258859555 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 11.905386411347749 }, { "rank": 3, "tokenId": 22369, "text": "TRUE", "score": 1.1733218032136605 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": -1.5289501071777636 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": -3.169849252399513 }, { "rank": 6, "tokenId": 1847, "text": " true", "score": -3.757520248188804 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -4.229095357600272 }, { "rank": 8, "tokenId": 6288, "text": " True", "score": -4.6248341189031095 } ] }, "f32_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.079586029052734, "winnerGap": 1.1741914749145508, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.079586029052734 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 11.905394554138184 }, { "rank": 3, "tokenId": 22369, "text": "TRUE", "score": 1.173322319984436 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": -1.528949499130249 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": -3.169849395751953 }, { "rank": 6, "tokenId": 1847, "text": " true", "score": -3.757512331008911 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -4.22909688949585 }, { "rank": 8, "tokenId": 6288, "text": " True", "score": -4.624831199645996 } ] }, "f32_reverse": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.079577445983887, "winnerGap": 1.1741971969604492, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.079577445983887 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 11.905380249023438 }, { "rank": 3, "tokenId": 22369, "text": "TRUE", "score": 1.1733251810073853 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": -1.5289520025253296 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": -3.16984486579895 }, { "rank": 6, "tokenId": 1847, "text": " true", "score": -3.7575204372406006 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -4.2290940284729 }, { "rank": 8, "tokenId": 6288, "text": " True", "score": -4.6248345375061035 } ] }, "f32_tree64": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.079581260681152, "winnerGap": 1.1741952896118164, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.079581260681152 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 11.905385971069336 }, { "rank": 3, "tokenId": 22369, "text": "TRUE", "score": 1.1733217239379883 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": -1.5289499759674072 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": -3.169848918914795 }, { "rank": 6, "tokenId": 1847, "text": " true", "score": -3.757519245147705 }, { "rank": 7, "tokenId": 1018, "text": "**", "score": -4.229095935821533 }, { "rank": 8, "tokenId": 6288, "text": " True", "score": -4.624831199645996 } ] }, "f16_forward": { "winnerTokenId": 4339, "winnerText": "True", "winnerScore": 13.109375, "winnerGap": 1.015625, "top": [ { "rank": 1, "tokenId": 4339, "text": "True", "score": 13.109375 }, { "rank": 2, "tokenId": 3397, "text": "true", "score": 12.09375 }, { "rank": 3, "tokenId": 22369, "text": "TRUE", "score": 1.1982421875 }, { "rank": 4, "tokenId": 9277, "text": "False", "score": -1.5341796875 }, { "rank": 5, "tokenId": 4530, "text": "false", "score": -3.1953125 }, { "rank": 6, "tokenId": 1847, "text": " true", "score": -3.771484375 }, { "rank": 7, "tokenId": 236772, "text": "-", "score": -3.873046875 }, { "rank": 8, "tokenId": 1018, "text": "**", "score": -3.974609375 } ] } }, "winnerTextsByMode": { "exact": "True", "f32_forward": "True", "f32_reverse": "True", "f32_tree64": "True", "f16_forward": "True" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "True" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "tf-water-wet-oneword-a", "text": "Answer with exactly one word only.\nAnswer with exactly one word: true or false. Question: Water is wet. Answer:", "stableTopK": [ { "rank": 1, "tokenId": 3397, "text": "true", "logit": 10.460183143615723, "prob": 0.9999038577079773 }, { "rank": 2, "tokenId": 4339, "text": "True", "logit": 1.200913667678833, "prob": 0.00009521570609649643 }, { "rank": 3, "tokenId": 4530, "text": "false", "logit": -3.5336484909057617, "prob": 8.365933013010363e-7 }, { "rank": 4, "tokenId": 1847, "text": " true", "logit": -7.247951984405518, "prob": 2.0389794741504375e-8 }, { "rank": 5, "tokenId": 57985, "text": "truth", "logit": -8.33574104309082, "prob": 6.87055923265234e-9 }, { "rank": 6, "tokenId": 106, "text": "", "logit": -9.222704887390137, "prob": 2.8300137788761504e-9 }, { "rank": 7, "tokenId": 9277, "text": "False", "logit": -9.298182487487793, "prob": 2.6242732431569493e-9 }, { "rank": 8, "tokenId": 2717, "text": "```", "logit": -9.547550201416016, "prob": 2.045078772994202e-9 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 10.91796226988535, "winnerGap": 9.716403797382913, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 10.91796226988535 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 1.2015584725024375 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": -3.5501288236179676 }, { "rank": 4, "tokenId": 1847, "text": " true", "score": -7.394129414552022 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -8.560778948234756 }, { "rank": 6, "tokenId": 106, "text": "", "score": -9.530930736933719 }, { "rank": 7, "tokenId": 9277, "text": "False", "score": -9.614350510820298 }, { "rank": 8, "tokenId": 2717, "text": "```", "score": -9.89101972214352 } ] }, "f32_forward": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 10.917962074279785, "winnerGap": 9.71640419960022, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 10.917962074279785 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 1.2015578746795654 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": -3.550133466720581 }, { "rank": 4, "tokenId": 1847, "text": " true", "score": -7.394131660461426 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -8.560779571533203 }, { "rank": 6, "tokenId": 106, "text": "", "score": -9.530933380126953 }, { "rank": 7, "tokenId": 9277, "text": "False", "score": -9.614356994628906 }, { "rank": 8, "tokenId": 2717, "text": "```", "score": -9.891016006469727 } ] }, "f32_reverse": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 10.917961120605469, "winnerGap": 9.71640133857727, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 10.917961120605469 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 1.2015597820281982 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": -3.550126791000366 }, { "rank": 4, "tokenId": 1847, "text": " true", "score": -7.394131183624268 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -8.56077766418457 }, { "rank": 6, "tokenId": 106, "text": "", "score": -9.530935287475586 }, { "rank": 7, "tokenId": 9277, "text": "False", "score": -9.614337921142578 }, { "rank": 8, "tokenId": 2717, "text": "```", "score": -9.891016006469727 } ] }, "f32_tree64": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 10.917962074279785, "winnerGap": 9.716402769088745, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 10.917962074279785 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 1.20155930519104 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": -3.55012845993042 }, { "rank": 4, "tokenId": 1847, "text": " true", "score": -7.394128799438477 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -8.560778617858887 }, { "rank": 6, "tokenId": 106, "text": "", "score": -9.530929565429688 }, { "rank": 7, "tokenId": 9277, "text": "False", "score": -9.614351272583008 }, { "rank": 8, "tokenId": 2717, "text": "```", "score": -9.891021728515625 } ] }, "f16_forward": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 11.03125, "winnerGap": 10.8031005859375, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 11.03125 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 0.2281494140625 }, { "rank": 3, "tokenId": 4530, "text": "false", "score": -3.541015625 }, { "rank": 4, "tokenId": 57985, "text": "truth", "score": -4.58203125 }, { "rank": 5, "tokenId": 28883, "text": "touch", "score": -4.91796875 }, { "rank": 6, "tokenId": 109416, "text": "demon", "score": -5.0546875 }, { "rank": 7, "tokenId": 69746, "text": "verd", "score": -6.296875 }, { "rank": 8, "tokenId": 511, "text": "is", "score": -6.7578125 } ] } }, "winnerTextsByMode": { "exact": "true", "f32_forward": "true", "f32_reverse": "true", "f32_tree64": "true", "f16_forward": "true" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "true" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "true-earth-sun-oneword-b", "text": "Respond with one word only.\nAnswer with exactly one word: true or false.\nStatement: The Earth orbits the Sun.\nAnswer:", "stableTopK": [ { "rank": 1, "tokenId": 3397, "text": "true", "logit": 18.18128204345703, "prob": 0.9999867081642151 }, { "rank": 2, "tokenId": 4339, "text": "True", "logit": 6.952526569366455, "prob": 0.000013286411558510736 }, { "rank": 3, "tokenId": 1847, "text": " true", "logit": -2.702712297439575, "prob": 8.515119165330987e-10 }, { "rank": 4, "tokenId": 236745, "text": "t", "logit": -3.2988712787628174, "prob": 4.691180777172121e-10 }, { "rank": 5, "tokenId": 57985, "text": "truth", "logit": -4.285802841186523, "prob": 1.748490380260037e-10 }, { "rank": 6, "tokenId": 106, "text": "", "logit": -5.49957275390625, "prob": 5.194331423319376e-11 }, { "rank": 7, "tokenId": 2717, "text": "```", "logit": -6.259413242340088, "prob": 2.429602005693532e-11 }, { "rank": 8, "tokenId": 4530, "text": "false", "logit": -6.480332851409912, "prob": 1.9480091151269363e-11 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 21.079291736093467, "winnerGap": 13.998118992633081, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 21.079291736093467 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 7.081172743460385 }, { "rank": 3, "tokenId": 1847, "text": " true", "score": -2.7100596438757805 }, { "rank": 4, "tokenId": 236745, "text": "t", "score": -3.3122653022803803 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -4.315317205909874 }, { "rank": 6, "tokenId": 106, "text": "", "score": -5.562454739211923 }, { "rank": 7, "tokenId": 2717, "text": "```", "score": -6.3526934091284595 }, { "rank": 8, "tokenId": 4530, "text": "false", "score": -6.5840479601781645 } ] }, "f32_forward": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 21.07928466796875, "winnerGap": 13.998108386993408, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 21.07928466796875 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 7.081176280975342 }, { "rank": 3, "tokenId": 1847, "text": " true", "score": -2.7100577354431152 }, { "rank": 4, "tokenId": 236745, "text": "t", "score": -3.312265634536743 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -4.315320014953613 }, { "rank": 6, "tokenId": 106, "text": "", "score": -5.562455177307129 }, { "rank": 7, "tokenId": 2717, "text": "```", "score": -6.352696418762207 }, { "rank": 8, "tokenId": 4530, "text": "false", "score": -6.584054470062256 } ] }, "f32_reverse": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 21.079267501831055, "winnerGap": 13.998095512390137, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 21.079267501831055 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 7.081171989440918 }, { "rank": 3, "tokenId": 1847, "text": " true", "score": -2.7100577354431152 }, { "rank": 4, "tokenId": 236745, "text": "t", "score": -3.3122646808624268 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -4.315318584442139 }, { "rank": 6, "tokenId": 106, "text": "", "score": -5.562455654144287 }, { "rank": 7, "tokenId": 2717, "text": "```", "score": -6.352693557739258 }, { "rank": 8, "tokenId": 4530, "text": "false", "score": -6.584048271179199 } ] }, "f32_tree64": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 21.07929229736328, "winnerGap": 13.998119354248047, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 21.07929229736328 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 7.081172943115234 }, { "rank": 3, "tokenId": 1847, "text": " true", "score": -2.7100610733032227 }, { "rank": 4, "tokenId": 236745, "text": "t", "score": -3.312264919281006 }, { "rank": 5, "tokenId": 57985, "text": "truth", "score": -4.315317153930664 }, { "rank": 6, "tokenId": 106, "text": "", "score": -5.5624542236328125 }, { "rank": 7, "tokenId": 2717, "text": "```", "score": -6.352693557739258 }, { "rank": 8, "tokenId": 4530, "text": "false", "score": -6.584050178527832 } ] }, "f16_forward": { "winnerTokenId": 3397, "winnerText": "true", "winnerScore": 21.1875, "winnerGap": 14.05078125, "top": [ { "rank": 1, "tokenId": 3397, "text": "true", "score": 21.1875 }, { "rank": 2, "tokenId": 4339, "text": "True", "score": 7.13671875 }, { "rank": 3, "tokenId": 1847, "text": " true", "score": -2.896484375 }, { "rank": 4, "tokenId": 236745, "text": "t", "score": -3.400390625 }, { "rank": 5, "tokenId": 3292, "text": "$\\", "score": -3.59765625 }, { "rank": 6, "tokenId": 22369, "text": "TRUE", "score": -3.71875 }, { "rank": 7, "tokenId": 57985, "text": "truth", "score": -4.3359375 }, { "rank": 8, "tokenId": 106, "text": "", "score": -5.54296875 } ] } }, "winnerTextsByMode": { "exact": "true", "f32_forward": "true", "f32_reverse": "true", "f32_tree64": "true", "f16_forward": "true" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "true" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "truth-lie-true-false-depends-oneword-c", "text": "No explanation. One word only.\nTrue, false, or depends: telling a painful truth is always kinder than a comforting lie", "stableTopK": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "logit": 12.511417388916016, "prob": 0.9996535778045654 }, { "rank": 2, "tokenId": 9277, "text": "False", "logit": 4.501962661743164, "prob": 0.000332190771587193 }, { "rank": 3, "tokenId": 113214, "text": "depends", "logit": 1.0860658884048462, "prob": 0.000010911448043771088 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "logit": -0.997079610824585, "prob": 0.0000013588885394710815 }, { "rank": 5, "tokenId": 3771, "text": "No", "logit": -1.7914456129074097, "prob": 6.140377308838652e-7 }, { "rank": 6, "tokenId": 4339, "text": "True", "logit": -1.9528553485870361, "prob": 5.22511356848554e-7 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "logit": -2.820298671722412, "prob": 2.1946731010302756e-7 }, { "rank": 8, "tokenId": 79629, "text": "Depending", "logit": -3.3602797985076904, "prob": 1.2789659820100496e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 13.323363633085222, "winnerGap": 8.787140512343058, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 13.323363633085222 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 4.536223120742164 }, { "rank": 3, "tokenId": 113214, "text": "depends", "score": 1.0865408119868247 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": -0.9974516569911067 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": -1.7935793161899838 }, { "rank": 6, "tokenId": 4339, "text": "True", "score": -1.9556239794108023 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "score": -2.828654927699901 }, { "rank": 8, "tokenId": 79629, "text": "Depending", "score": -3.374440808690359 } ] }, "f32_forward": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 13.32336711883545, "winnerGap": 8.78714656829834, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 13.32336711883545 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 4.536220550537109 }, { "rank": 3, "tokenId": 113214, "text": "depends", "score": 1.08653724193573 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": -0.9974514842033386 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": -1.7935794591903687 }, { "rank": 6, "tokenId": 4339, "text": "True", "score": -1.9556211233139038 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "score": -2.828652858734131 }, { "rank": 8, "tokenId": 79629, "text": "Depending", "score": -3.3744444847106934 } ] }, "f32_reverse": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 13.323363304138184, "winnerGap": 8.787139892578125, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 13.323363304138184 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 4.536223411560059 }, { "rank": 3, "tokenId": 113214, "text": "depends", "score": 1.086537480354309 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": -0.9974539279937744 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": -1.7935779094696045 }, { "rank": 6, "tokenId": 4339, "text": "True", "score": -1.9556235074996948 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "score": -2.8286569118499756 }, { "rank": 8, "tokenId": 79629, "text": "Depending", "score": -3.374429941177368 } ] }, "f32_tree64": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 13.3233642578125, "winnerGap": 8.787140846252441, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 13.3233642578125 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 4.536223411560059 }, { "rank": 3, "tokenId": 113214, "text": "depends", "score": 1.086540937423706 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": -0.9974513053894043 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": -1.7935795783996582 }, { "rank": 6, "tokenId": 4339, "text": "True", "score": -1.9556241035461426 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "score": -2.828655242919922 }, { "rank": 8, "tokenId": 79629, "text": "Depending", "score": -3.374439001083374 } ] }, "f16_forward": { "winnerTokenId": 179306, "winnerText": "Depends", "winnerScore": 13.25, "winnerGap": 8.69140625, "top": [ { "rank": 1, "tokenId": 179306, "text": "Depends", "score": 13.25 }, { "rank": 2, "tokenId": 9277, "text": "False", "score": 4.55859375 }, { "rank": 3, "tokenId": 113214, "text": "depends", "score": 1.216796875 }, { "rank": 4, "tokenId": 31900, "text": "Deb", "score": -0.7470703125 }, { "rank": 5, "tokenId": 4339, "text": "True", "score": -1.953125 }, { "rank": 6, "tokenId": 3771, "text": "No", "score": -2.037109375 }, { "rank": 7, "tokenId": 31049, "text": "Depend", "score": -2.8515625 }, { "rank": 8, "tokenId": 79629, "text": "Depending", "score": -3.390625 } ] } }, "winnerTextsByMode": { "exact": "Depends", "f32_forward": "Depends", "f32_reverse": "Depends", "f32_tree64": "Depends", "f16_forward": "Depends" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Depends" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "val-age-wisdom-oneword-d", "text": "Choose one word only.\nWisdom comes with age — always, sometimes, or rarely?", "stableTopK": [ { "rank": 1, "tokenId": 53787, "text": "Always", "logit": 18.0198974609375, "prob": 0.5647631883621216 }, { "rank": 2, "tokenId": 35650, "text": "Sometimes", "logit": 17.56746482849121, "prob": 0.35923394560813904 }, { "rank": 3, "tokenId": 1018, "text": "**", "logit": 15.738326072692871, "prob": 0.05767564848065376 }, { "rank": 4, "tokenId": 128280, "text": "Rare", "logit": 14.272581100463867, "prob": 0.013317648321390152 }, { "rank": 5, "tokenId": 32631, "text": "Never", "logit": 11.688883781433105, "prob": 0.0010054073063656688 }, { "rank": 6, "tokenId": 77685, "text": "Wis", "logit": 11.641464233398438, "prob": 0.0009588441462256014 }, { "rank": 7, "tokenId": 47236, "text": "always", "logit": 11.398452758789062, "prob": 0.0007519855280406773 }, { "rank": 8, "tokenId": 68441, "text": "sometimes", "logit": 11.117679595947266, "prob": 0.0005678992019966245 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 53787, "winnerText": "Always", "winnerScore": 20.82552069875131, "winnerGap": 0.6980426597701097, "top": [ { "rank": 1, "tokenId": 53787, "text": "Always", "score": 20.82552069875131 }, { "rank": 2, "tokenId": 35650, "text": "Sometimes", "score": 20.1274780389812 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 17.4804104712249 }, { "rank": 4, "tokenId": 128280, "text": "Rare", "score": 15.524394178208695 }, { "rank": 5, "tokenId": 32631, "text": "Never", "score": 12.340891320611345 }, { "rank": 6, "tokenId": 77685, "text": "Wis", "score": 12.285025346565511 }, { "rank": 7, "tokenId": 47236, "text": "always", "score": 11.999978258643758 }, { "rank": 8, "tokenId": 68441, "text": "sometimes", "score": 11.673179609575577 } ] }, "f32_forward": { "winnerTokenId": 53787, "winnerText": "Always", "winnerScore": 20.825542449951172, "winnerGap": 0.6980342864990234, "top": [ { "rank": 1, "tokenId": 53787, "text": "Always", "score": 20.825542449951172 }, { "rank": 2, "tokenId": 35650, "text": "Sometimes", "score": 20.12750816345215 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 17.48042106628418 }, { "rank": 4, "tokenId": 128280, "text": "Rare", "score": 15.524393081665039 }, { "rank": 5, "tokenId": 32631, "text": "Never", "score": 12.340907096862793 }, { "rank": 6, "tokenId": 77685, "text": "Wis", "score": 12.285016059875488 }, { "rank": 7, "tokenId": 47236, "text": "always", "score": 11.999977111816406 }, { "rank": 8, "tokenId": 68441, "text": "sometimes", "score": 11.673176765441895 } ] }, "f32_reverse": { "winnerTokenId": 53787, "winnerText": "Always", "winnerScore": 20.825525283813477, "winnerGap": 0.6980514526367188, "top": [ { "rank": 1, "tokenId": 53787, "text": "Always", "score": 20.825525283813477 }, { "rank": 2, "tokenId": 35650, "text": "Sometimes", "score": 20.127473831176758 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 17.480419158935547 }, { "rank": 4, "tokenId": 128280, "text": "Rare", "score": 15.524394035339355 }, { "rank": 5, "tokenId": 32631, "text": "Never", "score": 12.340889930725098 }, { "rank": 6, "tokenId": 77685, "text": "Wis", "score": 12.285025596618652 }, { "rank": 7, "tokenId": 47236, "text": "always", "score": 11.99997615814209 }, { "rank": 8, "tokenId": 68441, "text": "sometimes", "score": 11.673176765441895 } ] }, "f32_tree64": { "winnerTokenId": 53787, "winnerText": "Always", "winnerScore": 20.825519561767578, "winnerGap": 0.6980400085449219, "top": [ { "rank": 1, "tokenId": 53787, "text": "Always", "score": 20.825519561767578 }, { "rank": 2, "tokenId": 35650, "text": "Sometimes", "score": 20.127479553222656 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 17.480411529541016 }, { "rank": 4, "tokenId": 128280, "text": "Rare", "score": 15.524394035339355 }, { "rank": 5, "tokenId": 32631, "text": "Never", "score": 12.340890884399414 }, { "rank": 6, "tokenId": 77685, "text": "Wis", "score": 12.285026550292969 }, { "rank": 7, "tokenId": 47236, "text": "always", "score": 11.999979019165039 }, { "rank": 8, "tokenId": 68441, "text": "sometimes", "score": 11.673179626464844 } ] }, "f16_forward": { "winnerTokenId": 53787, "winnerText": "Always", "winnerScore": 20.875, "winnerGap": 0.90625, "top": [ { "rank": 1, "tokenId": 53787, "text": "Always", "score": 20.875 }, { "rank": 2, "tokenId": 35650, "text": "Sometimes", "score": 19.96875 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 17.546875 }, { "rank": 4, "tokenId": 128280, "text": "Rare", "score": 15.8671875 }, { "rank": 5, "tokenId": 32631, "text": "Never", "score": 12.3984375 }, { "rank": 6, "tokenId": 77685, "text": "Wis", "score": 12.3046875 }, { "rank": 7, "tokenId": 47236, "text": "always", "score": 12.09375 }, { "rank": 8, "tokenId": 68441, "text": "sometimes", "score": 11.703125 } ] } }, "winnerTextsByMode": { "exact": "Always", "f32_forward": "Always", "f32_reverse": "Always", "f32_tree64": "Always", "f16_forward": "Always" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Always" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "val-art-offensive-oneword-a", "text": "Answer with exactly one word only.\nArt that offends should be protected or restricted. It should be", "stableTopK": [ { "rank": 1, "tokenId": 5024, "text": "Reg", "logit": 14.634437561035156, "prob": 0.7917659878730774 }, { "rank": 2, "tokenId": 49388, "text": "Protected", "logit": 12.834991455078125, "prob": 0.13095055520534515 }, { "rank": 3, "tokenId": 45523, "text": "regulated", "logit": 11.260830879211426, "prob": 0.02713051624596119 }, { "rank": 4, "tokenId": 26584, "text": "protected", "logit": 10.648076057434082, "prob": 0.014700837433338165 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "logit": 9.643501281738281, "prob": 0.0053834510035812855 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "logit": 9.460648536682129, "prob": 0.004483826924115419 }, { "rank": 7, "tokenId": 1584, "text": "Cont", "logit": 9.347167015075684, "prob": 0.00400280486792326 }, { "rank": 8, "tokenId": 154572, "text": "Discuss", "logit": 9.301607131958008, "prob": 0.0038245294708758593 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 5024, "winnerText": "Reg", "winnerScore": 15.99565134191701, "winnerGap": 2.278417792814693, "top": [ { "rank": 1, "tokenId": 5024, "text": "Reg", "score": 15.99565134191701 }, { "rank": 2, "tokenId": 49388, "text": "Protected", "score": 13.717233549102318 }, { "rank": 3, "tokenId": 45523, "text": "regulated", "score": 11.83946442860261 }, { "rank": 4, "tokenId": 26584, "text": "protected", "score": 11.13239432330127 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 9.997899850945444 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 9.794418414651403 }, { "rank": 7, "tokenId": 1584, "text": "Cont", "score": 9.668573238533725 }, { "rank": 8, "tokenId": 154572, "text": "Discuss", "score": 9.618140861219036 } ] }, "f32_forward": { "winnerTokenId": 5024, "winnerText": "Reg", "winnerScore": 15.99561882019043, "winnerGap": 2.2784061431884766, "top": [ { "rank": 1, "tokenId": 5024, "text": "Reg", "score": 15.99561882019043 }, { "rank": 2, "tokenId": 49388, "text": "Protected", "score": 13.717212677001953 }, { "rank": 3, "tokenId": 45523, "text": "regulated", "score": 11.839445114135742 }, { "rank": 4, "tokenId": 26584, "text": "protected", "score": 11.132393836975098 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 9.997896194458008 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 9.794422149658203 }, { "rank": 7, "tokenId": 1584, "text": "Cont", "score": 9.668569564819336 }, { "rank": 8, "tokenId": 154572, "text": "Discuss", "score": 9.618136405944824 } ] }, "f32_reverse": { "winnerTokenId": 5024, "winnerText": "Reg", "winnerScore": 15.99565601348877, "winnerGap": 2.2784290313720703, "top": [ { "rank": 1, "tokenId": 5024, "text": "Reg", "score": 15.99565601348877 }, { "rank": 2, "tokenId": 49388, "text": "Protected", "score": 13.7172269821167 }, { "rank": 3, "tokenId": 45523, "text": "regulated", "score": 11.83946418762207 }, { "rank": 4, "tokenId": 26584, "text": "protected", "score": 11.132396697998047 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 9.997901916503906 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 9.79442024230957 }, { "rank": 7, "tokenId": 1584, "text": "Cont", "score": 9.668575286865234 }, { "rank": 8, "tokenId": 154572, "text": "Discuss", "score": 9.618144035339355 } ] }, "f32_tree64": { "winnerTokenId": 5024, "winnerText": "Reg", "winnerScore": 15.99565315246582, "winnerGap": 2.27841854095459, "top": [ { "rank": 1, "tokenId": 5024, "text": "Reg", "score": 15.99565315246582 }, { "rank": 2, "tokenId": 49388, "text": "Protected", "score": 13.71723461151123 }, { "rank": 3, "tokenId": 45523, "text": "regulated", "score": 11.839462280273438 }, { "rank": 4, "tokenId": 26584, "text": "protected", "score": 11.132394790649414 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 9.997900009155273 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 9.794418334960938 }, { "rank": 7, "tokenId": 1584, "text": "Cont", "score": 9.668573379516602 }, { "rank": 8, "tokenId": 154572, "text": "Discuss", "score": 9.61814022064209 } ] }, "f16_forward": { "winnerTokenId": 5024, "winnerText": "Reg", "winnerScore": 16.171875, "winnerGap": 2.484375, "top": [ { "rank": 1, "tokenId": 5024, "text": "Reg", "score": 16.171875 }, { "rank": 2, "tokenId": 49388, "text": "Protected", "score": 13.6875 }, { "rank": 3, "tokenId": 45523, "text": "regulated", "score": 11.8984375 }, { "rank": 4, "tokenId": 26584, "text": "protected", "score": 11.046875 }, { "rank": 5, "tokenId": 31900, "text": "Deb", "score": 10.0546875 }, { "rank": 6, "tokenId": 1584, "text": "Cont", "score": 9.7109375 }, { "rank": 7, "tokenId": 154572, "text": "Discuss", "score": 9.53125 }, { "rank": 8, "tokenId": 198633, "text": "Controlled", "score": 9.1796875 } ] } }, "winnerTextsByMode": { "exact": "Reg", "f32_forward": "Reg", "f32_reverse": "Reg", "f32_tree64": "Reg", "f16_forward": "Reg" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Reg" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "val-forgetting-healing-oneword-b", "text": "Respond with one word only.\nForgetting is a form of healing or a form of denial. It is", "stableTopK": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "logit": 17.16947364807129, "prob": 0.9081375002861023 }, { "rank": 2, "tokenId": 22186, "text": "Both", "logit": 14.351068496704102, "prob": 0.054216790944337845 }, { "rank": 3, "tokenId": 22472, "text": "Den", "logit": 12.40041446685791, "prob": 0.007708600722253323 }, { "rank": 4, "tokenId": 74895, "text": "Escape", "logit": 11.924607276916504, "prob": 0.004789995029568672 }, { "rank": 5, "tokenId": 113054, "text": "Parad", "logit": 11.852209091186523, "prob": 0.00445546442642808 }, { "rank": 6, "tokenId": 236788, "text": "E", "logit": 11.785290718078613, "prob": 0.004167069215327501 }, { "rank": 7, "tokenId": 11242, "text": "Process", "logit": 11.379169464111328, "prob": 0.002776223700493574 }, { "rank": 8, "tokenId": 236776, "text": "A", "logit": 11.00184154510498, "prob": 0.001903632073663175 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 19.528793369837672, "winnerGap": 3.90278098199979, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 19.528793369837672 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 15.626012387837882 }, { "rank": 3, "tokenId": 22472, "text": "Den", "score": 13.189242277085285 }, { "rank": 4, "tokenId": 74895, "text": "Escape", "score": 12.619827258429531 }, { "rank": 5, "tokenId": 113054, "text": "Parad", "score": 12.533939880293204 }, { "rank": 6, "tokenId": 236788, "text": "E", "score": 12.454717284392444 }, { "rank": 7, "tokenId": 11242, "text": "Process", "score": 11.977449666837884 }, { "rank": 8, "tokenId": 236776, "text": "A", "score": 11.53912111991718 } ] }, "f32_forward": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 19.528783798217773, "winnerGap": 3.902777671813965, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 19.528783798217773 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 15.626006126403809 }, { "rank": 3, "tokenId": 22472, "text": "Den", "score": 13.189239501953125 }, { "rank": 4, "tokenId": 74895, "text": "Escape", "score": 12.61982250213623 }, { "rank": 5, "tokenId": 113054, "text": "Parad", "score": 12.533937454223633 }, { "rank": 6, "tokenId": 236788, "text": "E", "score": 12.454730987548828 }, { "rank": 7, "tokenId": 11242, "text": "Process", "score": 11.977447509765625 }, { "rank": 8, "tokenId": 236776, "text": "A", "score": 11.539111137390137 } ] }, "f32_reverse": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 19.528791427612305, "winnerGap": 3.90277099609375, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 19.528791427612305 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 15.626020431518555 }, { "rank": 3, "tokenId": 22472, "text": "Den", "score": 13.189238548278809 }, { "rank": 4, "tokenId": 74895, "text": "Escape", "score": 12.619833946228027 }, { "rank": 5, "tokenId": 113054, "text": "Parad", "score": 12.533935546875 }, { "rank": 6, "tokenId": 236788, "text": "E", "score": 12.454721450805664 }, { "rank": 7, "tokenId": 11242, "text": "Process", "score": 11.97745418548584 }, { "rank": 8, "tokenId": 236776, "text": "A", "score": 11.539119720458984 } ] }, "f32_tree64": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 19.52879524230957, "winnerGap": 3.902782440185547, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 19.52879524230957 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 15.626012802124023 }, { "rank": 3, "tokenId": 22472, "text": "Den", "score": 13.18924331665039 }, { "rank": 4, "tokenId": 74895, "text": "Escape", "score": 12.619827270507812 }, { "rank": 5, "tokenId": 113054, "text": "Parad", "score": 12.533940315246582 }, { "rank": 6, "tokenId": 236788, "text": "E", "score": 12.454715728759766 }, { "rank": 7, "tokenId": 11242, "text": "Process", "score": 11.977449417114258 }, { "rank": 8, "tokenId": 236776, "text": "A", "score": 11.539121627807617 } ] }, "f16_forward": { "winnerTokenId": 55144, "winnerText": "Amb", "winnerScore": 19.40625, "winnerGap": 3.7890625, "top": [ { "rank": 1, "tokenId": 55144, "text": "Amb", "score": 19.40625 }, { "rank": 2, "tokenId": 22186, "text": "Both", "score": 15.6171875 }, { "rank": 3, "tokenId": 22472, "text": "Den", "score": 12.75 }, { "rank": 4, "tokenId": 74895, "text": "Escape", "score": 12.5859375 }, { "rank": 5, "tokenId": 236788, "text": "E", "score": 12.4375 }, { "rank": 6, "tokenId": 11242, "text": "Process", "score": 12.0703125 }, { "rank": 7, "tokenId": 236776, "text": "A", "score": 11.5859375 }, { "rank": 8, "tokenId": 43109, "text": "Transition", "score": 11.203125 } ] } }, "winnerTextsByMode": { "exact": "Amb", "f32_forward": "Amb", "f32_reverse": "Amb", "f32_tree64": "Amb", "f16_forward": "Amb" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Amb" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "val-happiness-meaning-oneword-c", "text": "No explanation. One word only.\nA meaningful life is more important than a happy one — agree or disagree?", "stableTopK": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "logit": 12.127398490905762, "prob": 0.7198486924171448 }, { "rank": 2, "tokenId": 233950, "text": "Disagree", "logit": 11.170052528381348, "prob": 0.27635741233825684 }, { "rank": 3, "tokenId": 4830, "text": "Dis", "logit": 6.284204483032227, "prob": 0.0020872491877526045 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "logit": 5.552258491516113, "prob": 0.0010039085755124688 }, { "rank": 5, "tokenId": 3771, "text": "No", "logit": 3.6107177734375, "prob": 0.00014404354442376643 }, { "rank": 6, "tokenId": 115863, "text": "Analyze", "logit": 3.4244189262390137, "prob": 0.00011955980153288692 }, { "rank": 7, "tokenId": 236847, "text": "V", "logit": 2.5056025981903076, "prob": 0.00004770329178427346 }, { "rank": 8, "tokenId": 21288, "text": "Subject", "logit": 2.4625184535980225, "prob": 0.00004569167867884971 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 208903, "winnerText": "Agree", "winnerScore": 12.861442573092589, "winnerGap": 1.1275081744640048, "top": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "score": 12.861442573092589 }, { "rank": 2, "tokenId": 233950, "text": "Disagree", "score": 11.733934398628584 }, { "rank": 3, "tokenId": 4830, "text": "Dis", "score": 6.378620264829493 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": 5.616989938691418 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": 3.628306990979013 }, { "rank": 6, "tokenId": 115863, "text": "Analyze", "score": 3.439408131606818 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": 2.5114496225353875 }, { "rank": 8, "tokenId": 21288, "text": "Subject", "score": 2.4680706928492953 } ] }, "f32_forward": { "winnerTokenId": 208903, "winnerText": "Agree", "winnerScore": 12.861429214477539, "winnerGap": 1.127516746520996, "top": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "score": 12.861429214477539 }, { "rank": 2, "tokenId": 233950, "text": "Disagree", "score": 11.733912467956543 }, { "rank": 3, "tokenId": 4830, "text": "Dis", "score": 6.378619194030762 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": 5.616988182067871 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": 3.628300666809082 }, { "rank": 6, "tokenId": 115863, "text": "Analyze", "score": 3.4394078254699707 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": 2.5114495754241943 }, { "rank": 8, "tokenId": 21288, "text": "Subject", "score": 2.468066692352295 } ] }, "f32_reverse": { "winnerTokenId": 208903, "winnerText": "Agree", "winnerScore": 12.861442565917969, "winnerGap": 1.127507209777832, "top": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "score": 12.861442565917969 }, { "rank": 2, "tokenId": 233950, "text": "Disagree", "score": 11.733935356140137 }, { "rank": 3, "tokenId": 4830, "text": "Dis", "score": 6.378615856170654 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": 5.616989612579346 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": 3.628302812576294 }, { "rank": 6, "tokenId": 115863, "text": "Analyze", "score": 3.4394092559814453 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": 2.5114479064941406 }, { "rank": 8, "tokenId": 21288, "text": "Subject", "score": 2.4680697917938232 } ] }, "f32_tree64": { "winnerTokenId": 208903, "winnerText": "Agree", "winnerScore": 12.861442565917969, "winnerGap": 1.1275081634521484, "top": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "score": 12.861442565917969 }, { "rank": 2, "tokenId": 233950, "text": "Disagree", "score": 11.73393440246582 }, { "rank": 3, "tokenId": 4830, "text": "Dis", "score": 6.378619194030762 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": 5.616990089416504 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": 3.628309488296509 }, { "rank": 6, "tokenId": 115863, "text": "Analyze", "score": 3.439408540725708 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": 2.5114498138427734 }, { "rank": 8, "tokenId": 21288, "text": "Subject", "score": 2.4680700302124023 } ] }, "f16_forward": { "winnerTokenId": 208903, "winnerText": "Agree", "winnerScore": 12.8125, "winnerGap": 4.96484375, "top": [ { "rank": 1, "tokenId": 208903, "text": "Agree", "score": 12.8125 }, { "rank": 2, "tokenId": 233950, "text": "Disagree", "score": 7.84765625 }, { "rank": 3, "tokenId": 4830, "text": "Dis", "score": 6.5078125 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": 5.609375 }, { "rank": 5, "tokenId": 3771, "text": "No", "score": 3.630859375 }, { "rank": 6, "tokenId": 115863, "text": "Analyze", "score": 3.40234375 }, { "rank": 7, "tokenId": 236847, "text": "V", "score": 2.603515625 }, { "rank": 8, "tokenId": 21288, "text": "Subject", "score": 2.48828125 } ] } }, "winnerTextsByMode": { "exact": "Agree", "f32_forward": "Agree", "f32_reverse": "Agree", "f32_tree64": "Agree", "f16_forward": "Agree" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Agree" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "val-individual-collective-oneword-d", "text": "Choose one word only.\nIndividual rights or collective good — which comes first?", "stableTopK": [ { "rank": 1, "tokenId": 2094, "text": "This", "logit": 17.799991607666016, "prob": 0.7946107387542725 }, { "rank": 2, "tokenId": 48569, "text": "Individual", "logit": 15.99034309387207, "prob": 0.13008704781532288 }, { "rank": 3, "tokenId": 3637, "text": "Context", "logit": 14.186087608337402, "prob": 0.02141193114221096 }, { "rank": 4, "tokenId": 1018, "text": "**", "logit": 13.789642333984375, "prob": 0.014403958804905415 }, { "rank": 5, "tokenId": 206958, "text": "Collective", "logit": 13.52226734161377, "prob": 0.011024587787687778 }, { "rank": 6, "tokenId": 818, "text": "The", "logit": 13.11142349243164, "prob": 0.007310299668461084 }, { "rank": 7, "tokenId": 2704, "text": "Ind", "logit": 13.09139347076416, "prob": 0.007165330462157726 }, { "rank": 8, "tokenId": 179306, "text": "Depends", "logit": 12.725916862487793, "prob": 0.004971778020262718 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 2094, "winnerText": "This", "winnerScore": 20.483838058753488, "winnerGap": 2.6535663783108134, "top": [ { "rank": 1, "tokenId": 2094, "text": "This", "score": 20.483838058753488 }, { "rank": 2, "tokenId": 48569, "text": "Individual", "score": 17.830271680442674 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 15.412791892744096 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 14.906204141177003 }, { "rank": 5, "tokenId": 206958, "text": "Collective", "score": 14.568939066206909 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 14.05726559572608 }, { "rank": 7, "tokenId": 2704, "text": "Ind", "score": 14.032511407918633 }, { "rank": 8, "tokenId": 179306, "text": "Depends", "score": 13.583971257027237 } ] }, "f32_forward": { "winnerTokenId": 2094, "winnerText": "This", "winnerScore": 20.48383140563965, "winnerGap": 2.6535511016845703, "top": [ { "rank": 1, "tokenId": 2094, "text": "This", "score": 20.48383140563965 }, { "rank": 2, "tokenId": 48569, "text": "Individual", "score": 17.830280303955078 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 15.412799835205078 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 14.906197547912598 }, { "rank": 5, "tokenId": 206958, "text": "Collective", "score": 14.56894588470459 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 14.057267189025879 }, { "rank": 7, "tokenId": 2704, "text": "Ind", "score": 14.032509803771973 }, { "rank": 8, "tokenId": 179306, "text": "Depends", "score": 13.583974838256836 } ] }, "f32_reverse": { "winnerTokenId": 2094, "winnerText": "This", "winnerScore": 20.483837127685547, "winnerGap": 2.653566360473633, "top": [ { "rank": 1, "tokenId": 2094, "text": "This", "score": 20.483837127685547 }, { "rank": 2, "tokenId": 48569, "text": "Individual", "score": 17.830270767211914 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 15.412795066833496 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 14.90620231628418 }, { "rank": 5, "tokenId": 206958, "text": "Collective", "score": 14.56893253326416 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 14.057265281677246 }, { "rank": 7, "tokenId": 2704, "text": "Ind", "score": 14.032505989074707 }, { "rank": 8, "tokenId": 179306, "text": "Depends", "score": 13.583977699279785 } ] }, "f32_tree64": { "winnerTokenId": 2094, "winnerText": "This", "winnerScore": 20.483837127685547, "winnerGap": 2.653564453125, "top": [ { "rank": 1, "tokenId": 2094, "text": "This", "score": 20.483837127685547 }, { "rank": 2, "tokenId": 48569, "text": "Individual", "score": 17.830272674560547 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 15.412792205810547 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 14.90620231628418 }, { "rank": 5, "tokenId": 206958, "text": "Collective", "score": 14.568940162658691 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 14.057266235351562 }, { "rank": 7, "tokenId": 2704, "text": "Ind", "score": 14.032510757446289 }, { "rank": 8, "tokenId": 179306, "text": "Depends", "score": 13.583971977233887 } ] }, "f16_forward": { "winnerTokenId": 48569, "winnerText": "Individual", "winnerScore": 17.796875, "winnerGap": 1.34375, "top": [ { "rank": 1, "tokenId": 48569, "text": "Individual", "score": 17.796875 }, { "rank": 2, "tokenId": 2094, "text": "This", "score": 16.453125 }, { "rank": 3, "tokenId": 3637, "text": "Context", "score": 15.5078125 }, { "rank": 4, "tokenId": 1018, "text": "**", "score": 14.8984375 }, { "rank": 5, "tokenId": 206958, "text": "Collective", "score": 14.4765625 }, { "rank": 6, "tokenId": 2704, "text": "Ind", "score": 14.1171875 }, { "rank": 7, "tokenId": 179306, "text": "Depends", "score": 13.5625 }, { "rank": 8, "tokenId": 21288, "text": "Subject", "score": 13.125 } ] } }, "winnerTextsByMode": { "exact": "This", "f32_forward": "This", "f32_reverse": "This", "f32_tree64": "This", "f16_forward": "Individual" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "This", "Individual" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 2094, 563, 496, 9760, 30426, 532, 52491, 62557 ], "decodedText": "This is a classic ethical and philosophical dilemma", "steps": [ { "step": 0, "tokenId": 2094, "text": "This" }, { "step": 1, "tokenId": 563, "text": " is" }, { "step": 2, "tokenId": 496, "text": " a" }, { "step": 3, "tokenId": 9760, "text": " classic" }, { "step": 4, "tokenId": 30426, "text": " ethical" }, { "step": 5, "tokenId": 532, "text": " and" }, { "step": 6, "tokenId": 52491, "text": " philosophical" }, { "step": 7, "tokenId": 62557, "text": " dilemma" } ] }, "f16_forward": { "tokenIds": [ 48569, 106 ], "decodedText": "Individual", "steps": [ { "step": 0, "tokenId": 48569, "text": "Individual" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 7, "differingStepCount": 8, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "val-loyalty-honesty-oneword-a", "text": "Answer with exactly one word only.\nWhen loyalty and honesty conflict, you should choose", "stableTopK": [ { "rank": 1, "tokenId": 29579, "text": "Integr", "logit": 12.338476181030273, "prob": 0.9702756404876709 }, { "rank": 2, "tokenId": 201053, "text": "Ethics", "logit": 8.333846092224121, "prob": 0.01768912561237812 }, { "rank": 3, "tokenId": 37892, "text": "Hon", "logit": 7.0103349685668945, "prob": 0.004708827473223209 }, { "rank": 4, "tokenId": 74041, "text": "Truth", "logit": 6.882362365722656, "prob": 0.004143191501498222 }, { "rank": 5, "tokenId": 27505, "text": "Mor", "logit": 6.353060245513916, "prob": 0.0024404057767242193 }, { "rank": 6, "tokenId": 31728, "text": "Eth", "logit": 3.966996192932129, "prob": 0.0002244954666821286 }, { "rank": 7, "tokenId": 3778, "text": "Pr", "logit": 3.2662227153778076, "prob": 0.00011139494745293632 }, { "rank": 8, "tokenId": 64672, "text": "Princ", "logit": 2.9778854846954346, "prob": 0.00008349149720743299 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 29579, "winnerText": "Integr", "winnerScore": 13.114617501520627, "winnerGap": 4.555887855984121, "top": [ { "rank": 1, "tokenId": 29579, "text": "Integr", "score": 13.114617501520627 }, { "rank": 2, "tokenId": 201053, "text": "Ethics", "score": 8.558729645536506 }, { "rank": 3, "tokenId": 37892, "text": "Hon", "score": 7.14228683386181 }, { "rank": 4, "tokenId": 74041, "text": "Truth", "score": 7.007064978121527 }, { "rank": 5, "tokenId": 27505, "text": "Mor", "score": 6.4506732460076535 }, { "rank": 6, "tokenId": 31728, "text": "Eth", "score": 3.9903679642423064 }, { "rank": 7, "tokenId": 3778, "text": "Pr", "score": 3.279223698842003 }, { "rank": 8, "tokenId": 64672, "text": "Princ", "score": 2.9877248975878175 } ] }, "f32_forward": { "winnerTokenId": 29579, "winnerText": "Integr", "winnerScore": 13.11462116241455, "winnerGap": 4.55588436126709, "top": [ { "rank": 1, "tokenId": 29579, "text": "Integr", "score": 13.11462116241455 }, { "rank": 2, "tokenId": 201053, "text": "Ethics", "score": 8.558736801147461 }, { "rank": 3, "tokenId": 37892, "text": "Hon", "score": 7.142286777496338 }, { "rank": 4, "tokenId": 74041, "text": "Truth", "score": 7.00706148147583 }, { "rank": 5, "tokenId": 27505, "text": "Mor", "score": 6.450679302215576 }, { "rank": 6, "tokenId": 31728, "text": "Eth", "score": 3.990365505218506 }, { "rank": 7, "tokenId": 3778, "text": "Pr", "score": 3.279228925704956 }, { "rank": 8, "tokenId": 64672, "text": "Princ", "score": 2.9877266883850098 } ] }, "f32_reverse": { "winnerTokenId": 29579, "winnerText": "Integr", "winnerScore": 13.11462688446045, "winnerGap": 4.555895805358887, "top": [ { "rank": 1, "tokenId": 29579, "text": "Integr", "score": 13.11462688446045 }, { "rank": 2, "tokenId": 201053, "text": "Ethics", "score": 8.558731079101562 }, { "rank": 3, "tokenId": 37892, "text": "Hon", "score": 7.142282962799072 }, { "rank": 4, "tokenId": 74041, "text": "Truth", "score": 7.0070648193359375 }, { "rank": 5, "tokenId": 27505, "text": "Mor", "score": 6.450667858123779 }, { "rank": 6, "tokenId": 31728, "text": "Eth", "score": 3.9903724193573 }, { "rank": 7, "tokenId": 3778, "text": "Pr", "score": 3.279223918914795 }, { "rank": 8, "tokenId": 64672, "text": "Princ", "score": 2.9877243041992188 } ] }, "f32_tree64": { "winnerTokenId": 29579, "winnerText": "Integr", "winnerScore": 13.114617347717285, "winnerGap": 4.555887222290039, "top": [ { "rank": 1, "tokenId": 29579, "text": "Integr", "score": 13.114617347717285 }, { "rank": 2, "tokenId": 201053, "text": "Ethics", "score": 8.558730125427246 }, { "rank": 3, "tokenId": 37892, "text": "Hon", "score": 7.142286777496338 }, { "rank": 4, "tokenId": 74041, "text": "Truth", "score": 7.007065296173096 }, { "rank": 5, "tokenId": 27505, "text": "Mor", "score": 6.4506731033325195 }, { "rank": 6, "tokenId": 31728, "text": "Eth", "score": 3.990367889404297 }, { "rank": 7, "tokenId": 3778, "text": "Pr", "score": 3.279224157333374 }, { "rank": 8, "tokenId": 64672, "text": "Princ", "score": 2.9877240657806396 } ] }, "f16_forward": { "winnerTokenId": 29579, "winnerText": "Integr", "winnerScore": 13.1640625, "winnerGap": 4.640625, "top": [ { "rank": 1, "tokenId": 29579, "text": "Integr", "score": 13.1640625 }, { "rank": 2, "tokenId": 201053, "text": "Ethics", "score": 8.5234375 }, { "rank": 3, "tokenId": 37892, "text": "Hon", "score": 7.15625 }, { "rank": 4, "tokenId": 27505, "text": "Mor", "score": 6.42578125 }, { "rank": 5, "tokenId": 31728, "text": "Eth", "score": 4.03515625 }, { "rank": 6, "tokenId": 3778, "text": "Pr", "score": 3.20703125 }, { "rank": 7, "tokenId": 74041, "text": "Truth", "score": 3.0390625 }, { "rank": 8, "tokenId": 64672, "text": "Princ", "score": 2.9921875 } ] } }, "winnerTextsByMode": { "exact": "Integr", "f32_forward": "Integr", "f32_reverse": "Integr", "f32_tree64": "Integr", "f16_forward": "Integr" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Integr" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "val-meritocracy-oneword-b", "text": "Respond with one word only.\nMeritocracy is real or a myth. It is", "stableTopK": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "logit": 11.133609771728516, "prob": 0.9927041530609131 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "logit": 5.447918891906738, "prob": 0.0033694240264594555 }, { "rank": 3, "tokenId": 1584, "text": "Cont", "logit": 5.305968284606934, "prob": 0.002923528430983424 }, { "rank": 4, "tokenId": 14977, "text": "Question", "logit": 2.98343563079834, "prob": 0.00028657889924943447 }, { "rank": 5, "tokenId": 53417, "text": "deb", "logit": 2.5291783809661865, "prob": 0.00018195448501501232 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "logit": 1.9200879335403442, "prob": 0.00009895508992485702 }, { "rank": 7, "tokenId": 32258, "text": "Concept", "logit": 1.45609712600708, "prob": 0.00006221992953214794 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "logit": 1.1392772197723389, "prob": 0.00004532485036179423 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 11.691656498014577, "winnerGap": 6.182637625111353, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 11.691656498014577 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 5.509018872903225 }, { "rank": 3, "tokenId": 1584, "text": "Cont", "score": 5.362359867228619 }, { "rank": 4, "tokenId": 14977, "text": "Question", "score": 2.9933321671501254 }, { "rank": 5, "tokenId": 53417, "text": "deb", "score": 2.5351933575806074 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 1.922720374327735 }, { "rank": 7, "tokenId": 32258, "text": "Concept", "score": 1.457243607450228 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 1.139822776776394 } ] }, "f32_forward": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 11.691657066345215, "winnerGap": 6.182653903961182, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 11.691657066345215 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 5.509003162384033 }, { "rank": 3, "tokenId": 1584, "text": "Cont", "score": 5.362351894378662 }, { "rank": 4, "tokenId": 14977, "text": "Question", "score": 2.993326187133789 }, { "rank": 5, "tokenId": 53417, "text": "deb", "score": 2.5351924896240234 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 1.9227298498153687 }, { "rank": 7, "tokenId": 32258, "text": "Concept", "score": 1.4572383165359497 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 1.1398205757141113 } ] }, "f32_reverse": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 11.691659927368164, "winnerGap": 6.1826395988464355, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 11.691659927368164 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 5.5090203285217285 }, { "rank": 3, "tokenId": 1584, "text": "Cont", "score": 5.362360000610352 }, { "rank": 4, "tokenId": 14977, "text": "Question", "score": 2.993330717086792 }, { "rank": 5, "tokenId": 53417, "text": "deb", "score": 2.53519868850708 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 1.9227147102355957 }, { "rank": 7, "tokenId": 32258, "text": "Concept", "score": 1.4572405815124512 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 1.1398197412490845 } ] }, "f32_tree64": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 11.691657066345215, "winnerGap": 6.182637691497803, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 11.691657066345215 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 5.509019374847412 }, { "rank": 3, "tokenId": 1584, "text": "Cont", "score": 5.36236047744751 }, { "rank": 4, "tokenId": 14977, "text": "Question", "score": 2.9933323860168457 }, { "rank": 5, "tokenId": 53417, "text": "deb", "score": 2.5351943969726562 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 1.9227213859558105 }, { "rank": 7, "tokenId": 32258, "text": "Concept", "score": 1.457242727279663 }, { "rank": 8, "tokenId": 55144, "text": "Amb", "score": 1.1398224830627441 } ] }, "f16_forward": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 7.70703125, "winnerGap": 2.234375, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 7.70703125 }, { "rank": 2, "tokenId": 48333, "text": "Complex", "score": 5.47265625 }, { "rank": 3, "tokenId": 1584, "text": "Cont", "score": 5.41796875 }, { "rank": 4, "tokenId": 14977, "text": "Question", "score": 2.99609375 }, { "rank": 5, "tokenId": 53417, "text": "deb", "score": 2.466796875 }, { "rank": 6, "tokenId": 55144, "text": "Amb", "score": 1.1552734375 }, { "rank": 7, "tokenId": 236796, "text": "D", "score": 0.9013671875 }, { "rank": 8, "tokenId": 77712, "text": " debated", "score": 0.78515625 } ] } }, "winnerTextsByMode": { "exact": "Deb", "f32_forward": "Deb", "f32_reverse": "Deb", "f32_tree64": "Deb", "f16_forward": "Deb" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Deb" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "val-normal-defined-oneword-c", "text": "No explanation. One word only.\nNormal is defined by the majority or doesn't exist. It", "stableTopK": [ { "rank": 1, "tokenId": 11146, "text": "Rel", "logit": 6.629489421844482, "prob": 0.1741737276315689 }, { "rank": 2, "tokenId": 221040, "text": "Majority", "logit": 6.4094109535217285, "prob": 0.13976672291755676 }, { "rank": 3, "tokenId": 55751, "text": "Relative", "logit": 6.378198623657227, "prob": 0.1354716569185257 }, { "rank": 4, "tokenId": 234254, "text": "Anomaly", "logit": 6.2097392082214355, "prob": 0.11446887999773026 }, { "rank": 5, "tokenId": 98680, "text": "Statistical", "logit": 6.0516815185546875, "prob": 0.09773359447717667 }, { "rank": 6, "tokenId": 113054, "text": "Parad", "logit": 5.960383892059326, "prob": 0.08920595049858093 }, { "rank": 7, "tokenId": 29428, "text": "Norm", "logit": 4.586281776428223, "prob": 0.022575058043003082 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "logit": 4.384592533111572, "prob": 0.018451698124408722 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 11146, "winnerText": "Rel", "winnerScore": 6.740681202511489, "winnerGap": 0.23099383349966374, "top": [ { "rank": 1, "tokenId": 11146, "text": "Rel", "score": 6.740681202511489 }, { "rank": 2, "tokenId": 221040, "text": "Majority", "score": 6.509687369011825 }, { "rank": 3, "tokenId": 55751, "text": "Relative", "score": 6.476995911414193 }, { "rank": 4, "tokenId": 234254, "text": "Anomaly", "score": 6.30077998084051 }, { "rank": 5, "tokenId": 98680, "text": "Statistical", "score": 6.135833258560924 }, { "rank": 6, "tokenId": 113054, "text": "Parad", "score": 6.0407178902430445 }, { "rank": 7, "tokenId": 29428, "text": "Norm", "score": 4.622522122681371 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "score": 4.416217747666234 } ] }, "f32_forward": { "winnerTokenId": 11146, "winnerText": "Rel", "winnerScore": 6.74068546295166, "winnerGap": 0.23099184036254883, "top": [ { "rank": 1, "tokenId": 11146, "text": "Rel", "score": 6.74068546295166 }, { "rank": 2, "tokenId": 221040, "text": "Majority", "score": 6.509693622589111 }, { "rank": 3, "tokenId": 55751, "text": "Relative", "score": 6.476996421813965 }, { "rank": 4, "tokenId": 234254, "text": "Anomaly", "score": 6.300784111022949 }, { "rank": 5, "tokenId": 98680, "text": "Statistical", "score": 6.135834693908691 }, { "rank": 6, "tokenId": 113054, "text": "Parad", "score": 6.04071569442749 }, { "rank": 7, "tokenId": 29428, "text": "Norm", "score": 4.622523307800293 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "score": 4.4162116050720215 } ] }, "f32_reverse": { "winnerTokenId": 11146, "winnerText": "Rel", "winnerScore": 6.740682125091553, "winnerGap": 0.23099279403686523, "top": [ { "rank": 1, "tokenId": 11146, "text": "Rel", "score": 6.740682125091553 }, { "rank": 2, "tokenId": 221040, "text": "Majority", "score": 6.5096893310546875 }, { "rank": 3, "tokenId": 55751, "text": "Relative", "score": 6.476993560791016 }, { "rank": 4, "tokenId": 234254, "text": "Anomaly", "score": 6.300780773162842 }, { "rank": 5, "tokenId": 98680, "text": "Statistical", "score": 6.135834693908691 }, { "rank": 6, "tokenId": 113054, "text": "Parad", "score": 6.040720462799072 }, { "rank": 7, "tokenId": 29428, "text": "Norm", "score": 4.622524261474609 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "score": 4.416215896606445 } ] }, "f32_tree64": { "winnerTokenId": 11146, "winnerText": "Rel", "winnerScore": 6.7406816482543945, "winnerGap": 0.23099422454833984, "top": [ { "rank": 1, "tokenId": 11146, "text": "Rel", "score": 6.7406816482543945 }, { "rank": 2, "tokenId": 221040, "text": "Majority", "score": 6.509687423706055 }, { "rank": 3, "tokenId": 55751, "text": "Relative", "score": 6.476995944976807 }, { "rank": 4, "tokenId": 234254, "text": "Anomaly", "score": 6.300779819488525 }, { "rank": 5, "tokenId": 98680, "text": "Statistical", "score": 6.135832786560059 }, { "rank": 6, "tokenId": 113054, "text": "Parad", "score": 6.040717601776123 }, { "rank": 7, "tokenId": 29428, "text": "Norm", "score": 4.62252140045166 }, { "rank": 8, "tokenId": 2704, "text": "Ind", "score": 4.4162187576293945 } ] }, "f16_forward": { "winnerTokenId": 221040, "winnerText": "Majority", "winnerScore": 6.47265625, "winnerGap": 0.05078125, "top": [ { "rank": 1, "tokenId": 221040, "text": "Majority", "score": 6.47265625 }, { "rank": 2, "tokenId": 55751, "text": "Relative", "score": 6.421875 }, { "rank": 3, "tokenId": 234254, "text": "Anomaly", "score": 6.2578125 }, { "rank": 4, "tokenId": 98680, "text": "Statistical", "score": 6.09765625 }, { "rank": 5, "tokenId": 2704, "text": "Ind", "score": 4.4921875 }, { "rank": 6, "tokenId": 55144, "text": "Amb", "score": 4.41015625 }, { "rank": 7, "tokenId": 123307, "text": "Contrast", "score": 4.23828125 }, { "rank": 8, "tokenId": 24005, "text": "Abs", "score": 4.22265625 } ] } }, "winnerTextsByMode": { "exact": "Rel", "f32_forward": "Rel", "f32_reverse": "Rel", "f32_tree64": "Rel", "f16_forward": "Majority" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Rel", "Majority" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 11146, 8919, 1929 ], "decodedText": "Relativism", "steps": [ { "step": 0, "tokenId": 11146, "text": "Rel" }, { "step": 1, "tokenId": 8919, "text": "ativ" }, { "step": 2, "tokenId": 1929, "text": "ism" } ] }, "f16_forward": { "tokenIds": [ 221040, 106 ], "decodedText": "Majority", "steps": [ { "step": 0, "tokenId": 221040, "text": "Majority" }, { "step": 1, "tokenId": 106, "text": "" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 2, "differingStepCount": 3, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "val-privacy-security-oneword-d", "text": "Choose one word only.\nPrivacy matters more than security, or security matters more. The answer is", "stableTopK": [ { "rank": 1, "tokenId": 19120, "text": "Security", "logit": 23.36330795288086, "prob": 0.3898407816886902 }, { "rank": 2, "tokenId": 1018, "text": "**", "logit": 23.313749313354492, "prob": 0.3709917366504669 }, { "rank": 3, "tokenId": 52581, "text": "Privacy", "logit": 22.641244888305664, "prob": 0.18936480581760406 }, { "rank": 4, "tokenId": 2094, "text": "This", "logit": 19.83717155456543, "prob": 0.011468474753201008 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "logit": 19.792770385742188, "prob": 0.010970400646328926 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "logit": 19.46298599243164, "prob": 0.007888581603765488 }, { "rank": 7, "tokenId": 818, "text": "The", "logit": 19.206539154052734, "prob": 0.006104154512286186 }, { "rank": 8, "tokenId": 95889, "text": "Neither", "logit": 18.753633499145508, "prob": 0.003880887757986784 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 19120, "winnerText": "Security", "winnerScore": 31.267634078952952, "winnerGap": 0.12551083537713126, "top": [ { "rank": 1, "tokenId": 19120, "text": "Security", "score": 31.267634078952952 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 31.14212324357582 }, { "rank": 3, "tokenId": 52581, "text": "Privacy", "score": 29.514135312921738 }, { "rank": 4, "tokenId": 2094, "text": "This", "score": 23.850355987039308 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 23.771600374283224 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 23.194951373178213 }, { "rank": 7, "tokenId": 818, "text": "The", "score": 22.75629086636927 }, { "rank": 8, "tokenId": 95889, "text": "Neither", "score": 22.001018295934614 } ] }, "f32_forward": { "winnerTokenId": 19120, "winnerText": "Security", "winnerScore": 31.267635345458984, "winnerGap": 0.1255035400390625, "top": [ { "rank": 1, "tokenId": 19120, "text": "Security", "score": 31.267635345458984 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 31.142131805419922 }, { "rank": 3, "tokenId": 52581, "text": "Privacy", "score": 29.514122009277344 }, { "rank": 4, "tokenId": 2094, "text": "This", "score": 23.850332260131836 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 23.7716064453125 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 23.194929122924805 }, { "rank": 7, "tokenId": 818, "text": "The", "score": 22.756290435791016 }, { "rank": 8, "tokenId": 95889, "text": "Neither", "score": 22.001005172729492 } ] }, "f32_reverse": { "winnerTokenId": 19120, "winnerText": "Security", "winnerScore": 31.26763343811035, "winnerGap": 0.12550926208496094, "top": [ { "rank": 1, "tokenId": 19120, "text": "Security", "score": 31.26763343811035 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 31.14212417602539 }, { "rank": 3, "tokenId": 52581, "text": "Privacy", "score": 29.514114379882812 }, { "rank": 4, "tokenId": 2094, "text": "This", "score": 23.85035514831543 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 23.771596908569336 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 23.19492530822754 }, { "rank": 7, "tokenId": 818, "text": "The", "score": 22.756282806396484 }, { "rank": 8, "tokenId": 95889, "text": "Neither", "score": 22.000995635986328 } ] }, "f32_tree64": { "winnerTokenId": 19120, "winnerText": "Security", "winnerScore": 31.26763343811035, "winnerGap": 0.12551116943359375, "top": [ { "rank": 1, "tokenId": 19120, "text": "Security", "score": 31.26763343811035 }, { "rank": 2, "tokenId": 1018, "text": "**", "score": 31.142122268676758 }, { "rank": 3, "tokenId": 52581, "text": "Privacy", "score": 29.514135360717773 }, { "rank": 4, "tokenId": 2094, "text": "This", "score": 23.850357055664062 }, { "rank": 5, "tokenId": 179306, "text": "Depends", "score": 23.7716007232666 }, { "rank": 6, "tokenId": 21288, "text": "Subject", "score": 23.194950103759766 }, { "rank": 7, "tokenId": 818, "text": "The", "score": 22.75629234313965 }, { "rank": 8, "tokenId": 95889, "text": "Neither", "score": 22.001018524169922 } ] }, "f16_forward": { "winnerTokenId": 1018, "winnerText": "**", "winnerScore": 31.203125, "winnerGap": 1.6875, "top": [ { "rank": 1, "tokenId": 1018, "text": "**", "score": 31.203125 }, { "rank": 2, "tokenId": 52581, "text": "Privacy", "score": 29.515625 }, { "rank": 3, "tokenId": 2094, "text": "This", "score": 24.140625 }, { "rank": 4, "tokenId": 179306, "text": "Depends", "score": 23.75 }, { "rank": 5, "tokenId": 21288, "text": "Subject", "score": 23.40625 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 22.828125 }, { "rank": 7, "tokenId": 95889, "text": "Neither", "score": 21.9375 }, { "rank": 8, "tokenId": 3637, "text": "Context", "score": 21.484375 } ] } }, "winnerTextsByMode": { "exact": "Security", "f32_forward": "Security", "f32_reverse": "Security", "f32_tree64": "Security", "f16_forward": "**" }, "uniqueWinnerCount": 2, "uniqueWinners": [ "Security", "**" ], "f16VsF32WinnerChanged": true, "f32OrderingDisagreement": false, "branches": { "f32_forward": { "tokenIds": [ 19120, 106 ], "decodedText": "Security", "steps": [ { "step": 0, "tokenId": 19120, "text": "Security" }, { "step": 1, "tokenId": 106, "text": "" } ] }, "f16_forward": { "tokenIds": [ 1018, 52581, 1018 ], "decodedText": "**Privacy**", "steps": [ { "step": 0, "tokenId": 1018, "text": "**" }, { "step": 1, "tokenId": 52581, "text": "Privacy" }, { "step": 2, "tokenId": 1018, "text": "**" } ] } }, "branchComparison": { "firstDifferentStep": 0, "lastDifferentStep": 2, "differingStepCount": 3, "healedAtStep": null, "persistsThroughEnd": true } }, { "id": "val-profit-ethics-oneword-a", "text": "Answer with exactly one word only.\nA company's primary duty is to shareholders or to society. It is to", "stableTopK": [ { "rank": 1, "tokenId": 26247, "text": "Share", "logit": 13.752735137939453, "prob": 0.9340615272521973 }, { "rank": 2, "tokenId": 23788, "text": "share", "logit": 11.064190864562988, "prob": 0.06349734216928482 }, { "rank": 3, "tokenId": 123197, "text": "Society", "logit": 7.609778881072998, "prob": 0.0020068895537406206 }, { "rank": 4, "tokenId": 111423, "text": "Soci", "logit": 5.541585445404053, "prob": 0.00025369884679093957 }, { "rank": 5, "tokenId": 43370, "text": "Prim", "logit": 4.408426761627197, "prob": 0.000081694706750568 }, { "rank": 6, "tokenId": 150062, "text": "society", "logit": 3.5686066150665283, "prob": 0.000035274810215923935 }, { "rank": 7, "tokenId": 22186, "text": "Both", "logit": 2.9148659706115723, "prob": 0.000018346308934269473 }, { "rank": 8, "tokenId": 50227, "text": "Benef", "logit": 2.1530275344848633, "prob": 0.000008564193194615655 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 26247, "winnerText": "Share", "winnerScore": 14.859446248493768, "winnerGap": 3.248225544058389, "top": [ { "rank": 1, "tokenId": 26247, "text": "Share", "score": 14.859446248493768 }, { "rank": 2, "tokenId": 23788, "text": "share", "score": 11.611220704435379 }, { "rank": 3, "tokenId": 123197, "text": "Society", "score": 7.779598096300033 }, { "rank": 4, "tokenId": 111423, "text": "Soci", "score": 5.605936105661385 }, { "rank": 5, "tokenId": 43370, "text": "Prim", "score": 4.440575666068817 }, { "rank": 6, "tokenId": 150062, "text": "society", "score": 3.5855860716807584 }, { "rank": 7, "tokenId": 22186, "text": "Both", "score": 2.9240897571224878 }, { "rank": 8, "tokenId": 50227, "text": "Benef", "score": 2.156738457696271 } ] }, "f32_forward": { "winnerTokenId": 26247, "winnerText": "Share", "winnerScore": 14.859455108642578, "winnerGap": 3.248231887817383, "top": [ { "rank": 1, "tokenId": 26247, "text": "Share", "score": 14.859455108642578 }, { "rank": 2, "tokenId": 23788, "text": "share", "score": 11.611223220825195 }, { "rank": 3, "tokenId": 123197, "text": "Society", "score": 7.779613018035889 }, { "rank": 4, "tokenId": 111423, "text": "Soci", "score": 5.60593843460083 }, { "rank": 5, "tokenId": 43370, "text": "Prim", "score": 4.440569877624512 }, { "rank": 6, "tokenId": 150062, "text": "society", "score": 3.585587501525879 }, { "rank": 7, "tokenId": 22186, "text": "Both", "score": 2.9240975379943848 }, { "rank": 8, "tokenId": 50227, "text": "Benef", "score": 2.156736135482788 } ] }, "f32_reverse": { "winnerTokenId": 26247, "winnerText": "Share", "winnerScore": 14.859445571899414, "winnerGap": 3.248223304748535, "top": [ { "rank": 1, "tokenId": 26247, "text": "Share", "score": 14.859445571899414 }, { "rank": 2, "tokenId": 23788, "text": "share", "score": 11.611222267150879 }, { "rank": 3, "tokenId": 123197, "text": "Society", "score": 7.779602527618408 }, { "rank": 4, "tokenId": 111423, "text": "Soci", "score": 5.605935096740723 }, { "rank": 5, "tokenId": 43370, "text": "Prim", "score": 4.440569877624512 }, { "rank": 6, "tokenId": 150062, "text": "society", "score": 3.5855915546417236 }, { "rank": 7, "tokenId": 22186, "text": "Both", "score": 2.9240951538085938 }, { "rank": 8, "tokenId": 50227, "text": "Benef", "score": 2.156737804412842 } ] }, "f32_tree64": { "winnerTokenId": 26247, "winnerText": "Share", "winnerScore": 14.85944652557373, "winnerGap": 3.248225212097168, "top": [ { "rank": 1, "tokenId": 26247, "text": "Share", "score": 14.85944652557373 }, { "rank": 2, "tokenId": 23788, "text": "share", "score": 11.611221313476562 }, { "rank": 3, "tokenId": 123197, "text": "Society", "score": 7.779599189758301 }, { "rank": 4, "tokenId": 111423, "text": "Soci", "score": 5.605936527252197 }, { "rank": 5, "tokenId": 43370, "text": "Prim", "score": 4.440575122833252 }, { "rank": 6, "tokenId": 150062, "text": "society", "score": 3.5855860710144043 }, { "rank": 7, "tokenId": 22186, "text": "Both", "score": 2.924088954925537 }, { "rank": 8, "tokenId": 50227, "text": "Benef", "score": 2.1567392349243164 } ] }, "f16_forward": { "winnerTokenId": 26247, "winnerText": "Share", "winnerScore": 14.734375, "winnerGap": 6.90625, "top": [ { "rank": 1, "tokenId": 26247, "text": "Share", "score": 14.734375 }, { "rank": 2, "tokenId": 123197, "text": "Society", "score": 7.828125 }, { "rank": 3, "tokenId": 23788, "text": "share", "score": 7.65625 }, { "rank": 4, "tokenId": 111423, "text": "Soci", "score": 5.60546875 }, { "rank": 5, "tokenId": 43370, "text": "Prim", "score": 4.51953125 }, { "rank": 6, "tokenId": 150062, "text": "society", "score": 3.390625 }, { "rank": 7, "tokenId": 22186, "text": "Both", "score": 2.931640625 }, { "rank": 8, "tokenId": 50227, "text": "Benef", "score": 2.23046875 } ] } }, "winnerTextsByMode": { "exact": "Share", "f32_forward": "Share", "f32_reverse": "Share", "f32_tree64": "Share", "f16_forward": "Share" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Share" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "val-silence-complicity-oneword-b", "text": "Respond with one word only.\nSilence in the face of injustice is complicity or self-preservation. It is", "stableTopK": [ { "rank": 1, "tokenId": 1558, "text": "Com", "logit": 19.933502197265625, "prob": 0.9755610227584839 }, { "rank": 2, "tokenId": 854, "text": "com", "logit": 16.076189041137695, "prob": 0.020608415827155113 }, { "rank": 3, "tokenId": 186741, "text": "Silence", "logit": 13.443784713745117, "prob": 0.0014818557538092136 }, { "rank": 4, "tokenId": 70408, "text": "Compliance", "logit": 11.95962142944336, "prob": 0.0003359247639309615 }, { "rank": 5, "tokenId": 24040, "text": "Accept", "logit": 11.882426261901855, "prob": 0.00031096863676793873 }, { "rank": 6, "tokenId": 236776, "text": "A", "logit": 11.483844757080078, "prob": 0.00020874441543128341 }, { "rank": 7, "tokenId": 8891, "text": "Comp", "logit": 11.12655258178711, "prob": 0.00014603091403841972 }, { "rank": 8, "tokenId": 9901, "text": " complic", "logit": 10.794084548950195, "prob": 0.00010472630674485117 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 1558, "winnerText": "Com", "winnerScore": 24.022176647678755, "winnerGap": 6.071741425581489, "top": [ { "rank": 1, "tokenId": 1558, "text": "Com", "score": 24.022176647678755 }, { "rank": 2, "tokenId": 854, "text": "com", "score": 17.950435222097266 }, { "rank": 3, "tokenId": 186741, "text": "Silence", "score": 14.470591285750245 }, { "rank": 4, "tokenId": 70408, "text": "Compliance", "score": 12.661430143502342 }, { "rank": 5, "tokenId": 24040, "text": "Accept", "score": 12.569759380440365 }, { "rank": 6, "tokenId": 236776, "text": "A", "score": 12.099902708110905 }, { "rank": 7, "tokenId": 8891, "text": "Comp", "score": 11.683466689717818 }, { "rank": 8, "tokenId": 9901, "text": " complic", "score": 11.299781676188783 } ] }, "f32_forward": { "winnerTokenId": 1558, "winnerText": "Com", "winnerScore": 24.02214813232422, "winnerGap": 6.071714401245117, "top": [ { "rank": 1, "tokenId": 1558, "text": "Com", "score": 24.02214813232422 }, { "rank": 2, "tokenId": 854, "text": "com", "score": 17.9504337310791 }, { "rank": 3, "tokenId": 186741, "text": "Silence", "score": 14.470602989196777 }, { "rank": 4, "tokenId": 70408, "text": "Compliance", "score": 12.66143798828125 }, { "rank": 5, "tokenId": 24040, "text": "Accept", "score": 12.569755554199219 }, { "rank": 6, "tokenId": 236776, "text": "A", "score": 12.099899291992188 }, { "rank": 7, "tokenId": 8891, "text": "Comp", "score": 11.683479309082031 }, { "rank": 8, "tokenId": 9901, "text": " complic", "score": 11.29977035522461 } ] }, "f32_reverse": { "winnerTokenId": 1558, "winnerText": "Com", "winnerScore": 24.02217674255371, "winnerGap": 6.071739196777344, "top": [ { "rank": 1, "tokenId": 1558, "text": "Com", "score": 24.02217674255371 }, { "rank": 2, "tokenId": 854, "text": "com", "score": 17.950437545776367 }, { "rank": 3, "tokenId": 186741, "text": "Silence", "score": 14.470589637756348 }, { "rank": 4, "tokenId": 70408, "text": "Compliance", "score": 12.661437034606934 }, { "rank": 5, "tokenId": 24040, "text": "Accept", "score": 12.569756507873535 }, { "rank": 6, "tokenId": 236776, "text": "A", "score": 12.099889755249023 }, { "rank": 7, "tokenId": 8891, "text": "Comp", "score": 11.68346118927002 }, { "rank": 8, "tokenId": 9901, "text": " complic", "score": 11.299783706665039 } ] }, "f32_tree64": { "winnerTokenId": 1558, "winnerText": "Com", "winnerScore": 24.022174835205078, "winnerGap": 6.071739196777344, "top": [ { "rank": 1, "tokenId": 1558, "text": "Com", "score": 24.022174835205078 }, { "rank": 2, "tokenId": 854, "text": "com", "score": 17.950435638427734 }, { "rank": 3, "tokenId": 186741, "text": "Silence", "score": 14.470592498779297 }, { "rank": 4, "tokenId": 70408, "text": "Compliance", "score": 12.661430358886719 }, { "rank": 5, "tokenId": 24040, "text": "Accept", "score": 12.569759368896484 }, { "rank": 6, "tokenId": 236776, "text": "A", "score": 12.099903106689453 }, { "rank": 7, "tokenId": 8891, "text": "Comp", "score": 11.683465957641602 }, { "rank": 8, "tokenId": 9901, "text": " complic", "score": 11.299779891967773 } ] }, "f16_forward": { "winnerTokenId": 1558, "winnerText": "Com", "winnerScore": 24.03125, "winnerGap": 6.046875, "top": [ { "rank": 1, "tokenId": 1558, "text": "Com", "score": 24.03125 }, { "rank": 2, "tokenId": 854, "text": "com", "score": 17.984375 }, { "rank": 3, "tokenId": 186741, "text": "Silence", "score": 14.5 }, { "rank": 4, "tokenId": 70408, "text": "Compliance", "score": 12.6796875 }, { "rank": 5, "tokenId": 24040, "text": "Accept", "score": 12.671875 }, { "rank": 6, "tokenId": 236776, "text": "A", "score": 11.8359375 }, { "rank": 7, "tokenId": 8891, "text": "Comp", "score": 11.7421875 }, { "rank": 8, "tokenId": 8653, "text": "Pass", "score": 11.5234375 } ] } }, "winnerTextsByMode": { "exact": "Com", "f32_forward": "Com", "f32_reverse": "Com", "f32_tree64": "Com", "f16_forward": "Com" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Com" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "val-tradition-progress-oneword-c", "text": "No explanation. One word only.\nTradition or progress: a society should prioritize", "stableTopK": [ { "rank": 1, "tokenId": 20558, "text": "Progress", "logit": 9.59786319732666, "prob": 0.9997869729995728 }, { "rank": 2, "tokenId": 16717, "text": "progress", "logit": 0.6580050587654114, "prob": 0.00013103170203976333 }, { "rank": 3, "tokenId": 79655, "text": "Trad", "logit": -0.424106240272522, "prob": 0.00004440393240656704 }, { "rank": 4, "tokenId": 25846, "text": "Balance", "logit": -1.0471588373184204, "prob": 0.000023814041924197227 }, { "rank": 5, "tokenId": 62554, "text": "Adapt", "logit": -3.085472822189331, "prob": 0.000003101734364463482 }, { "rank": 6, "tokenId": 5609, "text": " progress", "logit": -3.831984519958496, "prob": 0.0000014702754924655892 }, { "rank": 7, "tokenId": 3778, "text": "Pr", "logit": -4.33337926864624, "prob": 8.905242339096731e-7 }, { "rank": 8, "tokenId": 22186, "text": "Both", "logit": -4.4407243728637695, "prob": 7.99882798219187e-7 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 20558, "winnerText": "Progress", "winnerScore": 9.947031522026393, "winnerGap": 9.288921952849478, "top": [ { "rank": 1, "tokenId": 20558, "text": "Progress", "score": 9.947031522026393 }, { "rank": 2, "tokenId": 16717, "text": "progress", "score": 0.658109569176915 }, { "rank": 3, "tokenId": 79655, "text": "Trad", "score": -0.42413775399901965 }, { "rank": 4, "tokenId": 25846, "text": "Balance", "score": -1.0475864682478715 }, { "rank": 5, "tokenId": 62554, "text": "Adapt", "score": -3.096421295693176 }, { "rank": 6, "tokenId": 5609, "text": " progress", "score": -3.8530343286542603 }, { "rank": 7, "tokenId": 3778, "text": "Pr", "score": -4.363897727963135 }, { "rank": 8, "tokenId": 22186, "text": "Both", "score": -4.47359016606363 } ] }, "f32_forward": { "winnerTokenId": 20558, "winnerText": "Progress", "winnerScore": 9.947030067443848, "winnerGap": 9.288921356201172, "top": [ { "rank": 1, "tokenId": 20558, "text": "Progress", "score": 9.947030067443848 }, { "rank": 2, "tokenId": 16717, "text": "progress", "score": 0.6581087112426758 }, { "rank": 3, "tokenId": 79655, "text": "Trad", "score": -0.42414167523384094 }, { "rank": 4, "tokenId": 25846, "text": "Balance", "score": -1.0475903749465942 }, { "rank": 5, "tokenId": 62554, "text": "Adapt", "score": -3.096423864364624 }, { "rank": 6, "tokenId": 5609, "text": " progress", "score": -3.853034257888794 }, { "rank": 7, "tokenId": 3778, "text": "Pr", "score": -4.363898277282715 }, { "rank": 8, "tokenId": 22186, "text": "Both", "score": -4.47358512878418 } ] }, "f32_reverse": { "winnerTokenId": 20558, "winnerText": "Progress", "winnerScore": 9.947019577026367, "winnerGap": 9.288911402225494, "top": [ { "rank": 1, "tokenId": 20558, "text": "Progress", "score": 9.947019577026367 }, { "rank": 2, "tokenId": 16717, "text": "progress", "score": 0.6581081748008728 }, { "rank": 3, "tokenId": 79655, "text": "Trad", "score": -0.42413875460624695 }, { "rank": 4, "tokenId": 25846, "text": "Balance", "score": -1.0475873947143555 }, { "rank": 5, "tokenId": 62554, "text": "Adapt", "score": -3.0964205265045166 }, { "rank": 6, "tokenId": 5609, "text": " progress", "score": -3.8530404567718506 }, { "rank": 7, "tokenId": 3778, "text": "Pr", "score": -4.363898754119873 }, { "rank": 8, "tokenId": 22186, "text": "Both", "score": -4.473584175109863 } ] }, "f32_tree64": { "winnerTokenId": 20558, "winnerText": "Progress", "winnerScore": 9.947030067443848, "winnerGap": 9.288921177387238, "top": [ { "rank": 1, "tokenId": 20558, "text": "Progress", "score": 9.947030067443848 }, { "rank": 2, "tokenId": 16717, "text": "progress", "score": 0.6581088900566101 }, { "rank": 3, "tokenId": 79655, "text": "Trad", "score": -0.4241372346878052 }, { "rank": 4, "tokenId": 25846, "text": "Balance", "score": -1.0475859642028809 }, { "rank": 5, "tokenId": 62554, "text": "Adapt", "score": -3.096421241760254 }, { "rank": 6, "tokenId": 5609, "text": " progress", "score": -3.8530335426330566 }, { "rank": 7, "tokenId": 3778, "text": "Pr", "score": -4.363898277282715 }, { "rank": 8, "tokenId": 22186, "text": "Both", "score": -4.47359037399292 } ] }, "f16_forward": { "winnerTokenId": 20558, "winnerText": "Progress", "winnerScore": 10.0390625, "winnerGap": 9.3671875, "top": [ { "rank": 1, "tokenId": 20558, "text": "Progress", "score": 10.0390625 }, { "rank": 2, "tokenId": 16717, "text": "progress", "score": 0.671875 }, { "rank": 3, "tokenId": 5609, "text": " progress", "score": 0.10711669921875 }, { "rank": 4, "tokenId": 79655, "text": "Trad", "score": -0.366943359375 }, { "rank": 5, "tokenId": 25846, "text": "Balance", "score": -0.775390625 }, { "rank": 6, "tokenId": 62554, "text": "Adapt", "score": -3.07421875 }, { "rank": 7, "tokenId": 173959, "text": "Necess", "score": -3.6953125 }, { "rank": 8, "tokenId": 24627, "text": "Techn", "score": -4.078125 } ] } }, "winnerTextsByMode": { "exact": "Progress", "f32_forward": "Progress", "f32_reverse": "Progress", "f32_tree64": "Progress", "f16_forward": "Progress" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Progress" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null }, { "id": "virus-true-false-debated-oneword-d", "text": "Choose one word only.\nTrue, false, or debated: a virus is alive", "stableTopK": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "logit": 18.717893600463867, "prob": 0.9646995663642883 }, { "rank": 2, "tokenId": 53417, "text": "deb", "logit": 15.206891059875488, "prob": 0.02881263941526413 }, { "rank": 3, "tokenId": 1018, "text": "**", "logit": 13.684083938598633, "prob": 0.006284002214670181 }, { "rank": 4, "tokenId": 4339, "text": "True", "logit": 10.030548095703125, "prob": 0.00016275183588732034 }, { "rank": 5, "tokenId": 9277, "text": "False", "logit": 7.93416690826416, "prob": 0.000020002262317575514 }, { "rank": 6, "tokenId": 818, "text": "The", "logit": 6.267500400543213, "prob": 0.0000037779398098791717 }, { "rank": 7, "tokenId": 1584, "text": "Cont", "logit": 6.079125881195068, "prob": 0.0000031292843232222367 }, { "rank": 8, "tokenId": 893, "text": "de", "logit": 5.8612565994262695, "prob": 0.000002516665972507326 } ], "lmHead": { "dtype": "f16", "layout": "row", "shape": [ 262144, 1536 ] }, "modes": { "exact": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 21.94242461211096, "winnerGap": 5.18610851529915, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 21.94242461211096 }, { "rank": 2, "tokenId": 53417, "text": "deb", "score": 16.75631609681181 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 14.772641442767569 }, { "rank": 4, "tokenId": 4339, "text": "True", "score": 10.431590163354912 }, { "rank": 5, "tokenId": 9277, "text": "False", "score": 8.127324638857935 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 6.36114649627255 }, { "rank": 7, "tokenId": 1584, "text": "Cont", "score": 6.164442359665738 }, { "rank": 8, "tokenId": 893, "text": "de", "score": 5.937590013101828 } ] }, "f32_forward": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 21.942401885986328, "winnerGap": 5.186077117919922, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 21.942401885986328 }, { "rank": 2, "tokenId": 53417, "text": "deb", "score": 16.756324768066406 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 14.772648811340332 }, { "rank": 4, "tokenId": 4339, "text": "True", "score": 10.431604385375977 }, { "rank": 5, "tokenId": 9277, "text": "False", "score": 8.127328872680664 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 6.361148357391357 }, { "rank": 7, "tokenId": 1584, "text": "Cont", "score": 6.164442539215088 }, { "rank": 8, "tokenId": 893, "text": "de", "score": 5.937593460083008 } ] }, "f32_reverse": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 21.94243049621582, "winnerGap": 5.186126708984375, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 21.94243049621582 }, { "rank": 2, "tokenId": 53417, "text": "deb", "score": 16.756303787231445 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 14.772631645202637 }, { "rank": 4, "tokenId": 4339, "text": "True", "score": 10.4315824508667 }, { "rank": 5, "tokenId": 9277, "text": "False", "score": 8.127327919006348 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 6.361147403717041 }, { "rank": 7, "tokenId": 1584, "text": "Cont", "score": 6.164433002471924 }, { "rank": 8, "tokenId": 893, "text": "de", "score": 5.9375901222229 } ] }, "f32_tree64": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 21.942426681518555, "winnerGap": 5.18610954284668, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 21.942426681518555 }, { "rank": 2, "tokenId": 53417, "text": "deb", "score": 16.756317138671875 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 14.772642135620117 }, { "rank": 4, "tokenId": 4339, "text": "True", "score": 10.431589126586914 }, { "rank": 5, "tokenId": 9277, "text": "False", "score": 8.127324104309082 }, { "rank": 6, "tokenId": 818, "text": "The", "score": 6.361145496368408 }, { "rank": 7, "tokenId": 1584, "text": "Cont", "score": 6.164443492889404 }, { "rank": 8, "tokenId": 893, "text": "de", "score": 5.937589645385742 } ] }, "f16_forward": { "winnerTokenId": 31900, "winnerText": "Deb", "winnerScore": 21.9375, "winnerGap": 6.140625, "top": [ { "rank": 1, "tokenId": 31900, "text": "Deb", "score": 21.9375 }, { "rank": 2, "tokenId": 53417, "text": "deb", "score": 15.796875 }, { "rank": 3, "tokenId": 1018, "text": "**", "score": 14.703125 }, { "rank": 4, "tokenId": 818, "text": "The", "score": 6.36328125 }, { "rank": 5, "tokenId": 1584, "text": "Cont", "score": 6.15625 }, { "rank": 6, "tokenId": 893, "text": "de", "score": 5.90625 }, { "rank": 7, "tokenId": 11062, "text": "Dev", "score": 5.0625 }, { "rank": 8, "tokenId": 1408, "text": "##", "score": 4.75390625 } ] } }, "winnerTextsByMode": { "exact": "Deb", "f32_forward": "Deb", "f32_reverse": "Deb", "f32_tree64": "Deb", "f16_forward": "Deb" }, "uniqueWinnerCount": 1, "uniqueWinners": [ "Deb" ], "f16VsF32WinnerChanged": false, "f32OrderingDisagreement": false, "branches": null, "branchComparison": null } ] }