{"id":87,"notebook":88,"tag":"#088","status":"published","suite":"coding","suiteTitle":"Coding","exactMeans":"the function returned the right answer on every hidden case","base":{"model":"claude-haiku-5-5","thinking":"off"},"pairsDone":432,"maxPairs":432,"beaconRound":6625646,"beaconAt":"2026-11-09T05:00:00.000Z","beacon":{"chain":"8990e7a9aaed2ffed73dbd7092123d6f289930540d7651336225dc172e51b2ce","round":6625646,"signature":"a349be9ceb792200bcbf13381f37a9b34c0ae31246f2bb981fda060f1d48f2ac056a149d53d5543f3e5fa2c6655162c903d5cf2e0f59b78ef8e8b24afa0fb3fd114e90f0f670ab47fa54e59e77f82856ac82de218f305b57a12f0f110663c36c","randomness":"d49c724747971930f2b948289e31fc49bf58ba40d9e10b06c01d63a83c12a923","publishedAt":"2026-11-09T05:00:00.000Z"},"seed":"drand:6625646:d49c724747971930f2b948289e31fc49bf58ba40d9e10b06c01d63a83c12a923","preregHash":"2ee05a353294cbdddd9948374ef1c5817924761bf9b8b50a9fd619af05ea88af","prereg":{"lab":"OPSIN","arms":[{"role":"control-a","fingerprint":"7772fae351628ad523888a8a8405154f10eedab329a516a8e28d417470782cb7"},{"role":"control-b","fingerprint":"7772fae351628ad523888a8a8405154f10eedab329a516a8e28d417470782cb7"},{"part":{"kind":"line","text":"The office plants were watered this morning."},"role":"placebo","fingerprint":"b750c3dff6c90ff3e68495aba8a9adcc3a0d931016ba2a639a74bacb5235afc6"},{"part":{"kind":"line","text":"Do not floor or round intermediate values; with BigInt keep fees as exact ratios (multiply numerator and denominator by the fee's scale) and apply one floor or rounding only at the end, as the task states."},"role":"part","label":"Keep exact fractions","proposal":367,"fingerprint":"114ad579b0f4913119b3e04fd5503b392c4012980459420a9a94113cf78242f7"},{"part":{"kind":"line","text":"Do not add comments to the code."},"role":"part","label":"No comments","proposal":368,"fingerprint":"007ef22202026620a1f921e1a9320f2490a6f737e2cab05dfb03fc91948a810b"},{"part":{"kind":"line","text":"Whenever a result must be exact, use integer arithmetic (BigInt when numbers can be large), never floating point."},"role":"part","label":"Exact integers","proposal":369,"fingerprint":"c8a95690f17928df8c86e7a5f9a45227eb6451d809a58c950aee142e9df6d497"},{"part":{"kind":"line","text":"Keep it simple: prefer clear code over clever code."},"role":"part","label":"Keep it simple","proposal":370,"fingerprint":"9948f8080c90052a2d3a3f21c938bed8f5048aa5d81f41c181a117bc470586a2"},{"part":{"kind":"line","text":"Before writing code, write a short plan of the steps."},"role":"part","label":"Plan first","proposal":371,"fingerprint":"a556e121db26060106d5e2d5067e1c525f3815ed1aecf8469f9b267ee9c529cd"},{"part":{"kind":"line","text":"I'm going to tip $200 for a perfect solution!"},"role":"part","label":"A $200 tip","proposal":372,"fingerprint":"554e6448c7a4772bbfad73de1a2c417435498a8df0e74f571283ac02c6eadaf7"}],"base":{"model":"claude-haiku-5-5","thinking":"off"},"kind":"cycle","rule":{"cost":"each part's cost over the controls' mean, both at list prices: a batched run was paid half, so it is doubled","INERT":"adjusted interval entirely inside ±5 points","errors":"a task with an errored run in any compared arm is left out of that comparison","metric":"exact: the function returned the right answer on every hidden case","EXCITES":"adjusted interval entirely above 0","control":"each part against the per-task mean of two unchanged runs of the base agent","interim":"before the last look, a part stops early only if |z| ≥ 3.29; INERT is decided at the last look only; a part that stops early with EXCITES or INHIBITS is provisional until a later notebook confirms it at full size","placebo":"a neutral line chosen by the beacon round's number from a fixed public list","version":2,"INHIBITS":"adjusted interval entirely below 0","interval":"percentile bootstrap over tasks, 20,000 resamples, seeded from the beacon seed and the proposal id, at 1 − 0.05/7: 7 arms (the parts and the placebo) are compared with the controls, so the chance that any arm of this notebook flags by luck stays at or under 5%","UNRESOLVED":"anything else","confirmations":"an arm that re-measures an earlier early stop never stops early","versusPlacebo":"each part is also compared with the placebo on the same tasks, 95% interval, reported beside its verdict"},"looks":[48,96,192,432],"suite":{"id":"coding","version":"tasks-1","exactMeans":"the function returned the right answer on every hidden case"},"beacon":{"rule":"first drand round at least 600 seconds after this preregistration","chain":"8990e7a9aaed2ffed73dbd7092123d6f289930540d7651336225dc172e51b2ce","round":6625646},"prices":"2026-11-04","version":2,"maxPairs":432,"foresight":{"arms":[{"basis":"measured","rates":{"INERT":0.4491,"EXCITES":0.2868,"INHIBITS":0.0561,"UNRESOLVED":0.2081},"record":{"INERT":0.3342,"EXCITES":0.4819,"INHIBITS":0.0001,"UNRESOLVED":0.1838},"chances":{"INERT":0.3342,"EXCITES":0.4819,"INHIBITS":0.0001,"UNRESOLVED":0.1838},"measured":2,"proposal":367},{"basis":"measured","rates":{"INERT":0.4491,"EXCITES":0.2868,"INHIBITS":0.0561,"UNRESOLVED":0.2081},"record":{"INERT":0.694,"EXCITES":0.0026,"INHIBITS":0.1443,"UNRESOLVED":0.159},"chances":{"INERT":0.6941,"EXCITES":0.0026,"INHIBITS":0.1443,"UNRESOLVED":0.159},"measured":1,"proposal":368},{"basis":"new","rates":{"INERT":0.6183,"EXCITES":0.1934,"INHIBITS":0.0592,"UNRESOLVED":0.1291},"record":{"INERT":0.6366,"EXCITES":0.1765,"INHIBITS":0.0314,"UNRESOLVED":0.1555},"chances":{"INERT":0.6413,"EXCITES":0.1556,"INHIBITS":0.0612,"UNRESOLVED":0.1419},"measured":0,"proposal":369},{"basis":"new","rates":{"INERT":0.6183,"EXCITES":0.1934,"INHIBITS":0.0592,"UNRESOLVED":0.1291},"record":{"INERT":0.6218,"EXCITES":0.1888,"INHIBITS":0.0326,"UNRESOLVED":0.1568},"chances":{"INERT":0.6772,"EXCITES":0.1484,"INHIBITS":0.0469,"UNRESOLVED":0.1276},"measured":0,"proposal":370},{"basis":"new","rates":{"INERT":0.6183,"EXCITES":0.1934,"INHIBITS":0.0592,"UNRESOLVED":0.1291},"record":{"INERT":0.6346,"EXCITES":0.1843,"INHIBITS":0.0309,"UNRESOLVED":0.1503},"chances":{"INERT":0.6574,"EXCITES":0.1548,"INHIBITS":0.0551,"UNRESOLVED":0.1327},"measured":0,"proposal":371},{"basis":"new","rates":{"INERT":0.6183,"EXCITES":0.1934,"INHIBITS":0.0592,"UNRESOLVED":0.1291},"record":{"INERT":0.6473,"EXCITES":0.1733,"INHIBITS":0.0309,"UNRESOLVED":0.1486},"chances":{"INERT":0.6832,"EXCITES":0.1418,"INHIBITS":0.0492,"UNRESOLVED":0.1257},"measured":0,"proposal":372}],"version":2,"weights":{"new":[0.35,0.65,0],"stack":[0.33,0.33,0.33],"measured":[0,1,0]}},"predictor":2,"writtenAt":"2026-11-09T04:49:41.071Z","predictedBy":"claude-sonnet-5-5","predictions":[{"p":0.55,"why":"Early-stop gain likely shrinks; second run only +2.9","effect":2.5,"verdict":"INERT","proposal":367},{"p":0.7,"why":"Comments rarely affect correctness; prior slightly negative","effect":-1,"verdict":"INERT","proposal":368},{"p":0.65,"why":"Generic exactness advice; base near ceiling at 93%","effect":1,"verdict":"INERT","proposal":369},{"p":0.78,"why":"Generic style advice, little effect on hidden-case correctness","effect":0,"verdict":"INERT","proposal":370},{"p":0.7,"why":"Planning may help slightly, but ceiling limits gains","effect":0.5,"verdict":"INERT","proposal":371},{"p":0.75,"why":"Tip prompts inert or slightly negative in other settings","effect":-1,"verdict":"INERT","proposal":372}]},"createdAt":"2026-11-09T04:49:42.059Z","publishedAt":"2026-11-09T05:30:02.602Z","error":null,"controlsTripped":false,"baseExact":0.941,"arms":[{"id":631,"role":"control-a","label":"Control A","kind":"control-a","text":null,"proposalId":null,"origin":null,"stoppedAt":null,"done":432,"exact":408,"prediction":null,"foresight":null,"verdict":null},{"id":632,"role":"control-b","label":"Control B","kind":"control-b","text":null,"proposalId":null,"origin":null,"stoppedAt":null,"done":432,"exact":405,"prediction":null,"foresight":null,"verdict":{"id":537,"verdict":"INERT","effect":-0.69,"lo":-3.24,"hi":1.62,"n":432,"costRatio":null,"stoppedEarly":false,"safety":null,"rule":2,"level":null,"raw95":null,"vsPlacebo":null,"provisional":false,"confirms":null,"stack":null}},{"id":633,"role":"placebo","label":"Placebo","kind":"placebo","text":"The office plants were watered this morning.","proposalId":null,"origin":null,"stoppedAt":null,"done":432,"exact":400,"prediction":null,"foresight":null,"verdict":{"id":538,"verdict":"INERT","effect":-1.5,"lo":-4.51,"hi":1.27,"n":432,"costRatio":0.97,"stoppedEarly":false,"safety":null,"rule":2,"level":0.992857,"raw95":{"hi":0.58,"lo":-3.59,"verdict":"INERT"},"vsPlacebo":null,"provisional":false,"confirms":null,"stack":null}},{"id":634,"role":"part","label":"Keep exact fractions","kind":"line","text":"Do not floor or round intermediate values; with BigInt keep fees as exact ratios (multiply numerator and denominator by the fee's scale) and apply one floor or rounding only at the end, as the task states.","proposalId":367,"origin":"OPSIN's second look at a surprise","stoppedAt":null,"done":432,"exact":415,"prediction":{"verdict":"INERT","p":0.55,"why":"Early-stop gain likely shrinks; second run only +2.9"},"foresight":{"basis":"measured","measured":2,"chances":{"INERT":0.3342,"EXCITES":0.4819,"INHIBITS":0.0001,"UNRESOLVED":0.1838}},"verdict":{"id":539,"verdict":"UNRESOLVED","effect":1.97,"lo":-1.27,"hi":5.21,"n":432,"costRatio":1.151,"stoppedEarly":false,"safety":null,"rule":2,"level":0.992857,"raw95":{"hi":4.4,"lo":-0.46,"verdict":"INERT"},"vsPlacebo":{"n":432,"hi":6.48,"lo":0.46,"effect":3.47},"provisional":false,"confirms":null,"stack":null}},{"id":635,"role":"part","label":"No comments","kind":"line","text":"Do not add comments to the code.","proposalId":368,"origin":"OPSIN's replication","stoppedAt":null,"done":432,"exact":383,"prediction":{"verdict":"INERT","p":0.7,"why":"Comments rarely affect correctness; prior slightly negative"},"foresight":{"basis":"measured","measured":1,"chances":{"INERT":0.6941,"EXCITES":0.0026,"INHIBITS":0.1443,"UNRESOLVED":0.159}},"verdict":{"id":540,"verdict":"INHIBITS","effect":-5.44,"lo":-9.49,"hi":-1.62,"n":432,"costRatio":0.822,"stoppedEarly":false,"safety":null,"rule":2,"level":0.992857,"raw95":{"hi":-2.66,"lo":-8.33,"verdict":"INHIBITS"},"vsPlacebo":{"n":432,"hi":-0.93,"lo":-6.94,"effect":-3.94},"provisional":false,"confirms":null,"stack":null}},{"id":636,"role":"part","label":"Exact integers","kind":"line","text":"Whenever a result must be exact, use integer arithmetic (BigInt when numbers can be large), never floating point.","proposalId":369,"origin":"OPSIN's pick · money-handling rules in agent rules files","stoppedAt":null,"done":432,"exact":406,"prediction":{"verdict":"INERT","p":0.65,"why":"Generic exactness advice; base near ceiling at 93%"},"foresight":{"basis":"new","measured":0,"chances":{"INERT":0.6413,"EXCITES":0.1556,"INHIBITS":0.0612,"UNRESOLVED":0.1419}},"verdict":{"id":541,"verdict":"INERT","effect":-0.12,"lo":-2.66,"hi":2.43,"n":432,"costRatio":1.135,"stoppedEarly":false,"safety":null,"rule":2,"level":0.992857,"raw95":{"hi":1.74,"lo":-1.97,"verdict":"INERT"},"vsPlacebo":{"n":432,"hi":4.17,"lo":-1.16,"effect":1.39},"provisional":false,"confirms":null,"stack":null}},{"id":637,"role":"part","label":"Keep it simple","kind":"line","text":"Keep it simple: prefer clear code over clever code.","proposalId":370,"origin":"OPSIN's pick · KISS, in countless CLAUDE.md files","stoppedAt":null,"done":432,"exact":399,"prediction":{"verdict":"INERT","p":0.78,"why":"Generic style advice, little effect on hidden-case correctness"},"foresight":{"basis":"new","measured":0,"chances":{"INERT":0.6772,"EXCITES":0.1484,"INHIBITS":0.0469,"UNRESOLVED":0.1276}},"verdict":{"id":542,"verdict":"INERT","effect":-1.74,"lo":-4.86,"hi":1.16,"n":432,"costRatio":1.067,"stoppedEarly":false,"safety":null,"rule":2,"level":0.992857,"raw95":{"hi":0.46,"lo":-3.94,"verdict":"INERT"},"vsPlacebo":{"n":432,"hi":2.55,"lo":-3.01,"effect":-0.23},"provisional":false,"confirms":null,"stack":null}},{"id":638,"role":"part","label":"Plan first","kind":"line","text":"Before writing code, write a short plan of the steps.","proposalId":371,"origin":"OPSIN's pick · plan-then-code rules in coding-agent prompts","stoppedAt":null,"done":432,"exact":405,"prediction":{"verdict":"INERT","p":0.7,"why":"Planning may help slightly, but ceiling limits gains"},"foresight":{"basis":"new","measured":0,"chances":{"INERT":0.6574,"EXCITES":0.1548,"INHIBITS":0.0551,"UNRESOLVED":0.1327}},"verdict":{"id":543,"verdict":"INERT","effect":-0.35,"lo":-2.89,"hi":2.08,"n":432,"costRatio":1.33,"stoppedEarly":false,"safety":null,"rule":2,"level":0.992857,"raw95":{"hi":1.39,"lo":-2.2,"verdict":"INERT"},"vsPlacebo":{"n":432,"hi":3.47,"lo":-1.16,"effect":1.16},"provisional":false,"confirms":null,"stack":null}},{"id":639,"role":"part","label":"A $200 tip","kind":"line","text":"I'm going to tip $200 for a perfect solution!","proposalId":372,"origin":"OPSIN's pick · the viral tipping test, December 2023","stoppedAt":null,"done":432,"exact":404,"prediction":{"verdict":"INERT","p":0.75,"why":"Tip prompts inert or slightly negative in other settings"},"foresight":{"basis":"new","measured":0,"chances":{"INERT":0.6832,"EXCITES":0.1418,"INHIBITS":0.0492,"UNRESOLVED":0.1257}},"verdict":{"id":544,"verdict":"INERT","effect":-0.58,"lo":-2.78,"hi":1.62,"n":432,"costRatio":1.016,"stoppedEarly":false,"safety":null,"rule":2,"level":0.992857,"raw95":{"hi":1.04,"lo":-2.2,"verdict":"INERT"},"vsPlacebo":{"n":432,"hi":3.47,"lo":-1.62,"effect":0.93},"provisional":false,"confirms":null,"stack":null}}]}