{"id":"f0a2b80637651f3f","label":"Solve it two ways","part":{"kind":"line","text":"Solve the problem two different ways and give the answer only if both agree."},"source":"self-consistency in one reply, after Wang et al., 2022","summary":"Helps on DeFi maths, Haiku 5.5 with thinking off (+4.0 points); hurts on Agent safety, Haiku 5.5 with thinking off (−4.0 points, replicated); helps on DeFi maths, Haiku 5.5 with thinking on (+5.3 points); changes nothing that matters on Agent safety, Haiku 5.5 with thinking on.","measurements":6,"settings":[{"setting":"DeFi maths, Haiku 5.5 with thinking off","suite":"defi-maths","base":{"model":"claude-haiku-5-5","thinking":"off"},"lean":"helps","status":"once","pooled":{"k":2,"hi":6.95,"i2":0,"lo":1.1,"effect":4.03},"notebooks":[{"notebook":"#015","verdict":"UNRESOLVED","effect":3.47,"interval":[-0.58,7.52],"pairs":432,"rule":1,"stoppedEarly":false,"provisional":false,"costRatio":1.687,"url":"https://opsinlab.io/notebooks/15","signed":"https://opsinlab.io/v1/verdicts/90"},{"notebook":"#016","verdict":"EXCITES","effect":4.63,"interval":[0.46,8.91],"pairs":432,"rule":1,"stoppedEarly":false,"provisional":false,"costRatio":1.716,"url":"https://opsinlab.io/notebooks/16","signed":"https://opsinlab.io/v1/verdicts/96"}]},{"setting":"Agent safety, Haiku 5.5 with thinking off","suite":"agent-safety","base":{"model":"claude-haiku-5-5","thinking":"off"},"lean":"hurts","status":"replicated","pooled":{"k":2,"hi":-2.51,"i2":0,"lo":-5.52,"effect":-4.01},"notebooks":[{"notebook":"#019","verdict":"INHIBITS","effect":-3.82,"interval":[-6.13,-1.62],"pairs":432,"rule":1,"stoppedEarly":false,"provisional":false,"costRatio":1.271,"url":"https://opsinlab.io/notebooks/19","signed":"https://opsinlab.io/v1/verdicts/115"},{"notebook":"#037","verdict":"INHIBITS","effect":-4.17,"interval":[-7.18,-1.62],"pairs":432,"rule":2,"stoppedEarly":false,"provisional":false,"costRatio":1.326,"url":"https://opsinlab.io/notebooks/37","signed":"https://opsinlab.io/v1/verdicts/215"}]},{"setting":"DeFi maths, Haiku 5.5 with thinking on","suite":"defi-maths","base":{"model":"claude-haiku-5-5","thinking":"on"},"lean":"helps","status":"once","pooled":{"k":1,"hi":7.99,"i2":0,"lo":2.66,"effect":5.32},"notebooks":[{"notebook":"#021","verdict":"EXCITES","effect":5.32,"interval":[2.66,7.99],"pairs":432,"rule":1,"stoppedEarly":false,"provisional":false,"costRatio":1.474,"url":"https://opsinlab.io/notebooks/21","signed":"https://opsinlab.io/v1/verdicts/130"}]},{"setting":"Agent safety, Haiku 5.5 with thinking on","suite":"agent-safety","base":{"model":"claude-haiku-5-5","thinking":"on"},"lean":"nothing","status":"once","pooled":{"k":1,"hi":1.74,"i2":1,"lo":-0.81,"effect":0.46},"notebooks":[{"notebook":"#020","verdict":"INERT","effect":0.46,"interval":[-0.81,1.74],"pairs":432,"rule":1,"stoppedEarly":false,"provisional":false,"costRatio":1.579,"url":"https://opsinlab.io/notebooks/20","signed":"https://opsinlab.io/v1/verdicts/123"}]}],"url":"https://opsinlab.io/parts/f0a2b80637651f3f"}