{"id":"aed0a14114f3e60e","label":"Read the question again","part":{"kind":"line","text":"Read the question again before you answer."},"source":"Xu et al., 2023, Re-Reading Improves Reasoning in Large Language Models","summary":"Helps on DeFi maths, Haiku 5.5 with thinking off (+5.2 points, replicated); hurts on DeFi maths, Haiku 5.5 with thinking on (−2.3 points); changes nothing that matters on Agent safety, Haiku 5.5 with thinking off.","measurements":6,"settings":[{"setting":"DeFi maths, Haiku 5.5 with thinking off","suite":"defi-maths","base":{"model":"claude-haiku-5-5","thinking":"off"},"lean":"helps","status":"replicated","pooled":{"k":3,"hi":7.56,"i2":0,"lo":2.74,"effect":5.15},"notebooks":[{"notebook":"#003","verdict":"UNRESOLVED","effect":3.94,"interval":[-0.35,8.22],"pairs":432,"rule":1,"stoppedEarly":false,"provisional":false,"costRatio":1.12,"url":"https://opsinlab.io/notebooks/3","signed":"https://opsinlab.io/v1/verdicts/18"},{"notebook":"#005","verdict":"EXCITES","effect":7.18,"interval":[2.89,11.57],"pairs":432,"rule":1,"stoppedEarly":false,"provisional":false,"costRatio":1.135,"url":"https://opsinlab.io/notebooks/5","signed":"https://opsinlab.io/v1/verdicts/23"},{"notebook":"#007","verdict":"EXCITES","effect":4.51,"interval":[0.58,8.45],"pairs":432,"rule":1,"stoppedEarly":false,"provisional":false,"costRatio":1.125,"url":"https://opsinlab.io/notebooks/7","signed":"https://opsinlab.io/v1/verdicts/34"}]},{"setting":"DeFi maths, Haiku 5.5 with thinking on","suite":"defi-maths","base":{"model":"claude-haiku-5-5","thinking":"on"},"lean":"hurts","status":"once","pooled":{"k":2,"hi":-0.06,"i2":0,"lo":-4.56,"effect":-2.31},"notebooks":[{"notebook":"#010","verdict":"UNRESOLVED","effect":-3.12,"interval":[-6.37,0],"pairs":432,"rule":1,"stoppedEarly":false,"provisional":false,"costRatio":1.087,"url":"https://opsinlab.io/notebooks/10","signed":"https://opsinlab.io/v1/verdicts/57"},{"notebook":"#011","verdict":"INERT","effect":-1.5,"interval":[-4.75,1.62],"pairs":432,"rule":1,"stoppedEarly":false,"provisional":false,"costRatio":1.082,"url":"https://opsinlab.io/notebooks/11","signed":"https://opsinlab.io/v1/verdicts/63"}]},{"setting":"Agent safety, Haiku 5.5 with thinking off","suite":"agent-safety","base":{"model":"claude-haiku-5-5","thinking":"off"},"lean":"nothing","status":"once","pooled":{"k":1,"hi":3.13,"i2":0,"lo":-1.04,"effect":1.04},"notebooks":[{"notebook":"#008","verdict":"INERT","effect":1.04,"interval":[-1.04,3.13],"pairs":432,"rule":1,"stoppedEarly":false,"provisional":false,"costRatio":0.955,"url":"https://opsinlab.io/notebooks/8","signed":"https://opsinlab.io/v1/verdicts/43"}]}],"url":"https://opsinlab.io/parts/aed0a14114f3e60e"}