{"id":"d47935e0024f0eec","label":"Let's think step by step","part":{"kind":"line","text":"Let's think step by step."},"source":"Kojima et al., 2022, Large Language Models are Zero-Shot Reasoners","summary":"Changes nothing that matters on DeFi maths, Haiku 5.5 with thinking off.","measurements":3,"settings":[{"setting":"DeFi maths, Haiku 5.5 with thinking off","suite":"defi-maths","base":{"model":"claude-haiku-5-5","thinking":"off"},"lean":"nothing","status":"once","pooled":{"k":3,"hi":0.87,"i2":0,"lo":-3.59,"effect":-1.36},"notebooks":[{"notebook":"#003","verdict":"UNRESOLVED","effect":-2.08,"interval":[-6.13,1.97],"pairs":432,"rule":1,"stoppedEarly":false,"provisional":false,"costRatio":1.004,"url":"https://opsinlab.io/notebooks/3","signed":"https://opsinlab.io/v1/verdicts/13"},{"notebook":"#005","verdict":"INERT","effect":-0.23,"interval":[-4.17,3.7],"pairs":432,"rule":1,"stoppedEarly":false,"provisional":false,"costRatio":0.988,"url":"https://opsinlab.io/notebooks/5","signed":"https://opsinlab.io/v1/verdicts/21"},{"notebook":"#007","verdict":"UNRESOLVED","effect":-1.74,"interval":[-5.44,1.85],"pairs":432,"rule":1,"stoppedEarly":false,"provisional":false,"costRatio":1.018,"url":"https://opsinlab.io/notebooks/7","signed":"https://opsinlab.io/v1/verdicts/32"}]}],"url":"https://opsinlab.io/parts/d47935e0024f0eec"}