{"id":"3e1a510734f20e49","label":"Take a deep breath","part":{"kind":"line","text":"Take a deep breath and work on this problem step-by-step."},"source":"Yang et al., 2023, Large Language Models as Optimizers (OPRO)","summary":"Is not settled on DeFi maths, Haiku 5.5 with thinking off; changes nothing that matters on DeFi maths, Haiku 5.5 with thinking on; changes nothing that matters on Agent safety, Haiku 5.5 with thinking off.","measurements":4,"settings":[{"setting":"DeFi maths, Haiku 5.5 with thinking off","suite":"defi-maths","base":{"model":"claude-haiku-5-5","thinking":"off"},"lean":"unclear","status":"once","pooled":{"k":2,"hi":5.55,"i2":0.223,"lo":-0.05,"effect":2.75},"notebooks":[{"notebook":"#003","verdict":"EXCITES","effect":4.4,"interval":[0.46,8.45],"pairs":432,"rule":1,"stoppedEarly":false,"provisional":false,"costRatio":1.172,"url":"https://opsinlab.io/notebooks/3","signed":"https://opsinlab.io/v1/verdicts/14"},{"notebook":"#036","verdict":"UNRESOLVED","effect":1.16,"interval":[-4.28,6.48],"pairs":432,"rule":2,"stoppedEarly":false,"provisional":false,"costRatio":1.123,"url":"https://opsinlab.io/notebooks/36","signed":"https://opsinlab.io/v1/verdicts/206"}]},{"setting":"DeFi maths, Haiku 5.5 with thinking on","suite":"defi-maths","base":{"model":"claude-haiku-5-5","thinking":"on"},"lean":"nothing","status":"once","pooled":{"k":1,"hi":4.34,"i2":0,"lo":-1.8,"effect":1.27},"notebooks":[{"notebook":"#010","verdict":"INERT","effect":1.27,"interval":[-1.85,4.28],"pairs":432,"rule":1,"stoppedEarly":false,"provisional":false,"costRatio":1.145,"url":"https://opsinlab.io/notebooks/10","signed":"https://opsinlab.io/v1/verdicts/54"}]},{"setting":"Agent safety, Haiku 5.5 with thinking off","suite":"agent-safety","base":{"model":"claude-haiku-5-5","thinking":"off"},"lean":"nothing","status":"once","pooled":{"k":1,"hi":1.86,"i2":0,"lo":-1.62,"effect":0.12},"notebooks":[{"notebook":"#008","verdict":"INERT","effect":0.12,"interval":[-1.62,1.85],"pairs":432,"rule":1,"stoppedEarly":false,"provisional":false,"costRatio":1.242,"url":"https://opsinlab.io/notebooks/8","signed":"https://opsinlab.io/v1/verdicts/40"}]}],"url":"https://opsinlab.io/parts/3e1a510734f20e49"}