2c:null 2d:["$","title",null,{"children":"Expected number of experiments to reach quality | Thijs Vogels | Observable"}] 32:T753,function problemPlot(problem) { const problemData = data.filter(d => d.problem === problem); const margin = 5; const plotTitle = htm`<${Flex} height=20 margin-bottom=6 margin-left=40> ${text(taskNames[problem], { align: "center" })}} `; const yScale = valueScales[problem]; const yAxisLabel = htm` <${Flex} width=30 margin-right=23> ${text(axisLabels[problem], { align: "center", color: "#333333", fontSize: 14, valign: "middle", rotate: -90 })} `; const plot = htm`<${Flex} flex-grow=1 padding-bottom=20> ${yAxisLabel} <${Flex} flex-grow=1> ${gridLines({ scale: yScale, orientation: "h" })} ${axis({ position: "left", scale: yScale, stroke: "#333333" })} ${optimizers.map(o => optimizerPlot(o, data.filter(d => d.optimizer === o)) )} `; function optimizerPlot(optimizer, data) { const domain = yScale.domain(); const numSteps = 15; return htm`<${Flex} flex-grow=1 margin-left=10 margin-right=10> ${d3 .range(domain[0], domain[1], (domain[1] - domain[0]) / numSteps) .map(target => ({ left, width, height }) => { return htm``; })} ${({ bottom, hcenter }) => htm`${optimizer}`} `; } return htm`<${Flex} margin=${margin} margin-bottom=15 width=${width / 3 - 2 * margin} height=200 flex-direction="column"> ${plotTitle} ${plot} `; }33:[{"id":"1564c2cc3aea4b6a549a562d491912af1452e8be5238f473d698f133d1281b5b5bd5aadaefbd2b7085a4e03a5cc2c54a58844c91e7809da8283e225f5020c97a","href":"https://static.observableusercontent.com/files/1564c2cc3aea4b6a549a562d491912af1452e8be5238f473d698f133d1281b5b5bd5aadaefbd2b7085a4e03a5cc2c54a58844c91e7809da8283e225f5020c97a","name":"results.csv","type":"text/csv","lastModified":1580479880151.952,"size":321704}] 30:{"id":"e6ed3a863a70687e","slug":"expected-number-of-experiments-to-reach-quality","title":"Expected number of experiments to reach quality","thumbnail":"30afd97319fbbb68ffd9809afee654175b76dc497f4786f189819101ac0e9436","access_level":"public","create_time":"$D2020-01-31T14:10:36.457Z","update_time":"$D2020-11-16T13:27:45.479Z","view_time":null,"first_public_time":"$D2020-11-16T13:27:53.092Z","fork":true,"stars":0,"starred":false,"owner":{"type":"team","id":"63d4ae9351b957f2","login":"tvogels","name":"Thijs Vogels","avatar_url":"https://avatars-next.observableusercontent.com/caecb0802edbb2531d0d2883357c53d701439f3641bbc772df9d728d289a6905","tier":"starter_2024"},"version":510,"publish_version":510,"role":null,"authored":false,"authors":[{"type":"individual","id":"7ebda7165bbc067e","login":"tvogels","name":"Thijs Vogels","avatar_url":"https://avatars-next.observableusercontent.com/caecb0802edbb2531d0d2883357c53d701439f3641bbc772df9d728d289a6905","tier":null}],"members":[{"type":"individual","id":"7ebda7165bbc067e","login":"tvogels","name":"Thijs Vogels","avatar_url":"https://avatars-next.observableusercontent.com/caecb0802edbb2531d0d2883357c53d701439f3641bbc772df9d728d289a6905","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"subscribed":null,"forks":1,"collection":null,"type":"notebook","body":{"title":"Expected number of experiments to reach quality","cells":[{"id":0,"value":"md`# Expected number of experiments to reach quality\n\n---`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":380,"value":"render(htm`\n <${Flex} svg flex-direction=\"column\">\n <${Flex} flex-wrap=\"wrap\" width=${width}>\n ${problems.map(problem => problemPlot(problem))}\n \n <${Flex} margin-top=5 justify-items=center>\n ${[1, 4, 16, 64, \"Never\"].map(\n num => htm`<${Flex} margin-right=10>\n <${Flex} width=20 height=20 margin-right=5>${bg(\n colorScale(num != \"Never\" ? Math.log(num) : 100)\n )}\n ${flextext(num != \"Never\" ? `${num} experiments` : num)}\n `\n )}\n \n \n`)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":387,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":406,"value":"colorScale = d3.scaleSequential([Math.log(100), 0], d3.interpolateRdYlGn)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":408,"value":"render(htm`<${Flex} svg>\n${d3.range(1, 100).map(\n target =>\n htm`<${Flex} height=10 width=5 margin=1>\n ${({ left, top, width, height }) =>\n htm``}\n `\n)}\n`)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":377,"value":"md`---`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":300,"value":"data = d3.csvParse(await FileAttachment(\"results.csv\").text(), d => ({\n problem: d.problem,\n optimizer: d.optimizer,\n score: +d.score,\n better: +d.better\n}))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":327,"value":"function expectedExperimentsForQuality(problem, optimizer, targetQuality) {\n let outcomes = data\n .filter(d => d.problem === problem && d.optimizer === optimizer)\n .map(d => d.score);\n if (betterDirections[problem] === -1) {\n outcomes = outcomes.map(o => -o);\n targetQuality = -targetQuality;\n }\n outcomes = outcomes.sort(d3.ascending);\n\n const index = outcomes.findIndex(o => o >= targetQuality);\n\n if (index === -1) {\n return Infinity;\n }\n\n // p is the probability that an experiment reacher the target quality\n const p = (outcomes.length - index) / outcomes.length;\n return 1 / p;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":128,"value":"md`### Library imports`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":316,"value":"optimizers = [\"AdamLR\", \"Adam\", \"SGD\", \"SGDMW\"]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":324,"value":"problems = Array.from(new Set(data.map(d => d.problem)))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":328,"value":"betterDirections = Object.fromEntries(\n problems.map(problem => [\n problem,\n data.find(d => d.problem === problem).better\n ])\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":338,"value":"valueScales[\"quadratic_deep\"].domain()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":312,"value":"import { valueScales, taskNames, axisLabels } from \"60028bc8fbbbd070\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2,"value":"import {\n d3,\n Flex,\n render,\n htm,\n flextext,\n area,\n text,\n graph,\n bg,\n axis,\n clipPath,\n legend,\n gridLines,\n boxplot\n} from \"b679d071989a855a@1154\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W33","stdlib":"1","titling":"auto"}} 31:"$30:body"