2b:null 2c:["$","title",null,{"children":"Magic Markup (PX'24 submission) -- Benchmark Generation Tools | Edward Misback | Observable"}] 31:T506,async function generateTest ( language=getRandomElementFromArray(languages), snippetType = getRandomElementFromArray(snippetTypes), updateIsPartial = getRandomElementFromArray([true, false]) ) { const problemDescription = await describeProblem(language, snippetType) const initialCode = await generateInitialCode(language, snippetType, problemDescription) const delimiter = await getDelimiter(initialCode, snippetType, language) // const example = getExample(snippetType) // TODO use if needed const snippetDescription = await describeSnippet(initialCode, snippetType) const codeWithSnippetDelimited = await getCodeWithSnippetDelimited(delimiter, initialCode, snippetType, snippetDescription)//, example) const updateDescription = await getUpdateDescription(problemDescription, initialCode, updateIsPartial) const updatedCodeWithSnippetDelimited = await getUpdatedCodeWithSnippetDelimited(updateDescription, updateIsPartial, delimiter, snippetType, snippetDescription, codeWithSnippetDelimited) return ({ language, snippetType, updateIsPartial, delimiter, codeWithSnippetDelimited, updateDescription, updatedCodeWithSnippetDelimited, // context problemDescription, snippetDescription, updateDescription }) }32:T44a,md` Tokenizer vis: https://platform.openai.com/tokenizer Single example token number + cost: (All use 3.5 unless otherwise noted) describeProblem: prompt was 55 tokens, completion 372 generateInitialCode: prompt was 436, completion 261 describeSnippet: 301, 93 getCodeWithSnippetDelimited: 736, 159 getUpdateDescription: 486, 260 getUpdatedCodeWithSnippetDelimited (GPT-4): 1076, 524 Total GPT 3.5 tokens: 2014 input, 1145 output GPT-3.5-turbo: $0.0005 / 1K tokens input, $0.0015 / 1K tokens output GPT-4-0125 (turbo): $0.01 / 1K tokens input, $0.03 / 1K tokens output Cost for one test: GPT-3.5: ~$.001013 input + ~$.001749 output = ~$.002762 (note calculation has error from earlier incorrect token number) GPT-4 turbo: $.01076 input + $.01572 = $.02648 Total: $.029242 / test Extrapolating to 100 tests: $2.9242 (If final step could be done with 3.5...: ($.002762 + $.001324) * 100 = $.4086) Incidentally, pricing for other commercial models: Prices per million tokens: https://docs.google.com/spreadsheets/d/1SwqpeKB19JK2Xk6NW2uRxi7T4ydBZPpB-x9OcBAIyLA/edit#gid=0 `33:T560,async function getCodeWithSnippetDelimited(delimiter, initialCode, snippetType, snippetDescription) { // may need gpt4 const prompt = `Consider the following program and snippet description (NOTE the tags are not part of the content!): ${initialCode} (${snippetType}) ${snippetDescription} Mark the described snippet in the program with "${delimiter}" on both sides. For example, a constant might be marked in the program below like so: def foo(): a = ${delimiter}42${delimiter} return a Respond with the entire program, and make sure to mark the selected ${snippetType} with "${delimiter}" on both sides. Your change MUST add a SINGLE PAIR of "${delimiter}" marks. Respond with code only. Do not add commentary, newlines, markdown syntax or alter the program in any other way. Do not surround the output with or other tags.` // it still adds newlines sometimes... // return askGPT35Instruct(prompt) // return askGPT4Turbo(prompt) return (await getChatCompletion('gpt-4-turbo-preview', [{role: 'system', content: `You are a code generation assistant that marks the beginning and end of a section of ${language} code with 2 "${delimiter}"s for highlighting.`}, {role: 'user', content: prompt}])).choices[0].message.content }34:T5d8,function getUpdatedCodeWithSnippetDelimited(updateDescription, isPartial, delimiter, snippetType, snippetDescription, codeWithSnippetDelimited) { const prompt = `Consider the following problem: ${problemDescription} Now consider this code that tries to solve the problem: ${codeWithSnippetDelimited} Note that a snippet from the code has been marked with a "${delimiter}" on both sides. This snippet is described as follows: (${snippetType}) ${snippetDescription} Now consider the following description of ${isPartial ? 'a partial update' : 'an update'} to the program: ${updateDescription} ${isPartial ? 'Note again this is a partial update, since we are interested in studying what the buffer might look like for partial changes.' : ''} Apply this update to the code as described. Your response should be purely code without any external discussion, and should fully copy any relevant sections of the original program. In order to obtain credit, you MUST maintain the "${delimiter}" marks on the snippet or its updated version. The contents of the snippet should be functionally identical in the new version of the code. Again, the updated version of the code MUST have a SINGLE pair of "${delimiter}" marks referring to the same snippet in its new position or form. ` return askGPT4Turbo(prompt) // NOTE this step requires GPT-4 }35:T48a,getUpdatedCodeWithSnippetDelimitedPrompt = `Consider the following problem: ${problemDescription} Now consider this code that tries to solve the problem: ${codeWithSnippetDelimited} Note that a snippet from the code has been marked with a "${delimiter}" on both sides. This snippet is described as follows: (${snippetType}) ${snippetDescription} Now consider the following description of ${true ? 'a partial update' : 'an update'} to the program: ${updateDescription} ${true ? 'Note again this is a partial update, since we are interested in studying what the buffer might look like for partial changes.' : ''} Apply this update to the code as described. In order to obtain credit, you MUST maintain the "${delimiter}" marks on the snippet or its updated version. The contents of the snippet should be functionally identical in the new version of the code. Again, the updated version of the code MUST have a SINGLE pair of "${delimiter}" marks referring to the same snippet in its new position or form. `36:[] 2f:{"id":"56aa7eb152e26567","slug":"tcu-benchmark-generator","title":"Magic Markup (PX'24 submission) -- Benchmark Generation Tools","thumbnail":"a9a54eaa4540d6eaa25a578f3587ff09f4f8f4d5abf48705ab937534dbb91252","access_level":"public","create_time":"$D2024-02-01T11:46:49.537Z","update_time":"$D2024-03-05T16:52:57.859Z","view_time":null,"first_public_time":"$D2024-02-01T11:46:49.537Z","fork":false,"stars":0,"starred":false,"owner":{"type":"team","id":"5508584884fd137b","login":"elmisback","name":"Edward Misback","avatar_url":"https://avatars-next.observableusercontent.com/9a42c09ac3fb39913eeacae071d5ba669ac49bf75c82372d0d717ec0acb59024","tier":"starter_2024"},"version":490,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"e03d30b651790cf0","login":"elmisback","name":"Edward Misback","avatar_url":"https://avatars-next.observableusercontent.com/9a42c09ac3fb39913eeacae071d5ba669ac49bf75c82372d0d717ec0acb59024","tier":null}],"members":[{"type":"individual","id":"e03d30b651790cf0","login":"elmisback","name":"Edward Misback","avatar_url":"https://avatars-next.observableusercontent.com/9a42c09ac3fb39913eeacae071d5ba669ac49bf75c82372d0d717ec0acb59024","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"subscribed":null,"forks":0,"collection":{"id":"93f0d756f0de9185","slug":"magic-markup","title":"Magic Markup","owner":{"id":"5508584884fd137b","login":"elmisback","name":"Edward Misback"}},"type":"notebook","body":{"title":"Magic Markup (PX'24 submission) -- Benchmark Generation Tools","cells":[{"id":0,"value":"# Magic Markup (PX'24 submission) -- Benchmark Generation Tools","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":24,"value":"$31","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":420,"value":"md`Completions API reference: https://platform.openai.com/docs/api-reference/completions/create`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":307,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":5,"value":"import {openai, askChatGPT, usageHistory} from \"9f772e010064bd3e\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":133,"value":"import {diff, getChatCompletion, getFirstChoice} from \"9f772e010064bd3e\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":269,"value":"import {localStorageArray} from \"9f772e010064bd3e\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":271,"value":"// usageHistory = localStorageArray('usage-history')","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":303,"value":"usageHistory","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":276,"value":"data = usageHistory.get()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":288,"value":"totalPromptTokens = usageHistory.get().filter(o => o.model.includes('3.5')).reduce((acc, v) => acc + v.usage.prompt_tokens, 0)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":291,"value":"totalCompletionTokens = usageHistory.get().filter(o => o.model.includes('3.5')).reduce((acc, v) => acc + v.usage.completion_tokens, 0)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":293,"value":"// Roughly 3192, broken down as \ntotalTokens = usageHistory.get().filter(o => o.model.includes('3.5')).reduce((acc, v) => acc + v.usage.total_tokens, 0)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":202,"value":"async function askGPT4Turbo(prompt) {\n const output = await getChatCompletion('gpt-4-turbo-preview', [{role: 'user', content: prompt}])\n usageHistory.push({model: output.model, usage: output.usage, origin: 'bench'})\n return getFirstChoice(output)\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":13,"value":"// askChatGPT(\"What is your name?\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":90,"value":"languages = ['Python', 'JavaScript', 'C', 'JSX', 'Racket' ] ","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":85,"value":"snippetTypes = ['constant', 'subexpression', 'variable assignment', 'loop body or code block', 'loop condition', 'function call']","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":82,"value":"snippetType = getRandomElementFromArray(snippetTypes)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":93,"value":"language = getRandomElementFromArray(languages)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":20,"value":"function describeProblem(language, snippetType) {\n const prompt = `Briefly describe an intermediate-level ${language} programming problem including at least one ${snippetType} that can be solved in a single file. Use a creative, real-world framing. Describe steps to solve this problem. Do not provide code yet.`\n return askChatGPT(prompt)\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":72,"value":"problemDescription = describeProblem(language, snippetType)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":209,"value":"md`${problemDescription}`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":429,"value":"async function askGPT35Instruct(prompt, maxTokens=500) {\n return (await (openai.completions.create({model:'gpt-3.5-turbo-instruct', prompt, max_tokens: maxTokens}))).choices[0].text\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":22,"value":"async function generateInitialCode(language, snippetType, problemDescription) {\n const prompt = `Consider the following problem description:\n\n\n${problemDescription}\n\n\nWrite ${language} code that solves this problem. Your response will be executed directly and must be valid ${language} code. Your code must include at least one ${snippetType}. Again, your response should be ONLY code, and will be dumped directly into a file (please do not format it with markdown).\n`\n //return askGPT35Instruct(prompt)//(await (openai.completions.create({model:'gpt-3.5-turbo-instruct', prompt, max_tokens: 500}))).choices[0].text\n return (await getChatCompletion('gpt-3.5-turbo', [{role: 'system', content: `You are a code generation assistant that produces valid ${language} code. Respond with unformatted code only.`}, {role: 'user', content: prompt}])).choices[0].message.content\n // return askChatGPT(prompt)\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":80,"value":"initialCode =\n(await generateInitialCode(language, snippetType, problemDescription))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":35,"value":"function getDelimiter(initialCode, snippetType, language) {\n const prompt = `You need to highlight a ${snippetType} from a program like the one below:\n\n${initialCode}\n\n\nAgain, you need to highlight a ${snippetType} from this program. However, your friend will receive the document in a plaintext editor, so you need to use special characters to delimit the highlighted section. For example, your response could be $$, if the program doesn't include dollar signs--you don't want your friend to get confused when reading the program. The delimiter will be used both for the beginning and end of the highlighted section. Respond with ONLY your choice of delimiter.`\n // prompt seems bad, hardcoding instead:\n return '★'\n return !(initialCode.includes('***')) ? '***' : !(initialCode.includes('****')) ? '****' : !(initialCode.includes('~~~')) ? '~~~' : '~~~~'\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":115,"value":"delimiter = getDelimiter(initialCode, language)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":138,"value":"function describeSnippet(initialCode, snippetType) {\n const prompt = `Describe a particular ${snippetType} in the program below.\n\n\n${initialCode}\n\n\nAgain, describe a ${snippetType} in this program.\n`\n\n return askChatGPT(prompt)\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":143,"value":"snippetDescription = describeSnippet(initialCode, snippetType)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":214,"value":"md`${snippetDescription}`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":33,"value":"$33","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":126,"value":"codeWithSnippetDelimited = \ngetCodeWithSnippetDelimited(delimiter, initialCode, snippetType, snippetDescription)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":136,"value":"diff(initialCode, codeWithSnippetDelimited)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":374,"value":"function validateSnippet(initialCode, delimiter, snippetDescription, codeWithSnippetDelimited) {\n if ([...codeWithSnippetDelimited.matchAll(/\\*\\*\\*/g)].length !== 2) {return false}\n const prompt = `Consider the following program, snippet description, and student response:\n\n\n${initialCode}\n\n\n\n${snippetDescription}\n\n\n\n${codeWithSnippetDelimited}\n\n\nA student was asked to mark the snippet in this program CORRECTLY, with ONLY 2 instances of \"${delimiter}\", without changing any other lines. Did the student complete the task successfully? (Y/N, respond with just the letter Y/N)\n`\n return askChatGPT(prompt)\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":378,"value":"validateSnippet(initialCode, delimiter, snippetDescription, codeWithSnippetDelimited)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":32,"value":"function getUpdateDescription(problemDescription, initialCode, isPartial) {\n const prompt = `Consider the following problem:\n\n\n${problemDescription}\n\n\nNow consider this code that tries to solve the problem:\n\n\n${initialCode}\n\n\nDescribe an interesting change or refactoring of this code that a real-world programmer might apply. Do not write any code yet.\n\n${isPartial ? 'We are interested in studying what the buffer might look like when this change is half-applied. Describe a state where this code change has only been partially applied, but do not write any code yet.' : ''}\n`\n return askChatGPT(prompt)\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":163,"value":"updateDescription = getUpdateDescription(problemDescription, initialCode, true)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":170,"value":"md`${updateDescription}`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":42,"value":"$34","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":167,"value":"diff(codeWithSnippetDelimited, updatedCodeWithSnippetDelimited)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":475,"value":"copy(JSON.stringify({a: codeWithSnippetDelimited, b: updatedCodeWithSnippetDelimited}))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":478,"value":"import {copy, asyncCopy} from '@ryanseddon/copy'","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":175,"value":"$35","pinned":false,"hidden":true,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":216,"value":"md`${getUpdatedCodeWithSnippetDelimitedPrompt}`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":181,"value":"// result = askChatGPT(getUpdatedCodeWithSnippetDelimitedPrompt)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":193,"value":"// result = getFirstChoice(await getChatCompletion('gpt-4-turbo-preview', [{role: 'user', content: getUpdatedCodeWithSnippetDelimitedPrompt}]))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":198,"value":"// diff(codeWithSnippetDelimited, result)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":165,"value":"updatedCodeWithSnippetDelimited = getUpdatedCodeWithSnippetDelimited(updateDescription, true, delimiter, snippetType, snippetDescription, codeWithSnippetDelimited)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":26,"value":"function getRandomElementFromArray(arr) {\n if (arr.length === 0) {\n return undefined; // Return undefined if the array is empty\n }\n const randomIndex = Math.floor(Math.random() * arr.length);\n return arr[randomIndex];\n }","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":28,"value":"getRandomElementFromArray([1,2,3])","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":66,"value":"function validateTestWithContext({\n delimiter,\n codeWithSnippetDelimited,\n updatedCodeWithSnippetDelimited,\n problemDescription,\n snippetDescription,\n updateDescription\n }) {\n // Check if the delimiter occurs exactly twice each in the codeWithSnippetDelimited and updatedCodeWithSnippetDelimited\n\n // Ask ChatGPT if the updated delimiter position makes sense.\n const prompt = ``\n const result = askChatGPT(prompt)\n \n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":237,"value":"function validateTestWithoutContext({\n delimiter,\n codeWithSnippetDelimited,\n updatedCodeWithSnippetDelimited\n }) {\n // Check if the delimiter occurs exactly twice each in the codeWithSnippetDelimited and updatedCodeWithSnippetDelimited\n\n // Ask ChatGPT if the updated delimiter position makes sense.\n const prompt = ``\n const result = askChatGPT(prompt)\n \n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"}],"files":"$W36","stdlib":"1","titling":"auto"}} 30:"$2f:body"