Live programming @ University of Washington
Re-Tagging DemoTagged Code UpdatesRe-tagging SystemMagic Markup (PX'24 submission) -- Evaluation
Magic Markup (PX'24 submission) -- Benchmark Generation Tools
async function generateTest (
language=getRandomElementFromArray(languages),
snippetType = getRandomElementFromArray(snippetTypes),
updateIsPartial = getRandomElementFromArray([true, false])
) {
const problemDescription = await describeProblem(language, snippetType)
const initialCode = await generateInitialCode(language, snippetType, problemDescription)
const delimiter = await getDelimiter(initialCode, snippetType, language)
// const example = getExample(snippetType) // TODO use if needed
const snippetDescription = await describeSnippet(initialCode, snippetType)
const codeWithSnippetDelimited = await getCodeWithSnippetDelimited(delimiter, initialCode, snippetType, snippetDescription)//, example)
const updateDescription = await getUpdateDescription(problemDescription, initialCode, updateIsPartial)
const updatedCodeWithSnippetDelimited = await getUpdatedCodeWithSnippetDelimited(updateDescription, updateIsPartial, delimiter, snippetType, snippetDescription, codeWithSnippetDelimited)
return ({
language,
snippetType,
updateIsPartial,
delimiter,
codeWithSnippetDelimited,
updateDescription,
updatedCodeWithSnippetDelimited,
// context
problemDescription,
snippetDescription,
updateDescription
})
}
md`Completions API reference: https://platform.openai.com/docs/api-reference/completions/create`
import {openai, askChatGPT, usageHistory} from "9f772e010064bd3e"
import {diff, getChatCompletion, getFirstChoice} from "9f772e010064bd3e"
import {localStorageArray} from "9f772e010064bd3e"
// usageHistory = localStorageArray('usage-history')
usageHistory
data = usageHistory.get()
totalPromptTokens = usageHistory.get().filter(o => o.model.includes('3.5')).reduce((acc, v) => acc + v.usage.prompt_tokens, 0)
totalCompletionTokens = usageHistory.get().filter(o => o.model.includes('3.5')).reduce((acc, v) => acc + v.usage.completion_tokens, 0)
// Roughly 3192, broken down as
totalTokens = usageHistory.get().filter(o => o.model.includes('3.5')).reduce((acc, v) => acc + v.usage.total_tokens, 0)
async function askGPT4Turbo(prompt) {
const output = await getChatCompletion('gpt-4-turbo-preview', [{role: 'user', content: prompt}])
usageHistory.push({model: output.model, usage: output.usage, origin: 'bench'})
return getFirstChoice(output)
}
// askChatGPT("What is your name?")
languages = ['Python', 'JavaScript', 'C', 'JSX', 'Racket' ]
snippetTypes = ['constant', 'subexpression', 'variable assignment', 'loop body or code block', 'loop condition', 'function call']
snippetType = getRandomElementFromArray(snippetTypes)
language = getRandomElementFromArray(languages)
function describeProblem(language, snippetType) {
const prompt = `Briefly describe an intermediate-level ${language} programming problem including at least one ${snippetType} that can be solved in a single file. Use a creative, real-world framing. Describe steps to solve this problem. Do not provide code yet.`
return askChatGPT(prompt)
}
problemDescription = describeProblem(language, snippetType)
md`${problemDescription}`
async function askGPT35Instruct(prompt, maxTokens=500) {
return (await (openai.completions.create({model:'gpt-3.5-turbo-instruct', prompt, max_tokens: maxTokens}))).choices[0].text
}
async function generateInitialCode(language, snippetType, problemDescription) {
const prompt = `Consider the following problem description:
<problemDescription>
${problemDescription}
</problemDescription>
Write ${language} code that solves this problem. Your response will be executed directly and must be valid ${language} code. Your code must include at least one ${snippetType}. Again, your response should be ONLY code, and will be dumped directly into a file (please do not format it with markdown).
`
//return askGPT35Instruct(prompt)//(await (openai.completions.create({model:'gpt-3.5-turbo-instruct', prompt, max_tokens: 500}))).choices[0].text
return (await getChatCompletion('gpt-3.5-turbo', [{role: 'system', content: `You are a code generation assistant that produces valid ${language} code. Respond with unformatted code only.`}, {role: 'user', content: prompt}])).choices[0].message.content
// return askChatGPT(prompt)
}
initialCode =
(await generateInitialCode(language, snippetType, problemDescription))
function getDelimiter(initialCode, snippetType, language) {
const prompt = `You need to highlight a ${snippetType} from a program like the one below:
<exampleProgram>
${initialCode}
</exampleProgram>
Again, you need to highlight a ${snippetType} from this program. However, your friend will receive the document in a plaintext editor, so you need to use special characters to delimit the highlighted section. For example, your response could be $$, if the program doesn't include dollar signs--you don't want your friend to get confused when reading the program. The delimiter will be used both for the beginning and end of the highlighted section. Respond with ONLY your choice of delimiter.`
// prompt seems bad, hardcoding instead:
return '★'
return !(initialCode.includes('***')) ? '***' : !(initialCode.includes('****')) ? '****' : !(initialCode.includes('~~~')) ? '~~~' : '~~~~'
}
delimiter = getDelimiter(initialCode, language)
function describeSnippet(initialCode, snippetType) {
const prompt = `Describe a particular ${snippetType} in the program below.
<program>
${initialCode}
</program>
Again, describe a ${snippetType} in this program.
`
return askChatGPT(prompt)
}
snippetDescription = describeSnippet(initialCode, snippetType)
md`${snippetDescription}`
async function getCodeWithSnippetDelimited(delimiter, initialCode, snippetType, snippetDescription) {
// may need gpt4
const prompt = `Consider the following program and snippet description (NOTE the <program></program> tags are not part of the content!):
<program>
${initialCode}
</program>
<snippetDescription>
(${snippetType})
${snippetDescription}
</snippetDescription>
Mark the described snippet in the program with "${delimiter}" on both sides. For example, a constant might be marked in the program below like so:
<exampleResponse>
def foo():
a = ${delimiter}42${delimiter}
return a
</exampleResponse>
Respond with the entire program, and make sure to mark the selected ${snippetType} with "${delimiter}" on both sides. Your change MUST add a SINGLE PAIR of "${delimiter}" marks. Respond with code only. Do not add commentary, newlines, markdown syntax or alter the program in any other way. Do not surround the output with <program> or other tags.`
// it still adds newlines sometimes...
// return askGPT35Instruct(prompt)
// return askGPT4Turbo(prompt)
return (await getChatCompletion('gpt-4-turbo-preview', [{role: 'system', content: `You are a code generation assistant that marks the beginning and end of a section of ${language} code with 2 "${delimiter}"s for highlighting.`}, {role: 'user', content: prompt}])).choices[0].message.content
}
codeWithSnippetDelimited =
getCodeWithSnippetDelimited(delimiter, initialCode, snippetType, snippetDescription)
diff(initialCode, codeWithSnippetDelimited)
function validateSnippet(initialCode, delimiter, snippetDescription, codeWithSnippetDelimited) {
if ([...codeWithSnippetDelimited.matchAll(/\*\*\*/g)].length !== 2) {return false}
const prompt = `Consider the following program, snippet description, and student response:
<program>
${initialCode}
</program>
<snippetDescription>
${snippetDescription}
</snippetDescription>
<studentResponse>
${codeWithSnippetDelimited}
</studentResponse>
A student was asked to mark the snippet in this program CORRECTLY, with ONLY 2 instances of "${delimiter}", without changing any other lines. Did the student complete the task successfully? (Y/N, respond with just the letter Y/N)
`
return askChatGPT(prompt)
}
validateSnippet(initialCode, delimiter, snippetDescription, codeWithSnippetDelimited)
function getUpdateDescription(problemDescription, initialCode, isPartial) {
const prompt = `Consider the following problem:
<problemDescription>
${problemDescription}
</problemDescription>
Now consider this code that tries to solve the problem:
<program>
${initialCode}
</program>
Describe an interesting change or refactoring of this code that a real-world programmer might apply. Do not write any code yet.
${isPartial ? 'We are interested in studying what the buffer might look like when this change is half-applied. Describe a state where this code change has only been partially applied, but do not write any code yet.' : ''}
`
return askChatGPT(prompt)
}
updateDescription = getUpdateDescription(problemDescription, initialCode, true)
md`${updateDescription}`
function getUpdatedCodeWithSnippetDelimited(updateDescription, isPartial, delimiter, snippetType, snippetDescription, codeWithSnippetDelimited) {
const prompt = `Consider the following problem:
<problemDescription>
${problemDescription}
</problemDescription>
Now consider this code that tries to solve the problem:
<program>
${codeWithSnippetDelimited}
</program>
Note that a snippet from the code has been marked with a "${delimiter}" on both sides. This snippet is described as follows:
<snippetDescription>
(${snippetType})
${snippetDescription}
</snippetDescription>
Now consider the following description of ${isPartial ? 'a partial update' : 'an update'} to the program:
<updateDescription>
${updateDescription}
<updateDescription>
${isPartial ? 'Note again this is a partial update, since we are interested in studying what the buffer might look like for partial changes.' : ''}
Apply this update to the code as described. Your response should be purely code without any external discussion, and should fully copy any relevant sections of the original program. In order to obtain credit, you MUST maintain the "${delimiter}" marks on the snippet or its updated version. The contents of the snippet should be functionally identical in the new version of the code.
Again, the updated version of the code MUST have a SINGLE pair of "${delimiter}" marks referring to the same snippet in its new position or form.
`
return askGPT4Turbo(prompt) // NOTE this step requires GPT-4
}
diff(codeWithSnippetDelimited, updatedCodeWithSnippetDelimited)
copy(JSON.stringify({a: codeWithSnippetDelimited, b: updatedCodeWithSnippetDelimited}))
import {copy, asyncCopy} from '@ryanseddon/copy'
getUpdatedCodeWithSnippetDelimitedPrompt = `Consider the following problem:
<problemDescription>
${problemDescription}
</problemDescription>
Now consider this code that tries to solve the problem:
<program>
${codeWithSnippetDelimited}
</program>
Note that a snippet from the code has been marked with a "${delimiter}" on both sides. This snippet is described as follows:
<snippetDescription>
(${snippetType})
${snippetDescription}
</snippetDescription>
Now consider the following description of ${true ? 'a partial update' : 'an update'} to the program:
<updateDescription>
${updateDescription}
<updateDescription>
${true ? 'Note again this is a partial update, since we are interested in studying what the buffer might look like for partial changes.' : ''}
Apply this update to the code as described. In order to obtain credit, you MUST maintain the "${delimiter}" marks on the snippet or its updated version. The contents of the snippet should be functionally identical in the new version of the code.
Again, the updated version of the code MUST have a SINGLE pair of "${delimiter}" marks referring to the same snippet in its new position or form.
`
md`${getUpdatedCodeWithSnippetDelimitedPrompt}`
// result = askChatGPT(getUpdatedCodeWithSnippetDelimitedPrompt)
// result = getFirstChoice(await getChatCompletion('gpt-4-turbo-preview', [{role: 'user', content: getUpdatedCodeWithSnippetDelimitedPrompt}]))
// diff(codeWithSnippetDelimited, result)
updatedCodeWithSnippetDelimited = getUpdatedCodeWithSnippetDelimited(updateDescription, true, delimiter, snippetType, snippetDescription, codeWithSnippetDelimited)
function getRandomElementFromArray(arr) {
if (arr.length === 0) {
return undefined; // Return undefined if the array is empty
}
const randomIndex = Math.floor(Math.random() * arr.length);
return arr[randomIndex];
}
getRandomElementFromArray([1,2,3])
function validateTestWithContext({
delimiter,
codeWithSnippetDelimited,
updatedCodeWithSnippetDelimited,
problemDescription,
snippetDescription,
updateDescription
}) {
// Check if the delimiter occurs exactly twice each in the codeWithSnippetDelimited and updatedCodeWithSnippetDelimited
// Ask ChatGPT if the updated delimiter position makes sense.
const prompt = ``
const result = askChatGPT(prompt)
}
function validateTestWithoutContext({
delimiter,
codeWithSnippetDelimited,
updatedCodeWithSnippetDelimited
}) {
// Check if the delimiter occurs exactly twice each in the codeWithSnippetDelimited and updatedCodeWithSnippetDelimited
// Ask ChatGPT if the updated delimiter position makes sense.
const prompt = ``
const result = askChatGPT(prompt)
}
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
