2c:null 2d:["$","title",null,{"children":"Unicorn on Rainbow Error Analysis | Daniel Manning | Observable"}] 32:[{"id":"9155f7206a5593cdd931e31f6d40c78173242aa69f604c6264834f852a50c2d6a44c107dc551be72321d39b619af4902c794585afec0a53af3a93df77518c03b","href":"https://static.observableusercontent.com/files/9155f7206a5593cdd931e31f6d40c78173242aa69f604c6264834f852a50c2d6a44c107dc551be72321d39b619af4902c794585afec0a53af3a93df77518c03b","name":"socialiqa_task_1060900_predictions","type":"application/octet-stream","lastModified":1651204112181.595,"size":3908},{"id":"8532148511dd6dd8c9d3df5f6255b33b43153554513d323bbd04d46f9fa9828dc8fed7abea21e0090bdaa6338a29904874f55ed85c7fd06756dbd13c1278316f","href":"https://static.observableusercontent.com/files/8532148511dd6dd8c9d3df5f6255b33b43153554513d323bbd04d46f9fa9828dc8fed7abea21e0090bdaa6338a29904874f55ed85c7fd06756dbd13c1278316f","name":"socialiqa_task_targets","type":"application/octet-stream","lastModified":1651204126412.92,"size":3908},{"id":"85eb6d4878713aef5ae19b70caa47a5b8aac3829a5a2de56085b2096c43093eefc77348595a0f78462387987d8efa4235562caf53e659e73dbd1617214cfd2a1","href":"https://static.observableusercontent.com/files/85eb6d4878713aef5ae19b70caa47a5b8aac3829a5a2de56085b2096c43093eefc77348595a0f78462387987d8efa4235562caf53e659e73dbd1617214cfd2a1","name":"validation.socialiqa.csv","type":"text/csv","lastModified":1651204180156.945,"size":573000}] 30:{"id":"967fedf58366026d","slug":"unicorn-on-rainbow-error-analysis","title":"Unicorn on Rainbow Error Analysis","thumbnail":"4484d90b9692954164f1af7bee596dae181d8408abcb3188f5759a0869ac1d09","access_level":"public","create_time":"$D2022-04-29T03:47:18.381Z","update_time":"$D2022-04-29T05:47:42.280Z","view_time":null,"first_public_time":"$D2022-04-29T05:49:08.507Z","has_unpublished_edits":null,"fork":false,"stars":0,"starred":false,"owner":{"type":"team","id":"bd2105631a778392","login":"dmisdm","name":"Daniel Manning","avatar_url":"https://avatars-next.observableusercontent.com/5bced4a52a61cf5a9cfef28321bdd9a8146d201fd9ce4393ef164f5cc868ee5b","tier":"starter_2024","bio":""},"version":156,"publish_version":156,"role":null,"authored":false,"authors":[{"type":"individual","id":"b7f352d825637a4f","login":"dmisdm","name":"Daniel Manning","avatar_url":"https://avatars-next.observableusercontent.com/5bced4a52a61cf5a9cfef28321bdd9a8146d201fd9ce4393ef164f5cc868ee5b","tier":null,"bio":""}],"members":[{"type":"individual","id":"b7f352d825637a4f","login":"dmisdm","name":"Daniel Manning","avatar_url":"https://avatars-next.observableusercontent.com/5bced4a52a61cf5a9cfef28321bdd9a8146d201fd9ce4393ef164f5cc868ee5b","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Unicorn on Rainbow Error Analysis","cells":[{"id":0,"value":"# Unicorn on Rainbow Error Analysis","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":120,"value":"Probe the set of errors that UNICORN made during one of the experiments (sequential fine-tine, LR = 1e-3, full-task)","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":3,"value":"## Social IQa\nA context + question + multiple choice answer dataset, on the theory of mind domain","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":127,"value":"# View an incorrect prediction","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":137,"value":"viewof randomPredButton = Inputs.button(\"Roll the dice (choose random prediction)\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":82,"value":"viewof incorrectAnswerNumber = {\n return Inputs.range([1, incorrectAnswers.length], {\n value: initialPredictionIndex,\n label: \"Scroll through incorrect predictions\",\n step: 1\n });\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":91,"value":"\nContext:\n\n**${incorrectAnswers[incorrectAnswerNumber- 1].context}**\n\n
\nQuestion: \n\n**${incorrectAnswers[incorrectAnswerNumber- 1].question}**\n\n
\nPredicted answer (by UNICORN):\n\n**${incorrectAnswers[incorrectAnswerNumber- 1].predictedAnswer}**\n\n
\nCorrect answer:\n\n**${incorrectAnswers[incorrectAnswerNumber- 1].correctAnswer}**\n\n
\nThe other answer out of the 3:\n\n**${incorrectAnswers[incorrectAnswerNumber- 1].unchosenAnswer}**","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":23,"value":"### Correct: ${socialIqaCorrect.length} / ${socialIqaTotal}","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":44,"value":"## Table of incorrect predictions and their context+questions:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":49,"value":"viewof incorrectAnswersTable = Inputs.table(incorrectAnswers)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":53,"value":"incorrectAnswers = questions\n .map((q, i) => {\n const a = q.inputs.match(/([^<]*)<\\/answerA>/)[1];\n const b = q.inputs.match(/([^<]*)<\\/answerB>/)[1];\n const c = q.inputs.match(/([^<]*)<\\/answerC>/)[1];\n const prediction = parseInt(predictions[i]);\n const target = parseInt(q.targets);\n const answers = [a, b, c];\n const predictedAnswer = answers[prediction - 1];\n const correctAnswer = answers[target - 1];\n const unchosenAnswer = answers.filter(\n (a) => a !== predictedAnswer && a !== correctAnswer\n );\n\n return {\n context: q.inputs.match(/([^<]*)<\\/context>/)[1],\n question: q.inputs.match(/([^<]*)<\\/question>/)[1],\n target: q.targets,\n prediction,\n answerA: a,\n answerB: b,\n answerC: c,\n predictedAnswer,\n correctAnswer,\n unchosenAnswer\n };\n })\n .filter((q, i) => targets[i] !== predictions[i])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":15,"value":"questions = FileAttachment(\"validation.socialiqa.csv\").csv()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":17,"value":"targets = (await FileAttachment(\"socialiqa_task_targets\").text()).split(\"\\n\").filter(ans => !!ans)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":18,"value":"predictions = (await FileAttachment(\"socialiqa_task_1060900_predictions\").text()).split(\"\\n\").filter(ans => !!ans)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":31,"value":"socialIqaCorrect = targets.filter((value, i) => predictions[i] === value)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":35,"value":"socialIqaTotal = targets.length","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":146,"value":"mutable initialPredictionIndex = {\n if(randomPredButton == 0) {\n return 1\n }\n return Math.round(Math.random() * (incorrectAnswers.length - 1) + 1)\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":76,"value":"# Total Prediction ${predictions.length}\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"}],"files":"$W32","stdlib":"1"}} 31:"$30:body"