3f:null 40:["$","title",null,{"children":"Translation QA | Simon Elvery | Observable"}] 46:[] 44:{"id":"5cbc8d5b63961fa2","slug":"translation-qa","title":"Translation QA","thumbnail":"eeabeb4fb9e4a23a884ff39a5d769d8f33bd756b35866208543bbad6d2a48e44","access_level":"public","create_time":"$D2019-10-02T03:36:43.871Z","update_time":"$D2019-10-15T00:06:12.428Z","view_time":null,"first_public_time":"$D2019-10-02T22:22:52.988Z","has_unpublished_edits":null,"fork":false,"stars":1,"starred":false,"owner":{"type":"team","id":"6bc6d212b7394707","login":"drzax","name":"Simon Elvery","avatar_url":"https://avatars-next.observableusercontent.com/518e046b7af01ba7994ea71c20bc5677dce397f949b5c56efeac67d6a97128c0","tier":"starter_2024","bio":"","following":false},"version":694,"publish_version":694,"role":null,"authored":false,"authors":[{"type":"individual","id":"62cb6a20cb42b527","login":"drzax","name":"Simon Elvery","avatar_url":"https://avatars-next.observableusercontent.com/518e046b7af01ba7994ea71c20bc5677dce397f949b5c56efeac67d6a97128c0","tier":null,"bio":"","following":false}],"members":[{"type":"individual","id":"62cb6a20cb42b527","login":"drzax","name":"Simon Elvery","avatar_url":"https://avatars-next.observableusercontent.com/518e046b7af01ba7994ea71c20bc5677dce397f949b5c56efeac67d6a97128c0","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Translation QA","cells":[{"id":0,"value":"sectionTitle = md`\n# Translation QA\n\nA bunch of utilities and checks for doing translations on the Australia Talks project.\n\nJump to: \n\n* [code](#sectionCode)\n* [output](#sectionOutput)\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":101,"value":"viewof languageCode = select({\n title: \"Select language\",\n description: \"Language to QA\",\n options: [\"zh\", \"vi\", \"ar\"],\n value: params.get(\"languageCode\") || \"zh\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":271,"value":"viewof correctionsFile = file({\n title: \"Load corrections\",\n description: \"A CSV of corrections\",\n accept: \".csv\",\n multiple: true\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":570,"value":"viewof instance= text({\n title: \"Code\",\n description: \"Enter the code\",\n value: params.get(\"instance\")\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":262,"value":"md`\n## Stats\n\n* *[Missing translations](#sectionMissingTranslations):* ${missingTranslations.length > 0 ? missingTranslations.length : '🎉'}\n* *[Non-translations](#sectionNonTranslations) (excluding numerals):* ${nonTranslations.length > 0 ? nonTranslations.length : '🎉'}\n* *[Unapplied corrections](#sectionUnappliedCorrections):* ${unappliedCorrections.length}\n* *[Empty keys](#sectionEmptyKeys):* ${emptyKeys.length}\n* *[Ambiguous translations](#sectionAmbiguousTranslations):* ${ambiguousTranslations.length}\n* *Missing from corrections file:* ${correctionsFile.length ? uniquePhrases.length - corrections.length : 'unknown'}\n`;","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":345,"value":"sectionUnappliedCorrections = md`\n## Un-applied corrections\n\n| English | Original | Correction |\n|--------------|--------------------------|---------------------------|${(unappliedCorrections||[]).map(c => `\n| ${c.english} | ${c.originalTranslation} | ${c.correctedTranslation} |`)}\n\n[top](#top)\n`\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":575,"value":"sectionAmbiguousTranslations = html`\n

Ambiguous translations

\n\n${ambiguousTranslations.map(p => `\n

${p[0]}

\n\n${p[1].map(t => `

${t}

`).join(\"\")}`\n).join(\"\")}\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":500,"value":"sectionMissingTranslations = html`\n

Missing translations

\n\n \n \n ${missingTranslations.map(p => `\n \n \n \n `)}\n \n

English	Locations
${p[0]}	${p[2].length}

top

`\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":652,"value":"nonMissingTranslations = html`\n

Non-translations

\n\n \n \n ${nonTranslations.map(p => `\n \n \n \n `)}\n \n

English	Locations
${p[0]}	${p[2].length}

top

`\n","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":541,"value":"sectionEmptyKeys = md`\n## Empty keys\n\n${emptyKeys.map(k => `* ${k[2]}\n`)}\n\n[top](#top)`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":316,"value":"phrases.filter(phrase => {\n let phraseCorrections = corrections.filter(correction => correction.english === phrase[0] && correction.translation === phrase[1] && correction.correction.length);\n return phraseCorrections.length\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":311,"value":"sectionOutput = md`## Corrections file ouput`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":636,"value":"viewof showOutputContent = checkbox({\n options: [{ value: \"toggle\", label: \"Show\"}],\n value: \"\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":75,"value":"showOutputContent ? html`\n\n

\n${uniquePhrases.filter(p => Boolean(p[0])).map(p => `\n\n \n \n \n \n`)}

English	${p[0]}
Translation	${p[1]}
Correction
---

` : html`Check the box above to show.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":133,"value":"sectionCode = md`\n----\n\n*Ignore stuff below here*, it is just a bunch of code to implement what's at the [top of the page](#top)\n\n----\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":502,"value":"uniquePhrasesFormattedd = uniquePhrases.map(formatPhrases)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":581,"value":"formatPhrases = p => p.map((d,i) => i > 1 ? d : formatPhrase(d))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":595,"value":"formatPhrase = d => (d || \"\").toString().replace(/\\[([^\\]]+)\\]\\s?\$[^\$]+\\)/g,\"$1\").trim().replace('\\n','

')","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2,"value":"en = fetchSectionsData('en', instance);","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":231,"value":"tr = fetchSectionsData(languageCode, instance);","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":36,"value":"phrases = traverse(en).reduce(traverseReducer,[]);","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":244,"value":"traverseReducer = function(acc, node) {\n if (this.isLeaf && phraseKeys.includes(this.key)) {\n let phrase = pluck(tr, this.path);\n if (phrase) phrase = phrase.toString().trim();\n acc.push([node, phrase, this.path.join('.')]);\n }\n return acc;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":300,"value":"uniquePhrases = phrases.reduce((acc, p) => {\n let phrase = formatPhrases(p);\n let existing = acc.find(d => d[0] === phrase[0] && d[1] === phrase[1]);\n if (existing) {\n existing[2] = existing[2].concat(phrase[2]); \n } else {\n acc.push([phrase[0],phrase[1],[phrase[2]]]);\n }\n return acc;\n}, [])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":249,"value":"phraseKeys = ['text','title','viz_headline','headline','viz_pre_text','viz_post_text','label', 'narrative','description']","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":41,"value":"pluck = (obj, path) => {\n let keys = [...path];\n let current = obj;\n let res;\n let key;\n while (key = keys.shift()) {\n current = current[key];\n res = current;\n }\n return res;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":103,"value":"params = new URL(document.baseURI).searchParams","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":105,"value":"import {select, text, file, checkbox} from \"@drzax/inputs\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":22,"value":"traverse = require('https://bundle.run/traverse@0.6.6')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":152,"value":"document.addEventListener('click', e => {\n if (e.target.className === 'copiable') {\n copy(e.target.innerHTML);\n e.target.classList.add('copied');\n setTimeout(() => e.target.classList.remove('copied'), 500);\n }\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":156,"value":"function copy(text) {\n var fakeElem = document.body.appendChild(document.createElement('textarea'));\n fakeElem.style.position = 'absolute';\n fakeElem.style.left = '-9999px';\n fakeElem.setAttribute('readonly', '');\n fakeElem.value = text;\n fakeElem.select();\n try {\n return document.execCommand('copy');\n } catch (err) {\n return false;\n } finally {\n fakeElem.parentNode.removeChild(fakeElem);\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":189,"value":"submit = async data => {\n console.log(data)\n return fetch('https://australia-talks-translation-qa-85zdbazlx.now.sh/api/submit', {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json'\n },\n body: JSON.stringify(data)\n }).then(response => {\n if (!response.ok) throw new Error(response.status);\n return 'ok';\n}).catch(error => {return error;})\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":211,"value":"function form(form) {\n const container = html`

${form}`;\n form.addEventListener(\"submit\", event => {\n event.preventDefault();\n console.log(formValue(form));\n });\n // form.addEventListener(\"change\", () => container.dispatchEvent(new CustomEvent(\"input\")));\n // form.addEventListener(\"input\", () => container.value = formValue(form));\n container.value = formValue(form);\n return container\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":212,"value":"function formValue(form) {\n const object = {};\n for (const input of form.elements) {\n if (input.disabled || !input.hasAttribute(\"name\")) continue;\n let value = input.value;\n switch (input.type) {\n case \"range\":\n case \"number\": {\n value = input.valueAsNumber;\n break;\n }\n case \"date\": {\n value = input.valueAsDate;\n break;\n }\n case \"radio\": {\n if (!input.checked) continue;\n break;\n }\n case \"checkbox\": {\n if (input.checked) value = true;\n else if (input.name in object) continue;\n else value = false;\n break;\n }\n case \"file\": {\n value = input.multiple ? input.files : input.files[0];\n break;\n }\n }\n object[input.name] = value;\n }\n return object;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":226,"value":"async function fetchSectionsData(lang, instance) {\n return fetch('https://ap2-engine.voxpoplabs.com/sections', {headers: {'instance-id': instance, locale: lang}}).then(response => {\n if (!response.ok) throw new Error(response.status);\n return response.json();\n}).catch(error => {return error;})\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":242,"value":"missingTranslations = uniquePhrases.filter(d => {\n let a = (d[0]||\"\").toString().trim();\n let b = (d[1]||\"\").toString().trim();\n return (b.length === 0 && a.length > 0);\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":648,"value":"nonTranslations = uniquePhrases.filter(d => {\n let a = (d[0]||\"\").toString().trim();\n let b = (d[1]||\"\").toString().trim();\n return a === b && a.length > 0 && !a.match(/[0-9\\-]+/);\n})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":258,"value":"emptyKeys = phrases.filter(d => (d[0]||\"\").toString().trim().length === 0)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":276,"value":"corrections = correctionsFile.length ? dsv.csvParse((await Files.text(correctionsFile[0])).split(\"---,\\n\").join(\"\"))\n .reduce((a,d) => {\n if (d.field) {\n a.push(d);\n return a;\n }\n\n if (d.field === '') {\n let idx = a.length-1;\n a[idx].value = a[idx].value + \"\\n\" + d.value\n }\n\n return a;\n\n }, [])\n .reduce((a, d) => {\n if (d.field === 'English') {\n a.push({});\n }\n let idx = a.length-1;\n a[idx][d.field.toLowerCase()] = d.value;\n return a;\n }, []) : [];","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":632,"value":"corrections.filter(d => d.correction.length > 0)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":278,"value":"dsv = require('d3-dsv')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":343,"value":"unappliedCorrections = corrections.filter(c => c.correction.length).reduce((acc, c) => {\n let correctedPhrases = uniquePhrases\n .filter(\n phrase => \n phrase[0] === c.english && \n formatPhrase(phrase[1]) !== formatPhrase(c.correction)\n )\n\n correctedPhrases.forEach(phrase => { \n acc.push({\n english: phrase[0], \n originalTranslation: formatPhrase(phrase[1]), \n correctedTranslation: formatPhrase(c.correction)\n })\n });\n return acc;\n}, []);","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":577,"value":"ambiguousTranslations = uniquePhrases.reduce((acc, p) => {\n let existing = acc.find(d => d[0] === p[0]);\n if (!existing) {\n existing = [p[0],[]];\n acc.push(existing);\n }\n existing[1].push(p[1]);\n return acc;\n}, []).filter(d => d[1].length > 1);","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":677,"value":"uniquePhrases","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W46","stdlib":"1"}} 45:"$44:body"