2b:null 2c:["$","title",null,{"children":"Integration Test Flakiness | Observable | Observable"}] 31:[] 2f:{"id":"bf554c34c14cc2e1","slug":"integration-test-flakiness","title":"Integration Test Flakiness","thumbnail":"ec044b77d511dab263da5008552a4f25d32a60dbf9182cb984a3c1a1d1401e92","access_level":"public","create_time":"$D2021-04-06T18:41:27.841Z","update_time":"$D2021-09-21T23:34:20.590Z","view_time":null,"first_public_time":"$D2021-05-25T14:37:54.386Z","fork":true,"stars":47,"starred":false,"owner":{"type":"team","id":"f35c755083683fe5","login":"observablehq","name":"Observable","avatar_url":"https://avatars-next.observableusercontent.com/424c6c1f7794371c9915a160519bdfb34cf34e50069fcd6def38a1a04d4e9702","tier":"enterprise_2024"},"version":1621,"publish_version":1621,"role":null,"authored":false,"authors":[{"type":"individual","id":"595405df3c534894","login":"enjalot","name":"Ian Johnson","avatar_url":"https://avatars-next.observableusercontent.com/549bdb0c368027004b51c4e95459e9c224331e0fec9c4cc97d181f83e55ab7f9","tier":null}],"members":[{"type":"individual","id":"074c414ad1d825f5","login":"mbostock","name":"Mike Bostock","avatar_url":"https://avatars-next.observableusercontent.com/7dbba99ced40bd5ab8013c99b1a11da61231bce321ba1148a9d16ad0f121550a","tier":null,"role":"editor"},{"type":"individual","id":"0bb2842b01e0c652","login":"tophtucker","name":"Toph Tucker","avatar_url":"https://avatars-next.observableusercontent.com/3706c31a8f9c59bb716c022c04b9c073144cd1368b78ac85f49139be6c93c34e","tier":null,"role":"editor"},{"type":"team","id":"f35c755083683fe5","login":"observablehq","name":"Observable","avatar_url":"https://avatars-next.observableusercontent.com/424c6c1f7794371c9915a160519bdfb34cf34e50069fcd6def38a1a04d4e9702","tier":"enterprise_2024","role":"viewer"}],"comments":[{"id":"d8334976ec705d66","reply_to_id":null,"content":"I suspect you can get rid of vega-lite in this notebook now?","create_time":"$D2021-04-06T22:09:48.032Z","update_time":null,"resolved":true,"user":{"type":"individual","id":"c1daf2951ad616fb","login":"cobus","name":"Cobus Theunissen","avatar_url":"https://avatars-next.observableusercontent.com/4a17a065d7a0f5f9a64a852f7b26a4f875c2ff69eb17264c3bd9f81e69285c05","tier":null}},{"id":"8a8384a6acbf76ad","reply_to_id":"d8334976ec705d66","content":"indeed. forgot to delete","create_time":"$D2021-04-09T21:09:15.600Z","update_time":null,"resolved":true,"user":{"type":"individual","id":"595405df3c534894","login":"enjalot","name":"Ian Johnson","avatar_url":"https://avatars-next.observableusercontent.com/549bdb0c368027004b51c4e95459e9c224331e0fec9c4cc97d181f83e55ab7f9","tier":null}},{"id":"4443caea829194e7","reply_to_id":null,"content":"this may be strange UX, but by not putting `secret` in its own cell, we wont render it out in the inspector. I feel this also adds transparency around what this notebook is doing.","create_time":"$D2021-04-09T22:38:04.251Z","update_time":null,"resolved":true,"user":{"type":"individual","id":"595405df3c534894","login":"enjalot","name":"Ian Johnson","avatar_url":"https://avatars-next.observableusercontent.com/549bdb0c368027004b51c4e95459e9c224331e0fec9c4cc97d181f83e55ab7f9","tier":null}},{"id":"90665cd939fd2392","reply_to_id":null,"content":"perhaps this is a little less confusing than before. I don't want the secret to show up as the cell value because showing it as plain text could accidentally leak it via screenshares and screenshots","create_time":"$D2021-04-14T02:43:12.132Z","update_time":null,"resolved":true,"user":{"type":"individual","id":"595405df3c534894","login":"enjalot","name":"Ian Johnson","avatar_url":"https://avatars-next.observableusercontent.com/549bdb0c368027004b51c4e95459e9c224331e0fec9c4cc97d181f83e55ab7f9","tier":null}},{"id":"51695b424be27e63","reply_to_id":"90665cd939fd2392","content":"Nice idea","create_time":"$D2021-04-14T04:59:30.824Z","update_time":null,"resolved":true,"user":{"type":"individual","id":"c1daf2951ad616fb","login":"cobus","name":"Cobus Theunissen","avatar_url":"https://avatars-next.observableusercontent.com/4a17a065d7a0f5f9a64a852f7b26a4f875c2ff69eb17264c3bd9f81e69285c05","tier":null}}],"custom_thumbnail":true,"trashed":false,"subscribed":null,"forks":198,"collection":{"id":"0ca4eed6b9ea0315","slug":"templates-developer-workflows","title":"Templates: Developer Workflows","owner":{"id":"f35c755083683fe5","login":"observablehq","name":"Observable"}},"type":"notebook","body":{"title":"Integration Test Flakiness","cells":[{"id":0,"value":"# Integration Test Flakiness\n${toc(instructions)}\n\nThis template helps you quickly identify which jobs in your [GitHub Actions](https://docs.github.com/en/actions) are flaky and re-running. Flaky tests can be a thorn in your team’s side, subtly but constantly wasting engineering time. \n\nThe example dataset for this template is showing ${jobs.length} job runs from the ${repo} repository’s integration tests.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":1612,"value":"See [**this tutorial video**](https://www.youtube.com/watch?v=4kjot3BCVf8) for a quick overview on how to use this template.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":1615,"value":"---","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":326,"value":"flakyChart = md`### Confirmed flakiness\nWhich jobs fail, but then pass on a re-run?\n\n${flakyRates.length ? \"\" : \"_No flaky jobs found._\"}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":328,"value":"{\n return html`\n \n ${d3.rollups(flaky, g => g, d => d.name).sort(([, a], [, b]) => d3.descending(a.length, b.length)).map(([name, jobs]) => `\n `)\n }\n

${name}	${jobs.map(j => `${j.run_number} (${j.branch})`).join(', ')}

`\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":301,"value":"Plot.plot({\n height: flakyRates.length * 40,\n marginLeft: 150,\n x: {\n tickFormat: \"%\",\n domain: [0, 1],\n grid: true\n },\n y: {\n reverse: true\n },\n marks: [\n Plot.frame(),\n Plot.barX(flakyRates, { x: \"rate\", y: \"name\", fill: \"#666\" }),\n Plot.text(flakyRates, { x: \"rate\", y: \"name\", text: d => Math.floor(d.rate * 100) + \"%\", dx: 10 }),\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":353,"value":"md`### Possible flakiness\nWhich jobs fail, but we don’t have a re-run to compare it to? Excludes runs in which all jobs fail.\n\n${failedWithoutRetry.length ? \"\" : \"_No flaky jobs found._\"}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":358,"value":"{\n const tests = d3.groups(failedWithoutRetry, d => d.name).sort((a, b) => b[1].length - a[1].length)\n return html`\n \n ${tests.map(([name, jobs]) => `\n \n \n \n `)}\n

${name}	${jobs.map(j => `${j.run_number} (${j.branch})`).join(', ')}

`\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":365,"value":"Plot.plot({\n height: failureRates.length * 40,\n marginLeft: 150,\n x: {\n tickFormat: \"%\",\n domain: [0, 1],\n grid: true\n },\n y: {\n reverse: true\n },\n marks: [\n Plot.frame(),\n Plot.barX(failureRates, { x: \"rate\", y: \"name\", fill: \"#666\" }),\n Plot.text(failureRates, { x: \"rate\", y: \"name\", text: d => Math.floor(d.rate * 100) + \"%\", dx: 10 }),\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1596,"value":"md`Below, see all jobs colored by conclusion. Click a run to go to its logs on GitHub.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":716,"value":"punchcard = Plot.plot({\n marginLeft: 150,\n x: {\n axis: \"top\",\n padding: 0,\n label: null,\n tickFormat: d => (d | 0) % 5 === 0 ? d : null,\n grid: true,\n },\n y: {\n label: null,\n grid: true,\n },\n color,\n marks: [\n Plot.dot(jobs, Plot.group({ r: \"first\" }, { \n x: \"run_number\",\n y: \"name\",\n fill: \"conclusion\",\n fillOpacity: 0.58,\n r: d => d.conclusion === \"failure\" ? 3 : (d.conclusion === \"cancelled\" ? 2 : 1),\n })),\n Plot.tooltip(jobs, {\n x: \"run_number\",\n y: \"name\",\n content: d => ({ \n name: d.name,\n run_number: d.run_number,\n conclusion: d.conclusion,\n branch: d.branch,\n author: d.author,\n message: d.message\n }),\n direction: \"right\",\n dx: 15,\n dy: 5,\n onclick: (event, i) => window.open(jobs[i].html_url, \"_blank\")\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":878,"value":"instructions = md`---\n## How to connect to your repo\n\n### 1. Fork this template\nYour changes will not be saved until you fork.\n\n### 2. Authenticate with GitHub\n\nCreate a GitHub [personal access token](https://github.com/settings/tokens) with no scopes selected, or the **repo** scope if you need access to a private repository. \n\nSave the token in a [Secret](https://observablehq.com/settings/secrets) named \\`GITHUB_ACCESS_TOKEN\\` and refresh this page. Treat the Secret like a password! Do not share or publish a notebook with a Secret; if you do, the Secret will refuse to load.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1199,"value":"apiKeyStatus","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1203,"value":"rateLimitSummary","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1207,"value":"md`### 3. Select a repo\n\nIf you’re not authenticated, we load cached results for the [Svelte](https://github.com/sveltejs/svelte) repository as a demonstration. Without authentication, you are rate-limited to 60 requests per hour; with a token, it goes up to 5,000 requests per hour. If you change the repo three times without authenticating, you’ll hit the rate limit.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":18,"value":"repo = \"sveltejs/svelte\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1251,"value":"md`### 4. Filter and check your data\n\nA repo may have several associated workflows; those workflows can be filtered by which branch of the repo they ran on.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":54,"value":"viewof workflow = Inputs.radio(\n workflows, {\n format: d => `${d.name} (${d.state})`, \n value: workflows[0], \n label: \"Workflow\"\n }\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":85,"value":"viewof branch = Inputs.select([\"\"].concat(...branches), {label: \"Branch\", format: name => name || 'All'})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1538,"value":"md`Showing ${jobs.length} job runs.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":88,"value":"viewof selectedJobs = Inputs.table(jobs, {\n columns: [\n \"name\", \n \"branch\", \n \"author\", \n \"conclusion\", \n \"message\",\n \"completed_at\", \n \"html_url\"\n ],\n format: {\n conclusion: prettyConclusion,\n html_url: d => html`${d.split(\"/\").slice(2, 3).join(\"/\")}/…`,\n completed_at: timeFormat\n }\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":537,"value":"md`---\n## Appendix\nOnce you fork for your own authenticated use, you can simplify the data fetching code below by removing \\`isSample\\`.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1190,"value":"isSample = repo === \"sveltejs/svelte\" && !hasApiKey","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1213,"value":"// Once authenticated:\n// workflows = fetchWorkflows(repo)\nworkflows = isSample ? sampleWorkflows : fetchWorkflows(repo)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1216,"value":"// Once authenticated:\n// runs = fetchRuns(repo, workflow.id)\nruns = isSample ? sampleRuns : fetchRuns(repo, workflow.id)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1218,"value":"branches = new Set(runs.map(r => r.head_branch))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":994,"value":"md`You can filter the jobs using the criteria of your choice:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":247,"value":"jobs = parseJobs(runs)\n .filter((job) => !branch || job.run.head_branch === branch) // filter to the selected branch\n // Determine which jobs should be considered \"integration tests\" by defining filtering criteria below\n .filter((job) => { \n // For example you could filter on the name of the job:\n // return job.name.indexOf(\"Test\") >= 0\n \n // by default we visualize all of the jobs\n return true\n })","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":296,"value":"flaky = retriedJobs.filter(j => j.conclusion === 'failure'\n && retriedJobs.some(j2 => j2.run_id === j.run_id && j2.conclusion === 'success'))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":298,"value":"flakyRates = d3.rollups(flaky, r => r.length / runs.length, d => d.name)\n .map(([name, rate]) => ({ name, rate }))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":355,"value":"failedWithoutRetry = {\n const failedRuns = new Set(d3.rollups(jobs, r => r.every(d => d.conclusion === \"failure\" || d.conclusion === \"cancelled\"), d => d.run_number)\n .filter(([, all]) => all)\n .map(([run]) => run))\n const js = jobs\n .filter(j => !failedRuns.has(j.run_number))\n .filter(j => j.conclusion === 'failure')\n .filter(j => !retriedJobs.some(j2 => j2.run_id === j.run_id))\n return js\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":361,"value":"failureRates = d3.rollups(failedWithoutRetry, r => r.length / runs.length, d => d.name)\n .map(([name, rate]) => ({ name, rate }))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":294,"value":"retriedJobs = {\n const jobRetries = jobs.filter(j => j.run_number.split('.')[1] > 1)\n const retriedRuns = new Set(jobRetries.map(j => j.run_id))\n const retriedJobs = jobs.filter(j => retriedRuns.has(j.run_id))\n return retriedJobs\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1493,"value":"md `---`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1186,"value":"md`### Helpers`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":264,"value":"prettyConclusion = (d) => {\n if(d == \"success\") return \"✅ success\"\n if(d == \"failure\") return \"⛔️ failure\"\n if(d == \"cancelled\") return \"⬜️ cancelled\"\n return d\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1059,"value":"color = ({domain: [\"cancelled\", \"failure\", \"success\"], range: [\"gray\", \"#cb2431\", \"#2cbe4e\"]})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":381,"value":"timeParser = d3.timeParse(\"%Y-%m-%dT%H:%M:%S%Z\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1566,"value":"timeFormat = d3.timeFormat(\"%a %m/%d %I:%M\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1497,"value":"md `---`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1182,"value":"md`### Imports`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1180,"value":"import {fetchWorkflows, fetchRuns, parseJobs, sampleWorkflows, sampleRuns, rateLimitSummary, hasApiKey, apiKeyStatus} from \"@observablehq/github-actions-workflows-api\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1051,"value":"Plot = require(\"@observablehq/plot@0.1\").then(addTooltip)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":707,"value":"import {addTooltip} from \"@fil/experimental-plot-tooltip-01\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1544,"value":"import {toc} from \"@observablehq/templates-table-of-instructions\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1608,"value":"// For a notebook to access Secrets, it must statically reference one; this cell grants Secret access to the notebook from which we import the GitHub API helpers. `undefined` stops the Secret from appearing in the Inspector.\nundefined && Secret(\"GITHUB_ACCESS_TOKEN\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"}],"files":"$W31","stdlib":"1","titling":"auto"}} 30:"$2f:body"