2c:null 2d:["$","title",null,{"children":"Integration Test Flakiness | Leonardo Kewitz | Observable"}] 32:[] 30:{"id":"e707b72275785fe7","slug":null,"title":"Integration Test Flakiness","thumbnail":"dd5a1772a14291f042b47f12b0b1bcc85323a9a924aaa322565fda77a9e666c3","access_level":"public","create_time":"$D2024-01-02T09:14:55.328Z","update_time":"$D2024-07-29T05:57:00.970Z","view_time":null,"first_public_time":"$D2024-01-02T09:14:55.328Z","has_unpublished_edits":null,"fork":{"id":"ca30abdfa3384264","slug":null,"title":"Integration Test Flakiness","owner":{"id":"8d89b5258c1e92a3","login":"franchb","name":"Ilya Rusin"}},"stars":0,"starred":false,"owner":{"type":"team","id":"5d3f29b3abfdfeb7","login":"kewitz","name":"Leonardo Kewitz","avatar_url":"https://avatars-next.observableusercontent.com/2aef377a8659ae1f2c5e918afe50bf95dfa0ddae59ae8db3b62a2de659f81d8a","tier":"starter_2024"},"version":1623,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"58f4527ace0580f9","login":"kewitz","name":"Leonardo Kewitz","avatar_url":"https://avatars-next.observableusercontent.com/2aef377a8659ae1f2c5e918afe50bf95dfa0ddae59ae8db3b62a2de659f81d8a","tier":null}],"members":[{"type":"individual","id":"58f4527ace0580f9","login":"kewitz","name":"Leonardo Kewitz","avatar_url":"https://avatars-next.observableusercontent.com/2aef377a8659ae1f2c5e918afe50bf95dfa0ddae59ae8db3b62a2de659f81d8a","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"title":"Integration Test Flakiness","cells":[{"id":0,"value":"# Integration Test Flakiness\n${toc(instructions)}\n\nThis template helps you quickly identify which jobs in your [GitHub Actions](https://docs.github.com/en/actions) are flaky and re-running. Flaky tests can be a thorn in your team’s side, subtly but constantly wasting engineering time. \n\nThe example dataset for this template is showing ${jobs.length} job runs from the ${repo} repository’s integration tests.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1612,"value":"See [**this tutorial video**](https://www.youtube.com/watch?v=4kjot3BCVf8) for a quick overview on how to use this template.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1615,"value":"---","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":326,"value":"flakyChart = md`### Confirmed flakiness\nWhich jobs fail, but then pass on a re-run?\n\n${flakyRates.length ? \"\" : \"_No flaky jobs found._\"}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":328,"value":"{\n return html`\n \n ${d3.rollups(flaky, g => g, d => d.name).sort(([, a], [, b]) => d3.descending(a.length, b.length)).map(([name, jobs]) => `\n `)\n }\n

${name}	${jobs.map(j => `${j.run_number} (${j.branch})`).join(', ')}

`\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":301,"value":"Plot.plot({\n height: flakyRates.length * 40,\n marginLeft: 150,\n x: {\n tickFormat: \"%\",\n domain: [0, 1],\n grid: true\n },\n y: {\n reverse: true\n },\n marks: [\n Plot.frame(),\n Plot.barX(flakyRates, { x: \"rate\", y: \"name\", fill: \"#666\" }),\n Plot.text(flakyRates, { x: \"rate\", y: \"name\", text: d => Math.floor(d.rate * 100) + \"%\", dx: 10 }),\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":353,"value":"md`### Possible flakiness\nWhich jobs fail, but we don’t have a re-run to compare it to? Excludes runs in which all jobs fail.\n\n${failedWithoutRetry.length ? \"\" : \"_No flaky jobs found._\"}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":358,"value":"{\n const tests = d3.groups(failedWithoutRetry, d => d.name).sort((a, b) => b[1].length - a[1].length)\n return html`\n \n ${tests.map(([name, jobs]) => `\n \n \n \n `)}\n

${name}	${jobs.map(j => `${j.run_number} (${j.branch})`).join(', ')}

`\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":365,"value":"Plot.plot({\n height: failureRates.length * 40,\n marginLeft: 150,\n x: {\n tickFormat: \"%\",\n domain: [0, 1],\n grid: true\n },\n y: {\n reverse: true\n },\n marks: [\n Plot.frame(),\n Plot.barX(failureRates, { x: \"rate\", y: \"name\", fill: \"#666\" }),\n Plot.text(failureRates, { x: \"rate\", y: \"name\", text: d => Math.floor(d.rate * 100) + \"%\", dx: 10 }),\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1596,"value":"md`Below, see all jobs colored by conclusion. Click a run to go to its logs on GitHub.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":716,"value":"punchcard = Plot.plot({\n marginLeft: 150,\n x: {\n axis: \"top\",\n padding: 0,\n label: null,\n tickFormat: d => (d | 0) % 5 === 0 ? d : null,\n grid: true,\n },\n y: {\n label: null,\n grid: true,\n },\n color,\n marks: [\n Plot.dot(jobs, Plot.group({ r: \"first\" }, { \n x: \"run_number\",\n y: \"name\",\n fill: \"conclusion\",\n fillOpacity: 0.58,\n r: d => d.conclusion === \"failure\" ? 3 : (d.conclusion === \"cancelled\" ? 2 : 1),\n })),\n Plot.tooltip(jobs, {\n x: \"run_number\",\n y: \"name\",\n content: d => ({ \n name: d.name,\n run_number: d.run_number,\n conclusion: d.conclusion,\n branch: d.branch,\n author: d.author,\n message: d.message\n }),\n direction: \"right\",\n dx: 15,\n dy: 5,\n onclick: (event, i) => window.open(jobs[i].html_url, \"_blank\")\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":878,"value":"instructions = md`---\n## How to connect to your repo\n\n### 1. Fork this template\nYour changes will not be saved until you fork.\n\n### 2. Authenticate with GitHub\n\nCreate a GitHub [personal access token](https://github.com/settings/tokens) with no scopes selected, or the **repo** scope if you need access to a private repository. \n\nSave the token in a [Secret](https://observablehq.com/settings/secrets) named \\`GITHUB_ACCESS_TOKEN\\` and refresh this page. Treat the Secret like a password! Do not share or publish a notebook with a Secret; if you do, the Secret will refuse to load.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1199,"value":"apiKeyStatus","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1203,"value":"rateLimitSummary","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1207,"value":"md`### 3. Select a repo\n\nIf you’re not authenticated, we load cached results for the [Svelte](https://github.com/sveltejs/svelte) repository as a demonstration. Without authentication, you are rate-limited to 60 requests per hour; with a token, it goes up to 5,000 requests per hour. If you change the repo three times without authenticating, you’ll hit the rate limit.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":18,"value":"repo = \"opencollective/opencollective-frontend\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1251,"value":"md`### 4. Filter and check your data\n\nA repo may have several associated workflows; those workflows can be filtered by which branch of the repo they ran on.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":54,"value":"viewof workflow = Inputs.radio(\n workflows, {\n format: d => `${d.name} (${d.state})`, \n value: workflows[0], \n label: \"Workflow\"\n }\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":85,"value":"viewof branch = Inputs.select([\"\"].concat(...branches), {label: \"Branch\", format: name => name || 'All'})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1538,"value":"md`Showing ${jobs.length} job runs.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":88,"value":"viewof selectedJobs = Inputs.table(jobs, {\n columns: [\n \"name\", \n \"branch\", \n \"author\", \n \"conclusion\", \n \"message\",\n \"completed_at\", \n \"html_url\"\n ],\n format: {\n conclusion: prettyConclusion,\n html_url: d => html`${d.split(\"/\").slice(2, 3).join(\"/\")}/…`,\n completed_at: timeFormat\n }\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":537,"value":"md`---\n## Appendix\nOnce you fork for your own authenticated use, you can simplify the data fetching code below by removing \\`isSample\\`.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1190,"value":"isSample = repo === \"sveltejs/svelte\" && !hasApiKey","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1213,"value":"// Once authenticated:\n// workflows = fetchWorkflows(repo)\nworkflows = isSample ? sampleWorkflows : fetchWorkflows(repo)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1216,"value":"// Once authenticated:\n// runs = fetchRuns(repo, workflow.id)\nruns = isSample ? sampleRuns : fetchRuns(repo, workflow.id)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1218,"value":"branches = new Set(runs.map(r => r.head_branch))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":994,"value":"md`You can filter the jobs using the criteria of your choice:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":247,"value":"jobs = parseJobs(runs)\n .filter((job) => !branch || job.run.head_branch === branch) // filter to the selected branch\n // Determine which jobs should be considered \"integration tests\" by defining filtering criteria below\n .filter((job) => { \n // For example you could filter on the name of the job:\n // return job.name.indexOf(\"Test\") >= 0\n \n // by default we visualize all of the jobs\n return true\n })","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":296,"value":"flaky = retriedJobs.filter(j => j.conclusion === 'failure'\n && retriedJobs.some(j2 => j2.run_id === j.run_id && j2.conclusion === 'success'))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":298,"value":"flakyRates = d3.rollups(flaky, r => r.length / runs.length, d => d.name)\n .map(([name, rate]) => ({ name, rate }))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":355,"value":"failedWithoutRetry = {\n const failedRuns = new Set(d3.rollups(jobs, r => r.every(d => d.conclusion === \"failure\" || d.conclusion === \"cancelled\"), d => d.run_number)\n .filter(([, all]) => all)\n .map(([run]) => run))\n const js = jobs\n .filter(j => !failedRuns.has(j.run_number))\n .filter(j => j.conclusion === 'failure')\n .filter(j => !retriedJobs.some(j2 => j2.run_id === j.run_id))\n return js\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":361,"value":"failureRates = d3.rollups(failedWithoutRetry, r => r.length / runs.length, d => d.name)\n .map(([name, rate]) => ({ name, rate }))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":294,"value":"retriedJobs = {\n const jobRetries = jobs.filter(j => j.run_number.split('.')[1] > 1)\n const retriedRuns = new Set(jobRetries.map(j => j.run_id))\n const retriedJobs = jobs.filter(j => retriedRuns.has(j.run_id))\n return retriedJobs\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1493,"value":"md `---`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1186,"value":"md`### Helpers`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":264,"value":"prettyConclusion = (d) => {\n if(d == \"success\") return \"✅ success\"\n if(d == \"failure\") return \"⛔️ failure\"\n if(d == \"cancelled\") return \"⬜️ cancelled\"\n return d\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1059,"value":"color = ({domain: [\"cancelled\", \"failure\", \"success\"], range: [\"gray\", \"#cb2431\", \"#2cbe4e\"]})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":381,"value":"timeParser = d3.timeParse(\"%Y-%m-%dT%H:%M:%S%Z\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1566,"value":"timeFormat = d3.timeFormat(\"%a %m/%d %I:%M\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1497,"value":"md `---`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1182,"value":"md`### Imports`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1180,"value":"import {fetchWorkflows, fetchRuns, parseJobs, sampleWorkflows, sampleRuns, rateLimitSummary, hasApiKey, apiKeyStatus} from \"@observablehq/github-actions-workflows-api\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1051,"value":"Plot = require(\"@observablehq/plot@0.1\").then(addTooltip)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":707,"value":"import {addTooltip} from \"@fil/experimental-plot-tooltip-01\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1544,"value":"import {toc} from \"@observablehq/templates-table-of-instructions\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1608,"value":"// For a notebook to access Secrets, it must statically reference one; this cell grants Secret access to the notebook from which we import the GitHub API helpers. `undefined` stops the Secret from appearing in the Inspector.\nundefined && Secret(\"GITHUB_ACCESS_TOKEN\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W32","stdlib":"1","titling":"auto"}} 31:"$30:body"