45:["$","title",null,{"children":"Integration Test Flakiness | Marco Munizaga | Observable"}] 4d:[] 49:{"id":"010dc5a9524f1fae","slug":"integration-test-flakiness","title":"Integration Test Flakiness","thumbnail":"65172e7c865188de23720e1ed8ffb9dfb239d92c36e12ac2177c8b09920fa48d","access_level":"public","create_time":"$D2022-06-02T17:45:20.249Z","update_time":"$D2022-06-02T17:45:20.249Z","view_time":null,"first_public_time":"$D2022-06-02T17:45:42.362Z","has_unpublished_edits":null,"fork":true,"stars":0,"starred":false,"owner":{"type":"team","id":"f78b694564a4f086","login":"realmarcopolo","name":"Marco Munizaga","avatar_url":"https://avatars-next.observableusercontent.com/4bbd3f03d8bbc28aef6040bbf1229b5ae3e36ecb680640d6c2ce018b2b2043a8","tier":"starter_2024","bio":"homo faber","followed_by_you":false,"follows_you":false},"version":1710,"publish_version":1710,"role":null,"authored":false,"authors":[{"type":"individual","id":"d6614599df7c1ce7","login":"realmarcopolo","name":"Marco Munizaga","avatar_url":"https://avatars-next.observableusercontent.com/4bbd3f03d8bbc28aef6040bbf1229b5ae3e36ecb680640d6c2ce018b2b2043a8","tier":null,"bio":"homo faber","followed_by_you":false,"follows_you":false}],"members":[{"type":"individual","id":"d6614599df7c1ce7","login":"realmarcopolo","name":"Marco Munizaga","avatar_url":"https://avatars-next.observableusercontent.com/4bbd3f03d8bbc28aef6040bbf1229b5ae3e36ecb680640d6c2ce018b2b2043a8","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Integration Test Flakiness","cells":[{"id":0,"value":"# Integration Test Flakiness\n${toc(instructions)}\n\nThe example dataset is showing ${jobs.length} job runs from the ${repo} repository’s integration tests.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1615,"value":"---","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":326,"value":"flakyChart = md`### Confirmed flakiness\nWhich jobs fail, but then pass on a re-run?\n\n${flakyRates.length ? \"\" : \"_No flaky jobs found._\"}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":328,"value":"{\n return html`\n \n ${d3.rollups(flaky, g => g, d => d.name).sort(([, a], [, b]) => d3.descending(a.length, b.length)).map(([name, jobs]) => `\n `)\n }\n

${name}	${jobs.map(j => `${j.run_number} (${j.branch})`).join(', ')}

${name}	${jobs.map(j => `${j.run_number} (${j.branch})`).join(', ')}

`\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":365,"value":"Plot.plot({\n height: failureRates.length * 40,\n marginLeft: 150,\n x: {\n tickFormat: \"%\",\n domain: [0, 1],\n grid: true\n },\n y: {\n reverse: true\n },\n marks: [\n Plot.frame(),\n Plot.barX(failureRates, { x: \"rate\", y: \"name\", fill: \"#666\" }),\n Plot.text(failureRates, { x: \"rate\", y: \"name\", text: d => Math.floor(d.rate * 100) + \"%\", dx: 10 }),\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1596,"value":"md`Below, see all jobs colored by conclusion. Click a run to go to its logs on GitHub.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":716,"value":"punchcard = Plot.plot({\n marginLeft: 150,\n width: 1200,\n height: 400,\n r: {\n range: [0, 8]\n },\n x: {\n axis: \"top\",\n padding: 0,\n label: null,\n tickFormat: d => (d | 0) % 5 === 0 ? d : null,\n grid: true,\n },\n y: {\n label: null,\n grid: true,\n },\n color,\n marks: [\n Plot.dot(jobs, Plot.group({ r: \"first\" }, { \n x: \"run_number\",\n y: \"name\",\n fill: \"conclusion\",\n fillOpacity: 0.58,\n r: d => d.conclusion === \"failure\" ? 3 : (d.conclusion === \"cancelled\" ? 2 : 1),\n })),\n Plot.tooltip(jobs, {\n x: \"run_number\",\n y: \"name\",\n content: d => ({ \n name: d.name,\n run_number: d.run_number,\n conclusion: d.conclusion,\n branch: d.branch,\n author: d.author,\n message: d.message\n }),\n direction: \"right\",\n dx: 15,\n dy: 5,\n onclick: (event, i) => window.open(jobs[i].html_url, \"_blank\")\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1634,"value":"Flaky tests over time on ${repo}. Bigger blobs mean more retries to eventually pass.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":1630,"value":"punchcardFlaky = Plot.plot({\n width: 1200,\n height: 400,\n marginLeft: 150,\n x: {\n axis: \"top\",\n padding: 0,\n label: null,\n tickFormat: d => (d | 0) % 5 === 0 ? d : null,\n grid: true,\n },\n r: {\n range: [0, 15]\n },\n y: {\n label: null,\n grid: true,\n },\n color,\n marks: [\n Plot.dot(flaky, Plot.group({ r: \"first\" }, { \n x: \"run_number\",\n y: \"name\",\n fill: \"conclusion\",\n fillOpacity: 0.58,\n r: d => d.conclusion === \"failure\" ? (d.run_attempt * 3) : (d.conclusion === \"cancelled\" ? 2 : 1),\n })),\n Plot.tooltip(flaky, {\n x: \"run_number\",\n y: \"name\",\n content: d => ({ \n name: d.name,\n run_number: d.run_number,\n conclusion: d.conclusion,\n branch: d.branch,\n author: d.author,\n message: d.message\n }),\n direction: \"right\",\n dx: 15,\n dy: 5,\n onclick: (event, i) => window.open(flaky[i].html_url, \"_blank\")\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":878,"value":"instructions = md`---\n## How to connect to your repo\n\n### 1. Fork this template\nYour changes will not be saved until you fork.\n\n### 2. Authenticate with GitHub\n\nCreate a GitHub [personal access token](https://github.com/settings/tokens) with no scopes selected, or the **repo** scope if you need access to a private repository. \n\nSave the token in a [Secret](https://observablehq.com/settings/secrets) named \\`GITHUB_ACCESS_TOKEN\\` and refresh this page. Treat the Secret like a password! Do not share or publish a notebook with a Secret; if you do, the Secret will refuse to load.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1199,"value":"apiKeyStatus","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1203,"value":"rateLimitSummary","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1207,"value":"md`### 3. Select a repo\n\nIf you’re not authenticated, we load cached results for the [Svelte](https://github.com/sveltejs/svelte) repository as a demonstration. Without authentication, you are rate-limited to 60 requests per hour; with a token, it goes up to 5,000 requests per hour. If you change the repo three times without authenticating, you’ll hit the rate limit.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":18,"value":"repo = \"libp2p/go-libp2p\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1251,"value":"md`### 4. Filter and check your data\n\nA repo may have several associated workflows; those workflows can be filtered by which branch of the repo they ran on.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":54,"value":"viewof workflow = Inputs.radio(\n workflows, {\n format: d => `${d.name} (${d.state})`, \n value: workflows[0], \n label: \"Workflow\"\n }\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":85,"value":"viewof branch = Inputs.select([\"\"].concat(...branches), {label: \"Branch\", format: name => name || 'All'})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1538,"value":"md`Showing ${jobs.length} job runs.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":88,"value":"viewof selectedJobs = Inputs.table(jobs, {\n columns: [\n \"name\", \n \"branch\", \n \"author\", \n \"conclusion\", \n \"message\",\n \"completed_at\", \n \"html_url\"\n ],\n format: {\n conclusion: prettyConclusion,\n html_url: d => html`${d.split(\"/\").slice(2, 3).join(\"/\")}/…`,\n completed_at: timeFormat\n }\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":537,"value":"md`---\n## Appendix\nOnce you fork for your own authenticated use, you can simplify the data fetching code below by removing \\`isSample\\`.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1190,"value":"isSample = repo === \"sveltejs/svelte\" && !hasApiKey","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1213,"value":"// Once authenticated:\n// workflows = fetchWorkflows(repo)\nworkflows = isSample ? sampleWorkflows : fetchWorkflows(repo, {limit: 300})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1216,"value":"// Once authenticated:\n// runs = fetchRuns(repo, workflow.id)\nruns = isSample ? sampleRuns : fetchMoreRuns(repo, workflow.id, {runsLimit: 100, jobsLimit: 200, pages: 4})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1218,"value":"branches = new Set(runs.map(r => r.head_branch))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":994,"value":"md`You can filter the jobs using the criteria of your choice:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":247,"value":"jobs = parseJobs(runs)\n .filter((job) => !branch || job.run.head_branch === branch) // filter to the selected branch\n // Determine which jobs should be considered \"integration tests\" by defining filtering criteria below\n .filter((job) => { \n // For example you could filter on the name of the job:\n // return job.name.indexOf(\"Test\") >= 0\n \n // by default we visualize all of the jobs\n return true\n })","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":296,"value":"flaky = retriedJobs.filter(j => j.conclusion === 'failure'\n && retriedJobs.some(j2 => j2.run_id === j.run_id && j2.conclusion === 'success'))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1705,"value":"flakyRunIDs = JSON.stringify(flaky.map(f => [f.run_id, f.run_attempt]).reduce((acc, [k,v]) => (acc[k]=v,acc), {}))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":298,"value":"flakyRates = d3.rollups(flaky, r => r.length / runs.length, d => d.name)\n .map(([name, rate]) => ({ name, rate }))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":355,"value":"failedWithoutRetry = {\n const failedRuns = new Set(d3.rollups(jobs, r => r.every(d => d.conclusion === \"failure\" || d.conclusion === \"cancelled\"), d => d.run_number)\n .filter(([, all]) => all)\n .map(([run]) => run))\n const js = jobs\n .filter(j => !failedRuns.has(j.run_number))\n .filter(j => j.conclusion === 'failure')\n .filter(j => !retriedJobs.some(j2 => j2.run_id === j.run_id))\n return js\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":361,"value":"failureRates = d3.rollups(failedWithoutRetry, r => r.length / runs.length, d => d.name)\n .map(([name, rate]) => ({ name, rate }))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":294,"value":"retriedJobs = {\n const jobRetries = jobs.filter(j => j.run_number.split('.')[1] > 1)\n const retriedRuns = new Set(jobRetries.map(j => j.run_id))\n const retriedJobs = jobs.filter(j => retriedRuns.has(j.run_id))\n return retriedJobs\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1493,"value":"md `---`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1186,"value":"md`### Helpers`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":264,"value":"prettyConclusion = (d) => {\n if(d == \"success\") return \"✅ success\"\n if(d == \"failure\") return \"⛔️ failure\"\n if(d == \"cancelled\") return \"⬜️ cancelled\"\n return d\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1059,"value":"color = ({domain: [\"cancelled\", \"failure\", \"success\"], range: [\"gray\", \"#cb2431\", \"#2cbe4e\"]})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":381,"value":"timeParser = d3.timeParse(\"%Y-%m-%dT%H:%M:%S%Z\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1566,"value":"timeFormat = d3.timeFormat(\"%a %m/%d %I:%M\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1497,"value":"md `---`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1182,"value":"md`### Imports`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1180,"value":"import {fetchWorkflows, fetchRuns, parseJobs, sampleWorkflows, sampleRuns, rateLimitSummary, hasApiKey, apiKeyStatus, fetchGithub} from \"@observablehq/github-actions-workflows-api\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1641,"value":"fetchMoreRuns = async function*(repo, workflow, { runsLimit = hasApiKey ? 50 : 20, jobsLimit = 100, pages = 2 } = {}) {\n const allRuns = []\n for (var i = 0; i < pages; i++) {\n const runs = (await fetchGithub(`/repos/${repo}/actions/workflows/${workflow}/runs`, {\n per_page: runsLimit,\n page: i+1,\n })).workflow_runs;\n allRuns.push(...runs)\n }\n if (!allRuns) {\n return undefined;\n } else {\n for await (const [run, { jobs }] of allRuns.map(async (r) => [\n r,\n await fetchGithub(r.jobs_url.replace(\"https://api.github.com\", \"\"), {\n filter: \"all\",\n per_page: jobsLimit\n })\n ])) {\n run.jobs = jobs;\n yield allRuns;\n }\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1647,"value":"fetchMoreRuns(repo, workflow.id)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1051,"value":"Plot = require(\"@observablehq/plot@0.1\").then(addTooltip)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":707,"value":"import {addTooltip} from \"@fil/experimental-plot-tooltip-01\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1544,"value":"import {toc} from \"@observablehq/templates-table-of-instructions\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1608,"value":"// For a notebook to access Secrets, it must statically reference one; this cell grants Secret access to the notebook from which we import the GitHub API helpers. `undefined` stops the Secret from appearing in the Inspector.\nundefined && Secret(\"GITHUB_ACCESS_TOKEN\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1697,"value":"Secret(\"GITHUB_ACCESS_TOKEN\").length","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W4d","stdlib":"1"}} 44:null