2e:null 2f:["$","title",null,{"children":"Mastodon instances list | Nicolas Lambert | Observable"}] 34:T7d8,scrapPage = async function (params, page) { let _ = params; let url = `${_.proxy}${_.url}?lang=${_.lang}&minu=${_.minu}&maxu=${_.maxu}&minki=${_.minki}&minau=${_.minau}&black=${_.black}&creg=${_.creg}&appr=${_.appr}&lcok=${_.lcok}&ord=tusersd`; let htmlString = await fetch(`${url}&p=${page}`).then((res) => res.text()); const clean = (str) => { return str ? str.replace(" ", "") : ""; }; let $ = cheerio.load(htmlString); let nb = $(".ibody").length; let result = []; for (let i = 0; i < nb; i++) { let registrwarn = $(".ibody")[i]?.children[5]?.children[9]?.children[4]?.children[0]?.data; registrwarn = registrwarn == undefined ? "" : ` ${registrwarn}`; result.push({ instance: $(".ihead")[i]?.children[0]?.children[0]?.data, language: $(".ibody")[i]?.children[3]?.children[3]?.children[1]?.data.substr(1), users: +$(".ibody")[i]?.children[3]?.children[5]?.children[1]?.data, active_users_last_month: +$(".ibody")[i]?.children[3]?.children[7]?.children[1]?.data, active_users_last_6_month: +$(".ibody")[i]?.children[3]?.children[9]?.children[1]?.data, known_instances: +$(".ibody")[i]?.children[3]?.children[11]?.children[1]?.data, blacklisted: $(".ibody")[i]?.children[5]?.children[3]?.children[2]?.children[0] ?.data, registration: $(".ibody")[i]?.children[5]?.children[9]?.children[2]?.children[0] ?.data + registrwarn, email: $(".ibody")[i]?.children[5]?.children[11]?.children[2]?.children[0] ?.data // admin_account: // $(".ibody")[i]?.children[13]?.children[1]?.children[1]?.data, // date_of_creation: clean( // $(".ibody")[i]?.children[13]?.children[2]?.children[1]?.data // ), // admin_name: clean( // $(".ibody")[i]?.children[13]?.children[3]?.children[1]?.data // ) //image: $(".iimgc")[i]?.children[0]?.attribs?.src }); } return result; }35:[{"id":"dda53e85bca038e611d5436ebb16ff0f3b7a3fa916b3134402f006ffdc1aa305965348b43767a9d3bf028016f6345d50a209c80b8e5181be1333fbef665ab169","href":"https://static.observableusercontent.com/files/dda53e85bca038e611d5436ebb16ff0f3b7a3fa916b3134402f006ffdc1aa305965348b43767a9d3bf028016f6345d50a209c80b8e5181be1333fbef665ab169","name":"mastodon20221225.csv","type":"text/csv","lastModified":1671998574728.608,"size":710481},{"id":"95814e8df40ffc8a1a16e0d50cf6fd1b881470052127378009bf5c8c2fd789abdc2db9e8bb89d76ebcdf8e98af941af7c9c0c286eb353daa682a72d6cf4b4866","href":"https://static.observableusercontent.com/files/95814e8df40ffc8a1a16e0d50cf6fd1b881470052127378009bf5c8c2fd789abdc2db9e8bb89d76ebcdf8e98af941af7c9c0c286eb353daa682a72d6cf4b4866","name":"mastodon_22221114.csv","type":"text/csv","lastModified":1668546057291.866,"size":442336}] 32:{"id":"4c3db1b7368f7edb","slug":"mastodon-instances-list","title":"Mastodon instances list","thumbnail":"920693316cef718638d833dbe0ba8a5a63aef5d667e3ab527a25273b9d4258e6","access_level":"public","create_time":"$D2022-11-11T09:47:12.628Z","update_time":"$D2022-12-25T20:03:47.890Z","view_time":null,"first_public_time":"$D2022-11-15T09:26:09.276Z","has_unpublished_edits":null,"fork":false,"stars":11,"starred":false,"owner":{"type":"team","id":"6457aadd156df119","login":"neocarto","name":"Nicolas Lambert","avatar_url":"https://avatars-next.observableusercontent.com/c01b6664b447a3bb88f88dbfa170e3ef1d61f3a5b13c2fc613c8da5ef8bccb41","tier":"starter_2024"},"version":1975,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"01f8081399df8778","login":"neocarto","name":"Nicolas Lambert","avatar_url":"https://avatars-next.observableusercontent.com/c01b6664b447a3bb88f88dbfa170e3ef1d61f3a5b13c2fc613c8da5ef8bccb41","tier":null}],"members":[{"type":"individual","id":"01f8081399df8778","login":"neocarto","name":"Nicolas Lambert","avatar_url":"https://avatars-next.observableusercontent.com/c01b6664b447a3bb88f88dbfa170e3ef1d61f3a5b13c2fc613c8da5ef8bccb41","tier":null,"role":"editor"}],"comments":[{"id":"40bb08dea26267a6","reply_to_id":null,"content":"*\"Scraping\" ?","create_time":"$D2022-11-15T15:46:39.777Z","update_time":null,"resolved":true,"user":{"type":"individual","id":"07362516b5994994","login":"mootari","name":"Fabian Iwand","avatar_url":"https://avatars-next.observableusercontent.com/0d7defa821f38094c03bad23b9b360a5364e6e97e21fc238c39ddc48db7994ad","tier":null}},{"id":"92f5bb1b01cea65e","reply_to_id":null,"content":"What about naming the file using a YYYYMMDD date postfix? It certainly is sortable by date in case you save multiple files at different times _and_ it is not ambiguous about what is month and what is day...\nMy 2 cents","create_time":"$D2022-11-15T20:42:52.661Z","update_time":null,"resolved":true,"user":{"type":"individual","id":"c2f17e4031105178","login":"espinielli","name":"Enrico Spinielli","avatar_url":"https://avatars-next.observableusercontent.com/2bf5259d0c49a57cec6acc725062bdc41374a518de8772a2a051315f6be64567","tier":null}},{"id":"13f5291f3df560c3","reply_to_id":"92f5bb1b01cea65e","content":"Ok :-)","create_time":"$D2022-11-15T21:01:26.002Z","update_time":null,"resolved":true,"user":{"type":"individual","id":"01f8081399df8778","login":"neocarto","name":"Nicolas Lambert","avatar_url":"https://avatars-next.observableusercontent.com/c01b6664b447a3bb88f88dbfa170e3ef1d61f3a5b13c2fc613c8da5ef8bccb41","tier":null}}],"custom_thumbnail":true,"trashed":false,"lapsed":null,"subscribed":null,"forks":4,"collection":{"id":"938a1df250a32685","slug":"mastodon-visualizations","title":"Mastodon Visualizations","owner":{"id":"f35c755083683fe5","login":"observablehq","name":"Observable"}},"type":"notebook","body":{"title":"Mastodon instances list","cells":[{"id":0,"value":"# Mastodon instances list","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1164,"value":"There are currently __${intro.instances}__ Mastodon instances. \nSee the list on the [mastodon website](https://mastodon.help/instances).","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1921,"value":"You can now download the data as of December 25, 2022.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1903,"value":"button(\n await FileAttachment(\"mastodon20221225.csv\").csv(),\n \"mastodon20221225.csv\"\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1918,"value":"Or wait while up-to-date data is being retrieved.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1183,"value":"$${data.length < intro.instances ? `*Scraping in progress ${Math.round((data.length / intro.instances) * 100) }% (${data.length} / ${intro.instances})*` : button(data, \"mastodon.csv\") }","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1136,"value":"{\n let output;\n\n if (data.length < intro.instances) {\n output = html``;\n } else {\n output = html`Complete 👍`;\n }\n\n return output;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1127,"value":"mastodon = data","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"mastodon","database":"$undefined"},{"id":1232,"value":"### Appendix","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":15,"value":"params = ({\n proxy: \"https://corsproxy.io/?\",\n url: \"https://mastodon.help/instances\",\n lang: 0,\n minu: 0,\n maxu: 1000000000,\n minki: 0,\n minau: 0,\n black: 0,\n creg: 0,\n appr: 0,\n lcok: 0\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":65,"value":"cheerio = require(\"https://bundle.run/cheerio@1.0.0-rc.5\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1969,"value":"proxyUrl = (url) => `https://corsproxy.io/?${encodeURIComponent(url)}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1967,"value":"getUrl = ({ proxy, url, ...params }, page) =>\n proxyUrl(\n `${url}?${new URLSearchParams({\n ...params,\n ord: \"tusersd\",\n p: page\n })}`\n )","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":89,"value":"intro = {\n let htmlString = await fetch(getUrl(params, 1)).then((res) => res.text());\n\n let $ = cheerio.load(htmlString);\n let data = $(\".introe\")[0]\n .children.filter((d) => d.type == \"tag\")\n .map((d) => d.children[0].data.replaceAll(\",\", \"\"));\n\n // let data = $(\".introe\")[0]\n // .children.filter((d) => d.type == \"tag\")\n // .map((d) => +d.children[0].data.replaceAll(\".\", \"\"));\n\n return {\n instances: +data[0],\n users: +data[1],\n activelastmonth: +data[2],\n statuses: +data[3],\n nb_pages: Math.floor(data[0] / 10) + 1\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1612,"value":"getInstances_slow = async function* () {\n let output = [];\n let p = 1;\n for (let i = 1; i <= intro.nb_pages; i++) {\n let x = await scrapPage(params, i);\n output.push(x);\n yield output.flat();\n }\n}\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1721,"value":"getInstances = function* () {\n let output = [];\n let ids = [];\n let count = 0;\n\n for (let i = 1; i <= intro.nb_pages; i++) {\n scrapPage(params, i).then((d) => {\n let result = output.push(d);\n ids.push(i);\n count = ids.length;\n });\n }\n\n while (count <= intro.nb_pages + 1) {\n yield output.flat();\n if (count >= intro.nb_pages) count++;\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1084,"value":"data = getInstances()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":884,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1723,"value":"button = (data, filename = \"data.csv\") => {\n if (!data) throw new Error(\"Array of data required as first argument\");\n\n let downloadData;\n if (filename.includes(\".csv\")) {\n downloadData = new Blob([d3.csvFormat(data)], { type: \"text/csv\" });\n } else if (filename.includes(\".json\")) {\n downloadData = new Blob([JSON.stringify(data, null, 2)], {\n type: \"application/json\"\n });\n } else {\n downloadData = new Blob([data]);\n }\n\n const size = (downloadData.size / 1024).toFixed(0);\n const button = DOM.download(\n downloadData,\n filename,\n `Download ${filename} (~${size} KB)`\n );\n return button;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W35","stdlib":"1","titling":"auto"}} 33:"$32:body"