2c:null 2d:["$","title",null,{"children":"The Arxiv in WebGL. | Benjamin Schmidt | Observable"}] 32:T786,# The Arxiv in WebGL. I've been periodically working on [Deepscatter, a JS library for visualizing millions to billions of points using WebGL](https://github.com/CreatingData/deepscatter). It seems to be basically working, even inside Observable! As a trouble-shooter demonstration of some of its capabilities, here's a visualization of a million-some scientific articles from the ArXiv. The embedding here is from the [cartolabe](https://cartolabe.fr/about) project, which has made a similar browser but using map tiles rather than real-time rendered points. They tokenize articles, vectorize them using [Latent Semantic Analysis](https://en.wikipedia.org/wiki/Latent_semantic_analysis), and projected into two dimensions using [UMAP](https://umap-learn.readthedocs.io/en/latest/). You can **zoom** into regions to load more articles, **and click on any article for the full abstract and links**, thanks to arXiv's own open access interoperability standards. (The click handlers in deepscatter can be dynamically changed, and I use that here to load abstracts over the web.) By tiling data into quadtrees and shipping it over the web using Apache Arrow, it's possible to send a great deal more information about each book than in a flat map visualization--enough to show allow fairly rich abstraction over points, and engage in various types of filters. Full text search over metadata isn't currently supported, because Arrow is slow on strings--that will have to wait for someone to build a working DuckDB implementation that can run on a webworker. (A previous, non-webgl version library is described in more detail [here](https://creatingdata.us/techne/deep_scatterplots/); some other observable notebooks are [here](https://observablehq.com/collection/@bmschmidt/deep-scatterplots)). For a gentler introduction, check out David Mora's [Intro to deepscatter](https://observablehq.com/@davidnmora/intro-to-deepscatter) 33:T67f,viewof selected = { plot_updater const labels = plot._renderer.aes.dim("color").current.scale.domain() const el = DOM.element("div") const holder = d3.select(el) const checked = new Set() const moused = {current : undefined} let regex = new RegExp(regex_filter, "i") const filtered = labels.filter(d => d.match(regex)) holder.style("display", "flex").style("width", width * .66).style("font-style", "10px sans-serif").style("flex-wrap", "wrap") const slugs = holder.selectAll("div").data(filtered).join("div") .style("margin", "1px") .style("background-color", d => plot._renderer.aes.dim("color").current.scale(d)).text(d => d) .on("mouseover", (event, d) => { moused.current = d el.value = [moused.current, ...checked] el.dispatchEvent(new CustomEvent("input")); update_style() }) .on("mouseout", (event, d) => { moused.current = undefined el.value = checked.length ? [...checked] : labels el.dispatchEvent(new CustomEvent("input")); update_style() }) .on("click", (event, d) => { if (!checked.has(d)) {checked.add(d)} else {checked.delete(d)} el.value = checked.length ? [...checked] : labels el.dispatchEvent(new CustomEvent("input")); update_style() }) function update_style() { slugs.style("opacity", d => { if (checked.has(d)) {return 1} if (el.value[0] == d) {return 1} if (el.value.length < 1) {return .9} return .4 }) } if (checked.length) { el.value = [...checked] return el } el.value = filtered return el }34:T77a,/** * Changes XML to JSON * Modified version from here: http://davidwalsh.name/convert-xml-json * @param {string} xml XML DOM tree */ function xmlToJson(xml) { // Create the return object var obj = {}; if (xml.nodeType == 1) { // element // do attributes if (xml.attributes.length > 0) { obj["@attributes"] = {}; for (var j = 0; j < xml.attributes.length; j++) { var attribute = xml.attributes.item(j); obj["@attributes"][attribute.nodeName] = attribute.nodeValue; } } } else if (xml.nodeType == 3) { // text obj = xml.nodeValue; } // do children // If all text nodes inside, get concatenated text from them. var textNodes = [].slice.call(xml.childNodes).filter(function(node) { return node.nodeType === 3; }); if (xml.hasChildNodes() && xml.childNodes.length === textNodes.length) { obj = [].slice.call(xml.childNodes).reduce(function(text, node) { return text + node.nodeValue; }, ""); } else if (xml.hasChildNodes()) { for (var i = 0; i < xml.childNodes.length; i++) { var item = xml.childNodes.item(i); var nodeName = item.nodeName; if (typeof obj[nodeName] == "undefined") { obj[nodeName] = xmlToJson(item); } else { if (typeof obj[nodeName].push == "undefined") { var old = obj[nodeName]; obj[nodeName] = []; obj[nodeName].push(old); } obj[nodeName].push(xmlToJson(item)); } } } return obj; } /* Usage: 1. If you have an XML file URL: const response = await fetch('file_url'); const xmlString = await response.text(); var XmlNode = new DOMParser().parseFromString(xmlString, 'text/xml'); xmlToJson(XmlNode); 2. If you have an XML as string: var XmlNode = new DOMParser().parseFromString(yourXmlString, 'text/xml'); xmlToJson(XmlNode); 3. If you have the XML as a DOM Node: xmlToJson(YourXmlNode); */35:[] 30:{"id":"31af9740e0d7a929","slug":null,"title":"The Arxiv in WebGL.","thumbnail":"f496246804dc4006c2c85061e821215e365f148857229382a75891675b5616ba","access_level":"public","create_time":"$D2023-02-25T18:41:27.629Z","update_time":"$D2023-02-25T19:10:10.785Z","view_time":null,"first_public_time":"$D2023-02-25T18:41:27.629Z","has_unpublished_edits":null,"fork":{"id":"bca2dca1a864b760","slug":"arxiv","title":"The Arxiv in WebGL.","owner":{"id":"18bbeceb36c8f564","login":"bmschmidt","name":"Benjamin Schmidt"}},"stars":0,"starred":false,"owner":{"type":"team","id":"18bbeceb36c8f564","login":"bmschmidt","name":"Benjamin Schmidt","avatar_url":"https://avatars-next.observableusercontent.com/addc2adb5ec95cc72a64820e4a08488bad5d48d95a895eef85a45b136ed66792","tier":"starter_2024"},"version":890,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"33354ca265b21f01","login":"bmschmidt","name":"Benjamin Schmidt","avatar_url":"https://avatars-next.observableusercontent.com/addc2adb5ec95cc72a64820e4a08488bad5d48d95a895eef85a45b136ed66792","tier":null}],"members":[{"type":"individual","id":"33354ca265b21f01","login":"bmschmidt","name":"Benjamin Schmidt","avatar_url":"https://avatars-next.observableusercontent.com/addc2adb5ec95cc72a64820e4a08488bad5d48d95a895eef85a45b136ed66792","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":1,"collection":null,"type":"notebook","body":{"title":"The Arxiv in WebGL.","cells":[{"id":169,"value":"$32","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":435,"value":"viewof bbox = Inputs.select(new Map([\n [\"Zoom to a region\", undefined],\n [\"Natural Language Processing\", {\"x\":[-4.967787232372515,-2.401262546003603],\"y\":[-11.434250980592353,-9.509357465815668]}],\n [\"History\", {\"x\":[5.626518036921493,6.625216059591682],\"y\":[2.1329493595485243,2.881972876551167]}],\n [\"Cryptography\", {\"x\":[-8.194459371658855,-6.584608944124668],\"y\":[3.1899676117819498,4.397355432432591]}]\n]), {label: \"Zoom to a region.\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":468,"value":"viewof color_choice = {\n \n const options = new Map([[\n \"Subject\", {\n field: \"subject_1\",\n range: \"set1\",\n domain: [-2047, 2047]\n }],\n[ \"Publication Date\", {\n field: \"year\",\n range: \"viridis\",\n domain: [1995, 2020]\n }\n ]])\n \n return Inputs.radio(options, {label: \"Color showing\", value: options.get(\"Subject\")})\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":203,"value":"clack(doi)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":38,"value":"viewof alpha = Inputs.range([0, 50], {label: \"Target Opacity\", value: 25.8})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":322,"value":"viewof year = rangeSlider({title: \"year\", description: \"Filter to a range\", min: 1990, max: 2020, step: 1})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":0,"value":"div = html`

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":775,"value":"## Filter subjects by Regex","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":770,"value":"viewof regex_filter = Inputs.text({\"label\": \"Filter by regex\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":499,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":600,"value":"{\n let category_filters = selected.map((d) => `value === \"${d}\"`).join(\" || \");\n let share_of_categories = Math.log(151 / selected.length);\n if (category_filters.length == 0) {\n category_filters = \"true === false\";\n share_of_categories = 0;\n }\n console.log(share_of_categories + 1, selected.length);\n plot.plotAPI({\n point_size: point_size * (share_of_categories + 1),\n encoding: {\n filter: {\n field: \"subject_1\",\n lambda: `value => ${category_filters}`\n }\n },\n duration: 500,\n alpha: alpha * 10\n });\n return category_filters;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":735,"value":"{\n return plot.plotAPI(\n {\n point_size,\n 'encoding': {\n 'filter2': {\n field: 'year',\n op: 'within',\n a: (year[1] - year[0]) / 2,\n b: (year[1] + year[0]) / 2\n }\n }\n })\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":843,"value":"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":177,"value":"async function clack(doi) {\n const url = `https://export.arxiv.org/api/query?search_query=all:${(doi)}`\n const document = await d3.xml(url)\n const d = xmlToJson(document)['feed']['entry']\n let authors = d.author ? d.author : []\n if (!d.category) {return html`Error fetching data for ${doi}`}\n if (!authors.map) {\n authors = [authors] \n }\n \n return html`\n

${d.title} [link]

\n\n${d.published}\n${d.category ? d.category['@attributes'] ? d.category['@attributes'].term : '' : ''}\n\n

\nAuthors: ${authors.map(d => d.name).join(\", \")}\n

\n${d.summary}\n

\n `\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":264,"value":"mutable doi = \"1512.00913\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":58,"value":"# Deepscatter imports\n\nDeepscatter can be simply imported as a module and used to display a billion points in an observable notebook. Play with the API call parameters below.\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2,"value":"Deepscatter = (await import(\"https://benschmidt.org/deepscatter@2.6.0.js\"))\n .default","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":399,"value":"## Settings","pinned":true,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":50,"value":"viewof point_size = Inputs.range([0, 20], {transform: Math.sqrt, value: 5, label: \"Point size\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":66,"value":"viewof max_points = Inputs.range([100, 1e7], {transform: Math.log, value: 75000, label: \"Maximum points at once\", step: 100})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":409,"value":"md`## Filters\n\nFilters used to be slow, but work pretty well now!`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":34,"value":"plot_updater = plot.plotAPI({\n \"max_points\" : max_points, // a full cap. Not setting this can create network transfer headaches.\n alpha, // Target saturation for the full page.\n \"zoom_balance\" : 0.39, // Rate at which points increase size. https://observablehq.com/@bmschmidt/zoom-strategies-for-huge-scatterplots-with-three-js. Optional.\n \"point_size\": point_size, // Default point size before application of size scaling\n \"duration\": 2,\n \"encoding\": {\n \"x\": {\n field: \"x\",\n transform: \"literal\"\n },\n \"y\": {\n field: \"y\", \n transform: \"literal\"\n },\n \"color\": color_choice,\n \"filter\": {\n field: \"year\",\n op: \"within\",\n a: (year[1] - year[0])/2,\n b: (year[1] + year[0])/2,\n }\n }\n}).then(() => {\n //manually override the click function for observable.\n plot.click_function = function(dd, ev) {\n mutable doi = dd.axid\n }\n return plot.query\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":417,"value":"current_corners_helper = {\n const location = JSON.stringify({'zoom': {'bbox': plot._zoom.current_corners()}})\n return location\n return `plot.plotAPI(${location}).then(plot.plotAPI('zoom': undefined)))`\n} ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":674,"value":"","pinned":true,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":731,"value":"year_listener = {\n\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":405,"value":"zoom_listener = {\n plot.plotAPI({zoom: { bbox }}).then(plot.plotAPI({zoom: undefined})) \n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":679,"value":"plot.click_function","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":9,"value":"plot = {\n window.last_clicked = {}\n d3.select(\"#plot\").selectAll(\"div\").remove()\n const plot = new Deepscatter(\"#plot\", width, Math.floor(width * .75))\n await plot.plotAPI({\n \"source_url\" : 'https://benschmidt.org/arxiv',\n \"background_color\": [45, 0, 75],\n \"max_points\": 10000,\n \"alpha\": 5\n })\n return plot\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":881,"value":"function children(tile) {\n const [z, x, y] = tile.split(\"/\").map((d) => parseInt(d));\n const children = [];\n for (let i = 0; i < 4; i++) {\n children.push(`${z + 1}/${x * 2 + (i % 2)}/${y * 2 + Math.floor(i / 2)}`);\n }\n return children;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":885,"value":"children(\"2/4/4\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":327,"value":"import {rangeSlider} from '@mootari/range-slider'\n","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":850,"value":"function macro_tile(k, size = 3, parents = 1) {\n let [z, x, y] = k.split(\"/\").map((d) => parseInt(d));\n let moves = 0;\n while (!(moves > parents && z % size == 0)) {\n x = Math.floor(x / 2);\n y = Math.floor(y / 2);\n z = z - 1;\n moves++;\n }\n return `${z}/${x}/${y}`;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":852,"value":"plot._root.map((d) => d.key + \"->\" + macro_tile(d.key))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":191,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":94,"value":"d3 = require(\"d3-selection\", 'd3-fetch')","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W35","stdlib":"1","titling":"auto"}} 31:"$30:body"