31:T82b,parseBookwormData = function(json,locQuery) { // Changes the shape of the hierarchical json the API // delivers to a flat one with attribute names // which takes more space but plays more nicely with d3/javascript. var names = [] var bookworm = this if (json instanceof Array) { // const newOut = {}; json.forEach( (d, i) => { newOut[i] = d }) json = newOut names.push("search_limits") } names = names.concat(locQuery.groups).concat(locQuery.counttype); function flatten(hash,prepend) { prepend = prepend || []; var results = Object.keys(hash).map(function(key) { var newpend = prepend.concat(key) if (hash[key] instanceof Array) { return(newpend.concat(hash[key])) } else { var vals = flatten(hash[key],newpend) //is this doing anything different from return (vals)? return( vals.map(function(array) { return(array) }) ) } }) if (results[0][0] instanceof Array) { return(results.reduce(function(a,b){return(a.concat(b))})) } else { return(results) } } function toObject(names, values) { var result = {}; for (var i = 0; i < names.length; i++) { result[names[i]] = values[i];} return result; }; //run flatten initially with nothing prepended: as it recurses, that will get filled in. try { var flat = flatten(json); } catch(err) { var flat = [] } //add the labels. var results = flat.map(function(localdata){ return(toObject(names,localdata)); }) return(results) }32:T1415,function smooth(data, span, smoothingBy, quantKeys, kernel, weights) { if (span <= 0) {return data} let kernelSmoother; kernel = kernel || "sine" if (kernel == "average") { kernelSmoother = function(array) { return d3.mean(array) } } if (kernel=="sine") { kernelSmoother = function(array) { //By default, use a sine-wave shaped smoothing kernel. var length = array.length var midpoint = (length-1)/2 var weighter = function(i) { return Math.cos((midpoint-i)/(length+2)*Math.PI/2) } var totalWeights = 0; var i = 0 const weightedVals = array.map(function(d) { var weight = weighter(i); totalWeights += weight; i++; return (d * weight) }) var total = d3.sum(weightedVals)/totalWeights return total } } /*the funny thing about smoothing is that when it's multivariate, we need to create some entries to interpolate 0 counts: if Britain has a series from 1066 to 2000, and the US only from 1776 to 1860 and 1865 to 2000, we want to interpolate a whole bunch of zeroes before 1776 and 1860. But if there's a series of US grain and cattle exports but only British grain exports, there's no need to create a dummy set of British cattle exports consisting only of zeroes. Probably. (I don't know, maybe there is?) */ //Assuming zero for no data is also problematic in all sorts of ways; this needs a "step" variable. //This code accomplishes that //using D3.nest to avoid expensive filters. const allKeys = d3.keys(data[0]) var groupings = allKeys.filter(function(key) { if (quantKeys.indexOf(key)>-1) {return false} if (key == smoothingBy) {return false} return true }) var completeNesting = d3.nest() //nest by each of the groupings groupings.forEach(function(key) { completeNesting.key(function(d) { return d[key] }) }) //nest by the thing itself. completeNesting.key(function(d) { return d[smoothingBy] }) if (kernel !== "faussian") { var timeStamps = d3.set(data.map(d => d[smoothingBy])) .values() .sort(function(a,b) {return parseFloat(a)-parseFloat(b)}) var timeGroups = timeStamps .map(function(d,i) { return timeStamps.slice(d3.max([0,i-span]),d3.min([timeStamps.length, i + span + 1]) ) }) } var completeIndex = completeNesting.map(data) //One element per each interaction of variables in the groupings: // recreated by recursively descending the first levels of the completeIndex hierarchy. var recurse = function(map, labels) { var key = labels[0]; if (labels.length == 1) { return map.keys() .map(function(d) { var a = {}; a[key] = d; a['values'] = map.get(d) return a }) } else { let returnVal = [] map.keys().forEach(function(key2) { const vals = recurse(map.get(key2), labels.slice(1)) returnVal = returnVal.concat(vals.map(function(d) { d[key] = key2 return d })) }) return returnVal } } var hierarchy = recurse(completeIndex, groupings) var smoothedData = [] const blur = (data, properties) => data.map((d, i) => { const previous = (i === 0) ? i : i - 1; const next = (i === data.length - 1) ? i : i + 1; const datums = [data[previous], d, data[next]]; properties.forEach(property => { const wweights = datums.map(dd => { return dd[weights] }) const values = datums.map( (dd, ii) => { return dd[property] * wweights[ii] }) d[property] = d3.sum(values)/d3.sum(wweights) }) return d; }); hierarchy.forEach(function(hierarchyLevel) { //clone a new object off of the values for the level. var locEntries = {} hierarchyLevel['values'] .keys() .forEach(k => { locEntries[k] = hierarchyLevel['values'][k] }) if (kernel === "faussian") { const vals = JSON.parse(JSON.stringify(hierarchyLevel.values.values().map(d => d[0]))) d3.range(span).forEach( i => { locEntries = blur(vals, quantKeys.filter(d => d !== weights)) }) smoothedData.push(...locEntries) } else { timeGroups.map(function(timesToMerge, i) { var newOut = {}; d3.keys(hierarchyLevel).forEach(k => { if (k != "values") { newOut[k] = hierarchyLevel[k] } }) newOut[smoothingBy] = timeStamps[i] quantKeys.forEach(function(key) { //For each quantitative key, there's a different value to smooth. //Currently we work across them, not combining. (I think). const values = timesToMerge.map(function(d) { const val = hierarchyLevel.values.get(d) if (val==undefined) {return 0} return val[0][key] }) //Set the smoothed data equal to whatever smoothing kernel is being used newOut[key] = kernelSmoother(values) }) smoothedData.push(newOut) }) } }) return smoothedData; }33:[] 2f:{"title":"Hathi Bookworm Data with Vega-Lite","cells":[{"id":0,"value":"md`# Hathi Bookworm Data with Vega-Lite\n\nThe Bookworm API exposes a rich query language for investigating the content of the Hathi Trust.\n\nYears ago I spent implemented various charts in D3 to make Bookworm live on the web. Today, Vega-Lite is advanced enough that it's probably not wise for me to worry about things like axis ticks. Instead, we can start to fuse the declarative Bookworm grammar with the declarative Vega grammar.\n\nAll of this code is live in an observable notebook pulling straight from the Hathi Trust site. This is interesting, because it means anyone can edit it in--say--a live workshop to search for their own sources. No mucking around with jupyter!\n\nFirst we'll make a simple chart, with no smoothing and no interaction. This is something we can do with any plotting library, so it's not exciting yet.\n\n`\n\n","pinned":false,"hidden":false,"mode":"ojs"},{"id":177,"value":"md`\n\n# Basic line chart example.\n\n## Step 1. Defining a query\n\nHere I just use one of the online query examples.\n\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":99,"value":"query = new Object(\n {\n \"groups\" : [\"date_year\"],\n \"counttype\":\"WordsPerMillion\",\n \"words_collation\":\"Case_Insensitive\",\n \"database\":\"Bookworm2016\",\n \"search_limits\":[\n {\n \"word\":[\"polka\"],\n \"date_year\":{\"$gte\":1760,\"$lte\":2010},\"class__id\":[\"2\"]},\n {\n \"word\":[\"polka\"],\"date_year\":{\"$gte\":1760,\"$lte\":2010},\n \"class__id\":[\"11\"]}\n ],\n \"method\":\"data\",\n \"format\":\"json\"})","pinned":true,"hidden":false,"mode":"ojs"},{"id":179,"value":"md`### Download data from hathitrust.org\n\nDownload the data from Hathi.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":7,"value":"function bookwormFetch(query) {\n let domain = query.host || 'https://bookworm.htrc.illinois.edu'\n const url = encodeURI(JSON.stringify(query))\n // This wrapper should probably only apply in an emergency.\n if (domain.startsWith(\"http:\")) {\n domain = \"https://cors-anywhere.herokuapp.com/\" + domain\n }\n return d3.json(`${domain}/cgi-bin/dbbindings.py?query=${url}`).then(\n data => parseBookwormData(data.data, query)).then(\n data => add_bookworm_searches(data, query))\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":200,"value":"bookwormFetch(query)","pinned":false,"hidden":false,"mode":"ojs"},{"id":183,"value":"md`\n### Transformation function\n\nRaw bookworm data is not in a format that plays easily with D3; this function, from my bookwormD3 library, transforms it.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":893,"value":"function add_bookworm_searches(data, query, name = 'url') {\n data.forEach( d => {\n const q = JSON.parse(JSON.stringify(query))\n q.groups.forEach(g => {\n q['search_limits'][g] = data[g]\n //d[name] = JSON.stringify(q)\n \n // No, actually do this:\n d[name] = \"javascript:alert('\" + JSON.stringify(q.search_limits) + \"')\"\n })\n \n })\n return data\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":896,"value":"add_bookworm_searches(serverResponse, query)","pinned":false,"hidden":false,"mode":"ojs"},{"id":134,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":207,"value":"serverResponse = bookwormFetch(query)","pinned":false,"hidden":false,"mode":"ojs"},{"id":185,"value":"md`Proof of concept: a chart!`","pinned":false,"hidden":false,"mode":"ojs"},{"id":88,"value":"spec = new Object({\n width: 600,\n height: 300,\n data: {values: serverResponse},\n mark: \"line\",\n encoding: {\n x: {field: \"date_year\", type: \"quantitative\"},\n y: {field: \"WordsPerMillion\", type: \"quantitative\"},\n color: {field: \"search_limits\", type: \"nominal\"},\n href: {field: \"url\", type: \"nominal\"}\n }\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":158,"value":"spec.data","pinned":false,"hidden":false,"mode":"ojs"},{"id":15,"value":"embed(spec)","pinned":true,"hidden":false,"mode":"ojs"},{"id":291,"value":"md`# Area charts\n\nBookworm plays well with a variety of vega chart types. Here's an area chart of library contributions.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":293,"value":"md`\n\nFirst, a convenience function to fetch and parse a query. Be aware this returns a Promise, so it can't just be dropped into a data slot.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":296,"value":"md`Now to define a new query that gets the counts by library.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":230,"value":"new_query = new Object({\n groups: [\"contributing_library\", \"date_year\"],\n counttype: [\"TextCount\",\"WordCount\"],\n words_collation: \"Case_Insensitive\",\n database: \"Bookworm2016\",\n search_limits: {\"date_year\": {\"$gte\": 1800, \"$lte\": 2020}, \n // Only the top 11 libraries.\n \"contributing_library__id\": {\"$lte\": 12}\n },\n method: \"data\",\n format: \"json\"\n })","pinned":false,"hidden":false,"mode":"ojs"},{"id":963,"value":"uk_information = {\n let d = await bookwormFetch({\"groups\":[\"publication_country\",\"date_year\"],\"counttype\":[\"TextCount\",\"WordCount\"],\"words_collation\":\"Case_Insensitive\",\"database\":\"Bookworm2016\",\"search_limits\":{\"date_year\":{\"$gte\":1800,\"$lte\":2020},\"publication_country__id\":{\"$lte\":12}},\"method\":\"data\",\"format\":\"json\"})\n d = d.filter(datum => [\"USA\", \"United Kingdom\"].indexOf(datum.publication_country) > -1)\n d = (aq.from(d))\n return d.groupby(\"date_year\").derive({\"share\": `d => d.TextCount/sum(d.TextCount)`})\n const spec = new Object({\n \"width\": 700, \"height\": 400,\n \"data\": {values: d.objects()},\n \"mark\": \"line\",\n title: 'Percent in UK vs percent in USA',\n \"encoding\": {\n x: {field: \"date_year\", \"timeUnit\": \"year\", \"type\": \"temporal\",\n \"axis\": {\"domain\": false, \"format\": \"%Y\", \"tickSize\": 0}\n},\n y: {field: \"share\", type: \"quantitative\"},\n color: {field: \"publication_country\", type: \"nominal\"}\n }\n\n})\n return embed(spec)\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":952,"value":"countrySpec = {\n const d = await bookwormFetch({\"groups\":[\"publication_country\",\"date_year\"],\"counttype\":[\"TextCount\",\"WordCount\"],\"words_collation\":\"Case_Insensitive\",\"database\":\"Bookworm2016\",\"search_limits\":{\"date_year\":{\"$gte\":1800,\"$lte\":2020},\"publication_country__id\":{\"$lte\":12}},\"method\":\"data\",\"format\":\"json\"})\n const spec = new Object({\n \"width\": 700, \"height\": 400,\n \"data\": {values: d},\n \"mark\": \"area\",\n \"parse\": {\"date_year\": \"utc:'%Y'\"},\n \"encoding\": {\n x: {field: \"date_year\", \"timeUnit\": \"year\", \"type\": \"temporal\",\n \"axis\": {\"domain\": false, \"format\": \"%Y\", \"tickSize\": 0}\n},\n y: {field: \"TextCount\", type: \"quantitative\",\n \"stack\": \"center\"},\n color: {field: \"publication_country\", type: \"nominal\"}\n }\n\n})\n return embed(spec)\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":943,"value":"JSON.stringify(new_query)","pinned":true,"hidden":false,"mode":"ojs"},{"id":194,"value":"\nlibrary_data = bookwormFetch(new_query)\n","pinned":false,"hidden":false,"mode":"ojs"},{"id":251,"value":"areaspec = new Object({\n \"width\": 700, \"height\": 400,\n \"data\": {values: library_data},\n \"mark\": \"area\",\n \"parse\": {\"date_year\": \"utc:'%Y'\"},\n \"encoding\": {\n x: {field: \"date_year\", \"timeUnit\": \"year\", \"type\": \"temporal\",\n \"axis\": {\"domain\": false, \"format\": \"%Y\", \"tickSize\": 0}\n},\n y: {field: \"TextCount\", type: \"quantitative\",\n \"stack\": \"center\"},\n color: {field: \"contributing_library\", type: \"nominal\"}\n }\n\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":255,"value":"embed(areaspec)","pinned":false,"hidden":false,"mode":"ojs"},{"id":329,"value":"md`## Book lengths by library\n\nA funny effect: books get shorter after 1922`","pinned":false,"hidden":false,"mode":"ojs"},{"id":305,"value":"bookwormFetch(new_query)\n .then(data => {\n return {\n \"width\": 700, \"height\": 700,\n \"data\": {values: data.map(d => {d.length = d.WordCount/d.TextCount; return d})},\n \"mark\": {type: \"point\", \"tooltip\": {\"content\": \"data\"}},\n \"encoding\": {\n y: {field: \"length\", type: \"quantitative\", scale: {\"domain\": [0, 500000], clamp: true}},\n x: {field: \"date_year\", type: \"temporal\", \"timeUnit\": \"year\"},\n size: {field: \"TextCount\", scale: {\"type\": \"sqrt\", \"range\": [.01, 100]}},\n color: {field: \"contributing_library\", type: \"nominal\"}\n }\n }\n}).then( spec => embed(spec))","pinned":false,"hidden":false,"mode":"ojs"},{"id":378,"value":"md`# By LC class\n\nIs this a class based effect? We can modify the query and find out. Turns out not entirely--some classes are longer (especially general works, law, and--since 1970--bibliography and library science). But most seem to fall rapidly in length between 1910 and 1920.\n\nLanguage and literature is the shortest. The fall there is especially strong between 1914 and 19120, which makes me think wartime paper rationing may be involved.\n\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":370,"value":"new_query","pinned":false,"hidden":false,"mode":"ojs"},{"id":369,"value":"{\n // Don't mutate the original query.\n const q = JSON.parse(JSON.stringify(new_query));\n q.groups[0] = 'class';\n return bookwormFetch(q)\n .then(data => {\n return {\n \"width\": 640, \"height\": 700,\n \"data\": {values: data.map(d => {d.length = d.WordCount/d.TextCount; return d})},\n \"mark\": {type: \"point\", \"tooltip\": {\"content\": \"data\"}},\n \"selection\": {\n \"grid\": {\n \"type\": \"interval\", \"bind\": \"scales\"\n }\n },\n \"encoding\": {\n y: {field: \"length\", type: \"quantitative\", scale: {\"domain\": [0, 500000], clamp: true}},\n x: {field: \"date_year\", type: \"temporal\", \"timeUnit\": \"year\"},\n size: {field: \"TextCount\", scale: {\"type\": \"sqrt\", \"range\": [.01, 100]}},\n color: {field: q.groups[0], type: \"nominal\"}\n }\n }\n}).then( spec => embed(spec))\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":564,"value":"md`Reimplementing the kernel smoothing from the D3 js makes this a lot clearer. We can say declaratively that\n\n### Across multiple classes, book lengths increase to 1840, decrease to 1940, and increase after 1960.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":397,"value":"{\n // Don't mutate the original query.\n const q = JSON.parse(JSON.stringify(new_query));\n q.groups[0] = 'class';\n return bookwormFetch(q)\n .then(data => {\n let smoothed = data\n smoothed = smoothed.map(d => {d.length = d.WordCount/d.TextCount; return d})\n smoothed = smooth(smoothed, 5, \"date_year\", [\"WordCount\", \"TextCount\",\"length\"], \"sine\")\n return {\n \"width\": 800, \"height\": 600,\n \n \"data\": {values: smoothed},\n \"mark\": {type: \"point\", \"tooltip\": {\"content\": \"data\"}},\n \n \"encoding\": {\n y: {field: \"length\", type: \"quantitative\", scale: {\"domain\": [0, 500000], clamp: true}},\n x: {field: \"date_year\", type: \"temporal\", \"timeUnit\": \"year\"},\n size: {field: \"TextCount\", scale: {\"type\": \"sqrt\", \"range\": [.01, 100]}},\n color: {field: q.groups[0], type: \"nominal\"}\n }\n }\n}).then( spec => embed(spec))}\n","pinned":false,"hidden":false,"mode":"ojs"},{"id":583,"value":"md`## Smoothing function experiments.\n\nWhy not get Observable-fancy: we can fiddle with the smoothing parameter here.\n\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":576,"value":"md`Use this slider to change the smoothing kernel span from **${smoothing}**.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":574,"value":"viewof smoothing = html``","pinned":false,"hidden":false,"mode":"ojs"},{"id":665,"value":"viewof smoothType = html``","pinned":false,"hidden":false,"mode":"ojs"},{"id":727,"value":"{\n \n const query = {\n $schema: 'https://vega.github.io/schema/vega-lite/v3.json',\n groups: [\"class\", \"date_year\"],\n counttype: [\"WordsPerMillion\",\"WordCount\"],\n words_collation: \"Case_Insensitive\",\n database: \"Bookworm2016\",\n search_limits: {\"date_year\": {\"$gte\": 1900, \"$lte\": 2010}, \n // Only the top 11 libraries.\n \"class__id\": {\"$lte\": 20},\n \"word\":[...word3.split(/, ?/)]\n },\n method: \"data\",\n format: \"json\"\n }\n \n return bookwormFetch(query).then(data => {\n return {\n \"width\": 550, \"height\": 400,\n \"data\": {values: data},\n \"mark\": {type: \"rect\"},\n \"selection\": {\n \"grid\": {\n \"type\": \"interval\", \"bind\": \"scales\"\n }\n },\n \"encoding\": {\n y: {field: query.groups[0], type: \"nominal\"},\n x: {field: query.groups[1], type: \"ordinal\", \"timeUnit\": \"year\"},\n color: {field: query.counttype[0], type: \"quantitative\", scale: {type: \"magma\"}}\n }\n }\n}).then(spec => embed(spec))\n\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":345,"value":"embed_ = require(\"vega-embed@latest\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":782,"value":"embed = function(spec) {\n if (spec.encoding.tooltip === undefined) {\n spec.encoding.tooltip = d3.keys(spec.encoding).map(k => spec.encoding[k])\n }\n if (spec.selection == undefined) {\n spec.selection = {\n \"grid\": {\n \"type\": \"interval\", \"bind\": \"scales\"\n }\n }\n }\n \n return embed_(spec)\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":385,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":3,"value":"d3 = require(\"d3-fetch@1\", \"d3@v5\")","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W33","stdlib":"1","titling":"auto"}