31:T6b4,dataModel = md` # Javascript bindings to the Hathi Features data This notebook exists to document and to be imported into my other notebooks for working with Hathi Trust data. If you want to do some heavy research on those and you know how to code, you should: I recommend the HTRC's [extended feature reader](https://github.com/htrc/htrc-feature-reader) in python or my [Hathidy](https://github.com/HumanitiesDataAnalysis/hathidy) package for R. If do or don't code and you want to see aggregate trends, use the [HTRC Bookworm](https://bookworm.htrc.illinois.edu/develop/). The programming model here for working with Hathi features are three types of objects that I may break out into a node module at some point. The central class is the Volume. It's instantiated with a hathi trust ID. Since data is loaded against the Hathi trust, it can only be used in asynchronous settings; a typical use is new Volume(my_id).load().then(vol => {vol.whatever}). Since Observable allows cells to be asychronous *or* synchronous, a lot of that messiness isn't visible here. Chunking, counts, and other basic operations are done on the Volume object. Most calls are on the Library class; it allows for comparing multiple volumes against each other.. Counting operations--like TF-IDF--are defined as being features of 'Library' class. (My current heuristic is that counts from a 'volume' instance are quick and dirty associational objects; while counts form a library are well structured rows of key-value JSON. (I could call it it 'corpus' instead of library, but I want to avoid the connotations of that term.) Finally, there's a Page class; it's currently used only as a wrapper around a basic word count function. `32:T152c,class Volume { // A class to describe a Hathi Trust Volume. // Most methods are broken until the 'fetch' method has completed loading; // to build this into code, use Volume('yourid').load(), which returns the completed // volume inside a promise. constructor(id) { this.id = id; this.chunk_cache = new Map(); this.fetch(); this.options = { section: ["body"], what: "word", chunk: "page", size: 10000 }; } promise_from_htrc() { // First check a localStorage cache. let cache; if (window && window.indexedDB && false) { // cache = idb.get(this.id) //return Promise.resolve(JSON.parse(window.localStorage.getItem(this.id))) } else { cache = Promise.resolve(undefined); } return cache.then((cached_version) => { if (cached_version !== undefined) { console.log("Returning " + this.id + " from local cache"); return cached_version; } console.log("fetching " + this.id + " remotely"); return d3.json( `https://corsproxy.io/?https://data.analytics.hathitrust.org/htrc-ef-access/get?action=download-ids&id=${this.id}&output=json` ); // .then(data => { // if (window) { /* if (window && window.indexedDB && idb) { idb.set(this.id, data) } } */ // return data // }) }); } fetch() { // Fetch returns a promise to the data, and populates the cache at this.data. // 'load' is a more helpful wrapper that returns the whole `Volume` object itself wrapped in a promise. // Once it has executed, you can access the data element directly, but it will often be necessary // to wrap the fetch method in some promise resolution to work with it. if (this.promise) { return this.promise; } this.promise = this.promise_from_htrc().then((d) => { this.data = d; return d; }); return this.promise; // Defaults; likely to be overriden. this.options = { size: 10000, what: "words", doc: "page", ntiles: 24 }; } async load() { await this.fetch(); return this; } total_word_count(options) { // This is also in the metadata. const loc_counts = this.word_counts(options); const tot = Array.from(loc_counts.values()).reduce( (left, right) => +left + right, 0 ); return tot; } repr() { // Return an HTML representation of the book suitable for linking back to Hathi. const { title, pubDate, names, imprint, volumeIdentifier } = this.data.metadata; return `${names.join( "/" )}. ${title}. ${imprint}. ${this.id}`; } count(options) { if (options.what === undefined) { options.what = "word"; } if (options.doc === undefined) { options.doc = "volume"; } const { doc } = options; if (doc == "chunk") { return this.chunk_counts(options); } else if (doc == "page") { return this.page_counts(options); } else if (doc == "ntile") { const total = this.total_word_count(options); const ntile_size = Math.floor(1 + (total + 1) / options.ntiles); // It's a little sketchy to mutate this array in-place. // Will confuse people toggling back and forth. const counts = this.chunk_counts(options, ntile_size); return counts; } else if (doc == "volume") { return [this.word_counts(options)]; } return "Choose an allowed count type"; } chunk_counts(options, size) { // Divide into continuous chunks of size 'size''. // If 'type' == 'words', will divide into n word chunks // If 'type' == 'pages', will divide into n/500-page chunks. // This sets up a cache, so it would possibly to build // a memory leak. if (size == undefined && options.size) { size = options.size; } const { chunk_cache } = this; const { what } = options; if (chunk_cache.get(`${what}-${size}`)) { return chunk_cache.get(`${what}-${size}`); } const pages = this.page_counts(options); let lengths = []; let size2 = size; //if (what == "words") { lengths = pages.map((d) => d3.sum(Array.from(d.values()))); /*} else { lengths = pages.map(d => 1) }*/ const array_reducer = (a, x, i) => [...a, a.length > 0 ? x + a[i - 1] : x]; const cumulative_lengths = lengths.reduce(array_reducer, []); const chunk_lab = cumulative_lengths.map((n) => Math.floor(n / size2)); const max_chunk = d3.max(chunk_lab); const counts = d3.range(max_chunk + 1).map((d) => new Map()); pages.forEach((p, i) => { merge_counts(counts[chunk_lab[i]], p); }); chunk_cache.set(`${what}-${size}`, counts); return counts; } word_counts(options) { const pages = this.page_counts(options); return pages.reduce(merge_counts, new Map()); } page_counts(options) { const { what } = options; // Default to counting only the section. options.sections = options.sections || ["body"]; if (!this.data) { throw "Data must be (asynchronously) loaded; you may need to wrap your code after Volume.fetch().then(data=>{}"; } return this.data.features.pages.map((d) => new Page(d).counts(options)); } }33:T449,class Page { // A single page in an EF volume. constructor(data, volume) { this.data = data this.volume = volume } counts(options) { const { what, sections, regex } = options const page = this.data const counts = new Map() sections.forEach(section => { if (page[section]===undefined) {{}} Object.entries(page[section].tokenPosCount).forEach((o) => { const [word, v] = o Object.entries(v).forEach((o2) => { const [pos, count] = o2; if (regex && !word.match(regex)) { return } if (options.pos && options.pos != pos) { return } let key = word if (what == 'pos') { key = pos } else if (what == 'word&pos') { key = `${word}%&%${pos}` } else if (what == 'lowercase') { key = word.toLowerCase() } counts.set(key, (counts.get(key)|| 0) + parseInt(count)) }) }) }) return counts } }34:T1534, class Library { constructor(ids = []) { this.m = new Map() for (const k of ids) { this.m.set(k, new Volume(k)) } } /* OPERATORS: apply to chunks and update them */ word_counts(options) { const accessor = (options.what == "word" || options.what == 'lowercase') ? d => d.word : d => '' + d.word + d.pos const total_counts = d3.rollup(this.chunks(options), vals => d3.sum(vals.map(v => v.count)), accessor) return total_counts } top_words(n, options = {'what': 'word'}) { const counts = this.word_counts(options) return ([...counts.entries()]).sort((a, b) => b[1] - a[1]).map(d => d[0]).slice(0, n) } haplax(chunks, options) { // Tag each chunk by whether this is the only occurrence in the corpus. const accessor = options.what == "word" ? d => d.word : d => '' + d.word + d.pos const total_counts = d3.rollup(chunks, vals => d3.sum(vals.map(v => v.count)), accessor) chunks.forEach(chunk => { // Is the count here the total count? If so, this is a corpus haplax. chunk.haplax = total_counts.get(accessor(chunk)) == chunk.count }) return chunks } dunning(chunks, options) { // Does not check for non-occurring words in each chunk. const lib = this const documents = d3.group(chunks, d => d.index) const wordCounts = d3.rollup(chunks, d => d3.sum(d.map(e => e.count)), d => d.word) const total_words = d3.sum(wordCounts.values()) const output = [] for (const [doc_index, elements] of documents) { const c = d3.sum(elements.map(d => d.count)) const d = total_words - c elements.forEach(element => { // Formula from http://wordhoard.northwestern.edu/userman/analysis-comparewords.html // How many times is this word in this chunk? const { word, count } = element const a = count // How many times is the word *out* of this chunk? const b = d3.max([1, wordCounts.get(word) - a]) const E1 = c*(a+b)/total_words const E2 = d*(a+b)/total_words // Bind the calculations to the object in-place // PMI including itself in the corpus. element.PMI = Math.log(a/E1) // PMI compared to external corpus. element.PMI_2 = Math.log((a/c)/(d3.max([1, b])/d)) element.dunning = 2*((a*Math.log(a/E1)) + (b*Math.log(b/E2))) }) } return chunks//.sort((a, b) => b.PMI*b.dunning - a.PMI*a.dunning).filter(p => p.PMI > 0) } tfidf(chunks, options) { const documents = d3.group(chunks, d => d.index) const total_words = d3.sum(chunks.map(chunk => chunk.count)) const n_documents = Array.from(documents).length; const idf = d3.rollup(chunks, worddata => Math.log(n_documents/worddata.length), d => d.word) for (const elements of documents.values()) { const doc_wordcount = d3.sum(elements.map(d => d.count)) elements.forEach(element => { const { count, word } = element; const tf = count/doc_wordcount element.tf = tf element.tfidf = tf * idf.get(word) }) } } PMI(chunks, options) { // PMI and dunning are assigned at the same time. return this.dunning(chunks, options) } /* Basic strategies: ccreate chunks. This is the primary method. */ chunks(options, additional_metrics = []) { console.log(options) const output = [] let total_index = 0 for (let [htid, volume] of this.m.entries()) { let chunk_num = 0; for (const chunk of volume.count(options)) { total_index ++; chunk_num += 1; for (const [key, count] of chunk.entries()) { let word = undefined let pos = undefined if (options.what == "word&pos") { [word, pos] = key.split("%&%") } else if (options.what == "word" || options.what == 'lowercase') { word = key } else if (options.what == "pos") { pos = key } output.push({ 'word': word, 'pos': pos, 'count': count, 'vol_index': chunk_num, 'index': total_index, 'htid': htid }) } } } additional_metrics.forEach(metric => this[metric](output, options)) return output } get(id) { const { m } = this; if (m.get(id) !== undefined) { return m.get(id) } else { const v = new Volume(id) m.set(id, v) return v } } fetch_all(ids = []) { // asynchronously ensure that all books are loaded, and then return // the library itself. this.update(ids) const volumes = Array .from(this.m.values()) .map(book => book.fetch()) return Promise.all(volumes) .then(dummy => this) } update(ids) { // Quickly update the internal library with every element. ids.forEach(id => this.get(id)) return this } repr() { const books = Array.from(this.m.values()) return Promise.all(books.map(book => book.fetch())) .then( (data) => { const reprs = books.map(book => book.repr()) let output = "

Books plotted

${e}

" return output }) } }35:[] 2f:{"title":"Javascript bindings to the Hathi Features data","cells":[{"id":0,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":734,"value":"class Corpus {\n constructor() {\n this.volumes = new Map()\n }\n \n *add_ids(ids) {\n \n }\n \n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":16,"value":"lib = new Library().fetch_all([\"mdp.39076002651854\", \"pst.000061166424\"])","pinned":true,"hidden":false,"mode":"ojs"},{"id":354,"value":"html`

Top tokens

${lib.chunks({\"doc\": \"volume\", \"what\": \"word\"}).sort((a, b) => b.count - a.count).slice(0, 10).map(d => `${d.word} ${d.count}
`).join('')}`","pinned":true,"hidden":false,"mode":"ojs"},{"id":529,"value":"data = fetch(\n `https://corsproxy.io/?${encodeURIComponent(https://data.analytics.hathitrust.org/features-2020.03/${id_to_stubbytree(\n htid\n )}}`\n)\n .then(function (response) {\n return response.arrayBuffer();\n })\n .then((data) => bz2.decompress(new Uint8Array(data)))\n .then((array) => JSON.parse(new TextDecoder().decode(array)))","pinned":true,"hidden":false,"mode":"ojs"},{"id":555,"value":"unzip = require('compressjs@1.0.3/main.js').catch(() => window[\"define\"])","pinned":true,"hidden":false,"mode":"ojs"},{"id":723,"value":"class Volume2 {\n constructor(htid) {\n this.htid = htid;\n this.download();\n }\n\n download() {\n if (this._json) {\n return Promise.resolve(this._json);\n }\n\n const data = fetch(\n `https://corsproxy.io/?https://data.analytics.hathitrust.org/features-2020.03/${id_to_stubbytree(\n htid\n )}`\n )\n .then(function (response) {\n return response.arrayBuffer();\n })\n .then((data) => bz2.decompress(new Uint8Array(data)))\n .then((array) => {\n const json = JSON.parse(new TextDecoder().decode(array));\n this._json = json;\n return json;\n });\n }\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":718,"value":"bz2 = require('bz2@1.0.1/index.js').catch(() => window[\"bz2\"])","pinned":true,"hidden":false,"mode":"ojs"},{"id":531,"value":"htid = \"mdp.39076002651854\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":535,"value":"id_to_stubbytree(htid)","pinned":true,"hidden":false,"mode":"ojs"},{"id":740,"value":"g = fetch(\n `https://corsproxy.io/?https://data.analytics.hathitrust.org/features-2020.03/${id_to_stubbytree(\n htid\n )}`\n)","pinned":true,"hidden":false,"mode":"ojs"},{"id":534,"value":"id_to_stubbytree = function(htid) {\n const htid_clean = htid.replace(\"/\", \"+\").replace(\":\", \"=\")\n const [libid, volid] = htid.split('.')\n const volid_clean = volid.replace(\"/\", \"+\").replace(\":\", \"=\")\n\n const filename = volid_clean\n return [libid, volid_clean.split(\"\").filter((d, i) => i % 3 == 0).join(\"\"), htid_clean + \".json.bz2\"].join(\"/\")\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2,"value":"$32","pinned":true,"hidden":false,"mode":"ojs"},{"id":6,"value":"$33","pinned":true,"hidden":false,"mode":"ojs"},{"id":499,"value":"lib.top_words(100, {what: 'lowercase'})","pinned":true,"hidden":false,"mode":"ojs"},{"id":43,"value":"$34","pinned":true,"hidden":false,"mode":"ojs"},{"id":12,"value":"function merge_counts(a, b) {\n // Merge two counts dicts together into one.\n // Modifies the left-hand set in-place; beware of side effects.\n for (const [k, v] of b.entries()) {\n a.set(k, (a.get(k) || 0) + b.get(k))\n }\n return a\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":387,"value":"md`# Tests\n\nHere are some unit tests.\n\n`","pinned":true,"hidden":false,"mode":"ojs"},{"id":443,"value":"lib.chunks({'doc': 'ntile', 'ntiles': 100, what: 'word'}).length > lib.chunks({'doc': 'ntile', 'ntiles': 2, what: 'word'}).length","pinned":true,"hidden":false,"mode":"ojs"},{"id":452,"value":"lib.chunks({'doc': 'chunk', 'size': 100000, 'what': 'page'})","pinned":true,"hidden":false,"mode":"ojs"},{"id":390,"value":"// Are there 100 percentiles?\npercentile_test = d3.max(lib.chunks({'doc': 'ntile', 'ntiles': 100, what: 'word'}).map(d => d.vol_index)) == 100","pinned":true,"hidden":false,"mode":"ojs"},{"id":477,"value":"// Does it run Dunning scores successfully when asked?\ndunning_exists = lib.chunks({'doc': 'ntile', 'ntiles': 20}, [\"dunning\", \"tfidf\"])[0].dunning != undefined","pinned":true,"hidden":false,"mode":"ojs"},{"id":402,"value":"v1 = Array.from(lib.m.values())[0]","pinned":true,"hidden":false,"mode":"ojs"},{"id":405,"value":"word_counts_over_100 = v1.total_word_count({'what': \"word\", 'doc': \"page\"}) > 100","pinned":true,"hidden":false,"mode":"ojs"},{"id":226,"value":"md`# Outside imports`","pinned":true,"hidden":false,"mode":"ojs"},{"id":120,"value":"cola = require(\"webcola@3/WebCola/cola.min.js\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":8,"value":"d3 = require('d3', 'd3-fetch', 'd3-array')","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W35","stdlib":"1","titling":"auto"}