2a:null 2b:["$","title",null,{"children":"Shakespair | Chris Wallace | Observable"}] 30:T617,md`This is a character-level language generator, powered by a recurrent neural network (RNN) written in Python using the [Keras](https://keras.io/) framework. The network was trained offline on the text of the plays of William Shakespeare, where it created a character level representation of that language. The model is predicting live in the browser (no calls to an external server) by virtue of [Keras.js](https://github.com/transcranial/keras-js). For time reasons (and on AWS GPU instances, time is money), the network was not quite as big as it could have been, but I hope you find the results entertaining in any case. If this is the first time you've encountered a character level RNN and you're interested, I recommend reading these articles. In their context, nothing in this Observable notebook is very novel: - [The Unreasonable Effectiveness of Recurrent Neural Networks](http://karpathy.github.io/2015/05/21/rnn-effectiveness/) - [Adventures in Narrated Reality](https://medium.com/artists-and-machine-intelligence/adventures-in-narrated-reality-6516ff395ba3) - [Writing with the Machine](https://www.robinsloan.com/notes/writing-with-the-machine/) One of the tradeoffs of offloading prediction to the browser is that a lot of code gets repeated on the front end. There's a character RNN Keras [demo](https://github.com/keras-team/keras/blob/master/examples/lstm_text_generation.py) whose approach I followed closely in both front and backend implementation. You can find the accompanying repo here: https://github.com/cjwallace/shakespair/ `31:[] 2e:{"id":"54feecb8ef138aed","slug":"shakespair","title":"Shakespair","thumbnail":"0de06006f2818ae9727b4391de970171ba1f86097118fc1f4af02f21476dc8e3","access_level":"public","create_time":"$D2018-02-05T14:10:16.852Z","update_time":"$D2018-06-25T15:55:33.709Z","view_time":null,"first_public_time":"$D2018-03-21T10:05:03.497Z","has_unpublished_edits":null,"fork":false,"stars":3,"starred":false,"owner":{"type":"team","id":"0ac9abcf38e64f7b","login":"cjwallace","name":"Chris Wallace","avatar_url":"https://avatars-next.observableusercontent.com/ddff0ab0c8f4670e38fe8871e77257cde4558310a5516931b79aa56e27b59005","tier":"starter_2024"},"version":1138,"publish_version":1138,"role":null,"authored":false,"authors":[{"type":"individual","id":"b6d36fa8c41d8f92","login":"cjwallace","name":"Chris Wallace","avatar_url":"https://avatars-next.observableusercontent.com/ddff0ab0c8f4670e38fe8871e77257cde4558310a5516931b79aa56e27b59005","tier":null}],"members":[{"type":"individual","id":"b6d36fa8c41d8f92","login":"cjwallace","name":"Chris Wallace","avatar_url":"https://avatars-next.observableusercontent.com/ddff0ab0c8f4670e38fe8871e77257cde4558310a5516931b79aa56e27b59005","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":4,"collection":{"id":"a71d4c214fd19731","slug":"ml","title":"ml","owner":{"id":"0ac9abcf38e64f7b","login":"cjwallace","name":"Chris Wallace"}},"type":"notebook","body":{"titling":"auto","title":"Shakespair","cells":[{"id":0,"value":"md`# Shakespair\n### Using Keras.js to pair with Shakespeare!`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1108,"value":"md`_This notebook uses an asynchronous generator. Asynchronous iteration is very cool but alas, as of this writing, works only in the very latest browsers. See [Introduction to Asynchronous Iteration](https://beta.observablehq.com/@mbostock/introduction-to-asynchronous-iteration) for more details._`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":964,"value":"md`Begin typing a sentence here \n...`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1033,"value":"viewof text = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1035,"value":"md`... and Shakespeare completes it:\n

${out}

\n
`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1077,"value":"md`And here you can control the diversity of the output. A higher \"temperature\" means more diverse (and nonsensical!) text, a lower temperature means less diverse, potentially repetetive text. It's currenly set to ${temperature}.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1079,"value":"viewof temperature = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1103,"value":"md`### What's going on?`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1045,"value":"$30","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":114,"value":"md`#### Supporting code`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":958,"value":"md`We load a pretrained model from GitHub. The training procedure for the model is available in the repository linked above.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":443,"value":"model = await new keras.Model({\n \tfilepath: \"https://raw.githubusercontent.com/cjwallace/shakespair/master/data/model.bin\",\n \tgpu: true\n })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1056,"value":"md`Define some useful things - the allowed character set, \"stop characters\" that terminate sentences and the number of previous characters to consider (which is also the number of steps in the neural network). These are coupled to the backend implementation.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":40,"value":"characterSet = [' ', '!', '$', \"'\", ',', '-', '.', ':', '?', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":92,"value":"stopChars = ['.','?','!']","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":130,"value":"// length of input sentences\nnChar = 40","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1058,"value":"md`Strictly, our network requires 40 characters to predict, but we trained with some padding \\`$\\` characters so it will work with shorter sentences in user land.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":144,"value":"padInput = (input) => {\n if (input.length < nChar) {\n return '$'.repeat(nChar - input.length) + input}\n else {\n return input.slice(-nChar)\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1072,"value":"md`Helper function for sampling characters after the network has predicted their probabilities (this comes right out of the Keras LSTM example linked above). The temperature controls the diversity of samples - higher temperature means more diverse output. Try changing the value with the slider above.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":520,"value":"function sampleMultinomial(predictions) {\n var p = m.matrix(predictions)\n \t p = m.divide(m.log(p), temperature)\n \t p = m.divide(m.exp(p), m.sum(m.exp(p)))\n // sample random index, weighted by p\n var index = m.pickRandom(m.range(0,p.toArray().length),p)\n return index\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1094,"value":"md`Here is the main body of our prediction function. We have to do some matrix gymnastics, but it's not too bad.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":126,"value":"// using math.js for tensor manipulation\nfunction nextChar (input) {\n var seed = padInput(input);\n var xTest = m.zeros(1, nChar, characterSet.length),\n seedArray = seed.split('');\n // populate 3-dim array with one-hot encoded representation of input\n for (var i = 0; i < seed.length; i++) {\n\txTest.subset(m.index(0, i, characterSet.indexOf(seedArray[i])),1);\n }\n \n // Keras.js requires Float32Arrays, so converting from math.js matrix\n var prediction = model\n .ready()\n .then(() => {\n var flatData = m.flatten(xTest).toArray()\n\t const inputData = {\n input: new Float32Array(flatData)\n }\n return model.predict(inputData)\n }).then(outputData => {\n\t var out = Array.from(outputData.output),\n predIndex = sampleMultinomial(out),\n \t \t predChar = characterSet[predIndex];\n\t return predChar\n })\n \n // We have an array of probabilities to sample from\n return prediction\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1097,"value":"md`Finally, the output.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":896,"value":"out = {\n var x = text\n // restrict input set to characters the model knows how to handle\n x = x.replace(/[^-!?.,': a-zA-Z0-9]/g, '')\n var lst = x.slice(-1)\n \n // If the user leaves a stop character in the last position,\n // add a space so it does not short-circuit prediction\n if (stopChars.indexOf(lst) >= 0) {\n \tx = x.concat(\" \")\n lst = x.slice(-1)\n }\n // complete the sentence - generate until a stop character occurs\n while (stopChars.indexOf(lst) < 0) {\n // lowercase all input\n x = x.toLowerCase()\n \n // predict the next character\n \tvar nxt = await nextChar(x)\n\tx = x.concat(nxt)\n lst = x.slice(-1)\n \n // uppercase first character of sentence\n x = x[0].toUpperCase() + x.slice(1)\n yield x\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":112,"value":"md`#### Imports`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":99,"value":"m = require(\"https://unpkg.com/mathjs@4.0.0/dist/math.min.js\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":70,"value":"_ = require('underscore')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":6,"value":"keras = require('keras-js')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W31","stdlib":"1"}} 2f:"$2e:body"