2e:null 2f:["$","title",null,{"children":"Movielens recommendation using Tensorflow.js | Cornel Stefanache | Observable"}] 34:T844,data = { const movieProfile = {}; const userProfile = {}; const trainingData = {xs: [], ys: []}; const moviesCount = Object.keys(rawData.movies).length; const increment = 1 / moviesCount; for (let movie of Object.values(rawData.movies)) { const tagsArr = []; const { id, title } = movie; rawData.tags.forEach(tag => { tagsArr.push(movie.tags.indexOf(tag) !== -1 ? 1 : 0); }); movieProfile[movie.id] = { id, title, profile: tagsArr }; } for (let rating of Object.values(rawData.ratingsData)) { const { user: userIdx, movie: movieIdx, rating: ratingStr } = rating; const ratingVal = parseFloat(ratingStr); const ratingNormalized = ratingVal / 5; rating.rating = ratingVal; rating.ratingNormalized = ratingNormalized; let user = userProfile[userIdx]; if (!user) { user = { stats: [ 1, 0 ], tagsData: rawData.tags.map( () => 0 ), ratingData: d3.range(10).map( () => 0 ) } userProfile[userIdx] = user; } if (user.stats[0] > ratingNormalized) user.stats[0] = ratingNormalized; if (user.stats[1] < ratingNormalized) user.stats[1] = ratingNormalized; const movie = rawData.movies[movieIdx]; if (movie) { const { tags } = movie; tags.forEach( tag => { user.tagsData[rawData.tags.indexOf(tag)] += increment; }); user.ratingData[ Math.floor(ratingVal * 2) - 1 ] += increment; } } for (let rating of Object.values(rawData.ratingsData)) { const { user: userIdx, movie: movieIdx, ratingNormalized } = rating; const user = userProfile[userIdx]; const movie = movieProfile[movieIdx]; if (movie) { const { stats, tagsData, ratingData } = user; trainingData.xs.push([].concat(stats).concat(tagsData).concat(ratingData).concat(movie.profile)); trainingData.ys.push(ratingNormalized) } } return { movieProfile, userProfile, trainingData, features: trainingData.xs[0].length, trainedModel: false, moviesCount: Object.keys(movieProfile).length } }35:T12f9,renderPredictor = async () => { const result = document.createElement('div'); let profile, oldResults; const selectView = document.createElement('div'); const graphView = document.createElement('div'); result.innerHTML = "

Select user profile:

"; const userSelect = document.createElement('select'); userSelect.innerHTML = Object.keys(data.userProfile).map(id => ``).join(""); userSelect.addEventListener('change', () => { selectUser(userSelect.value); }); const tags = checkbox(rawData.tags); const tagsSet = new Set(); const searchTags = {}; selectView.innerHTML = `

Select at least one tag:

`; selectView.querySelector('#tagsview').appendChild(tags); const leftView = selectView.querySelector('#left'); const rightView = selectView.querySelector('#right'); const predict = async () => { userSelect.disabled = true; result.querySelectorAll('input').forEach( input => input.disabled = true); leftView.innerHTML = 'Processing...'; let results = []; let index = 0; for (let movie of Object.values(rawData.movies)) { if (tagsSet.size === 0 || movie.tags.filter(tag => tagsSet.has(tag)).length > 0) { const { stats, tagsData, ratingData } = profile; const movieProfile = data.movieProfile[movie.id].profile; const input = [].concat(stats).concat(tagsData).concat(ratingData).concat(movieProfile); const result = await model.predict(tf.tensor([input])).data(); results.push([movie.title, result[0]]); results = results.sort( (a, b) => { return b[1] - a[1]; }).splice(0, 30); leftView.innerHTML = `

Evaluating movie: ${index++} (Total: ${data.moviesCount})

`; results.forEach( (left, index) => { leftView.innerHTML += `

${index+1}. ${left[0]}

` }) } }; leftView.innerHTML = rightView.innerHTML = ''; results.forEach( (left, index) => { let right = ['']; if (oldResults && oldResults[index]) { right = oldResults[index]; } if (left[0] !== right[0]) { leftView.innerHTML += `

${index}. ${left[0]}

`; if (right[0] !== '') rightView.innerHTML += `

${index}. ${right[0]}

`; } else { leftView.innerHTML += `

${index+1}. ${left[0]}

`; if (right[0] !== '') rightView.innerHTML += `

${index+1}. ${right[0]}

`; } }); userSelect.disabled = false; result.querySelectorAll('input').forEach( input => input.disabled = false); oldResults = results; } tags.addEventListener('change', async evt => { const { checked, value } = evt.target; if (checked) { tagsSet.add(value); } else { tagsSet.delete(value); } }); const selectUser = async (value) => { profile = data.userProfile[value]; graphView.innerHTML = `

`; const rating = await vegalite({ height: 100, data: {values: profile.ratingData.reduce((memo, data, index) => { return memo.concat({index: index / 2, data}); }, [])}, mark: "bar", encoding: { x: { field: "index",type: "ordinal" }, y: { field: "data", type: "quantitative"} } }); const tags = await vegalite({ width: 500, height: 100, data: {values: profile.tagsData.reduce((memo, data, index) => { return memo.concat({tag: rawData.tags[index], data}); }, [])}, mark: "bar", encoding: { x: { field: "tag",type: "ordinal" }, y: { field: "data", type: "quantitative"} } }); graphView.querySelector("#graph1").appendChild(rating); graphView.querySelector("#graph2").appendChild(tags); } selectUser(1); const predictBtn = document.createElement('button'); predictBtn.innerText = 'get list'; predictBtn.addEventListener('click', async () => { await predict(); }); result.appendChild(userSelect); result.appendChild(graphView); result.appendChild(selectView); selectView.querySelector('#actions').appendChild(predictBtn); return result; }36:[] 32:{"id":"599919b665548e6c","slug":"movielens-recommendation-using-tensorflow-js","title":"Movielens recommendation using Tensorflow.js ","thumbnail":"1256a0c7030788139a07b4c8d89aeaa504c8eb63fefc75c4d09d973f6fde6b7a","access_level":"public","create_time":"$D2019-02-02T14:49:02.132Z","update_time":"$D2019-02-25T07:30:49.111Z","view_time":null,"first_public_time":"$D2019-02-04T06:18:11.836Z","has_unpublished_edits":null,"fork":false,"stars":9,"starred":false,"owner":{"type":"team","id":"423c95581e37a479","login":"cstefanache","name":"Cornel Stefanache","avatar_url":"https://avatars-next.observableusercontent.com/26083dee70ebc8f99b9ec2d7890fa4a8dabeccd8443043ee9e90aa2874204d4b","tier":"starter_2024"},"version":401,"publish_version":401,"role":null,"authored":false,"authors":[{"type":"individual","id":"618e83635ad291a1","login":"cstefanache","name":"Cornel Stefanache","avatar_url":"https://avatars-next.observableusercontent.com/26083dee70ebc8f99b9ec2d7890fa4a8dabeccd8443043ee9e90aa2874204d4b","tier":null}],"members":[{"type":"individual","id":"618e83635ad291a1","login":"cstefanache","name":"Cornel Stefanache","avatar_url":"https://avatars-next.observableusercontent.com/26083dee70ebc8f99b9ec2d7890fa4a8dabeccd8443043ee9e90aa2874204d4b","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":12,"collection":null,"type":"notebook","body":{"title":"Movielens recommendation using Tensorflow.js ","cells":[{"id":0,"value":"md`\n# Movielens recommendation using Tensorflow.js \n**;tldr;** - Select a training data size and the number of epochs and click on **[train]** button. Please be aware that by selecting a large training data and number of epochs the trainig will take more time to finish. After training the network you can **select one user profile** and (at least) **one tag** and click on the **[get list]** button to get the recommendations. The current recommendations will be rendered on two columns: left - current recommendations, right - previous ones.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":103,"value":"md`### Playground`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":153,"value":"viewof config = form(html``)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":161,"value":"\nviewof view = {\n const trainView = document.createElement('div');\n const train = () => {console.log('train');}\n trainView.innerHTML = `\n

Training data: ${config.datasize}

Epochs: ${config.epochs}

\n `\n const trainButton = document.createElement('button');\n trainButton.innerText = 'Train';\n trainButton.addEventListener('click', () => {\n trainButton.disabled = true;\n trainButton.innerText = 'Training...';\n setTimeout(async () => { \n await trainBatch();\n data.trainedModel = true;\n trainButton.disabled = false;\n trainButton.innerText = 'Train'; \n }, 100);\n })\n trainView.appendChild(trainButton);\n \n return trainView;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":326,"value":"viewof predictor = renderPredictor();","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":61,"value":"md`### Tensorflow.JS Model & Functions`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":64,"value":"model = {\n const model = tf.sequential();\n const count = data.trainingData.xs.length;\n const xsLength = data.features;\n \n model.add(tf.layers.dense({units: 8, inputShape: [xsLength], activation: 'relu6'}));\n // model.add(tf.layers.dense({ units: 4, aactivation: 'relu6' }));\n model.add(tf.layers.dense({units: 1}));\n \n model.compile({optimizer: 'sgd', loss: 'meanSquaredError', metrics: ['accuracy']});\n \n return model;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":386,"value":"modelView = {\n const modelView = new ModelView(model, { \n appendImmediately: false, \n renderer: 'd3',\n layer: {\n 'dense_Dense1_input': { columns: 4 }\n }});\n console.log(modelView)\n return modelView.getDOMElement();\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":11,"value":"md`### Data Preparation & Loading `","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":19,"value":" rawData = {\n const moviesData = await d3.csv('https://gist.githubusercontent.com/cstefanache/8b3fe104dc052597e6a1bac3a233f691/raw/8a2ff405ef47b279a4ae655c7e2c480e7de8afbb/movies.csv');\n const ratingsData = await d3.csv('https://gist.githubusercontent.com/cstefanache/8b3fe104dc052597e6a1bac3a233f691/raw/8a2ff405ef47b279a4ae655c7e2c480e7de8afbb/ratings.csv');\n \n const movies = {};\n const tags = [];\n \n moviesData.forEach( movie => {\n const {id, title, tags: movieTags} = movie;\n const tagsSplit = movieTags.split('|');\n tagsSplit.forEach(tag => { \n if (tags.indexOf(tag) === -1) {\n tags.push(tag);\n }\n });\n \n movies[id] = {\n id,\n title,\n tags: tagsSplit\n }\n });\n\n return {\n tags,\n movies, \n ratingsData\n }\n }","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":32,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":310,"value":"md`### Custom Tools`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":75,"value":"$35","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":71,"value":"trainBatch = async (update) => {\n console.log('start!');\n const batchIndex = 0;\n const batchSize = config.datasize;\n const epochs = config.epochs;\n const results = [];\n const xsLength = data.features;\n const from = batchIndex * batchSize;\n const to = from + batchSize;\n const xs = tf.tensor2d(data.trainingData.xs.slice(from, to), [batchSize, xsLength]);\n const ys = tf.tensor2d(data.trainingData.ys.slice(from, to), [batchSize, 1]);\n const history = await model.fit(xs, ys, {\n epochs,\n validationSplit: 0.2\n });\n \n console.log('end!');\n \n return history;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":6,"value":"md`### Imports `","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"d3 = require(\"d3\");","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":9,"value":"tf = require(\"@tensorflow/tfjs\");","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":110,"value":"vegalite = require(\"@observablehq/vega-lite@0.1\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":147,"value":"import {form} from \"@mbostock/form-input\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":227,"value":"import { checkbox } from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":380,"value":"ModelView = require(\"tfjs-model-view@0.1.2\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W36","stdlib":"1","titling":"auto"}} 33:"$32:body"