40:null 41:["$","title",null,{"children":"Exploring Fashion MNIST | Zeta | Observable"}] 47:T50c,controls = { const cam = viewof camera; const form = html``; const rand = d3.randomUniform(mesh.index.length); const jump = id => { let pos = meta.positions[id]; if (pos) { pos = v.mulN(null, v.normalize([], pos), v.mag(pos) + 0.25); cam.focus(pos, 0.07); } }; const formEl = d3.select(form).on("submit", e => e.preventDefault()); const btnGo = d3.select(form.elements["go"]).on("click", e => { jump(form.elements["id"].value); }); const btnRandom = d3.select(form.elements["random"]).on("click", e => { const id = rand()|0; form.elements["id"].value = id; jump(id); }); const btnFS = d3.select(form.elements["fullscreen"]).on("click", e => { viewof gl.parentElement.requestFullscreen(); }); invalidation.then(() => { formEl.on("submit", null); btnGo.on("click", null); btnFS.on("click", null); btnRandom.on("click", null); }); return form; }48:T4ab,md`--- ## Process Inspired by the pioneer works by [Google Arts](https://artsexperiments.withgoogle.com/tsnemap/) and [Google Creative Lab](https://experiments.withgoogle.com/drum-machine) of building tools for massive data visualization with [TSNE](https://lvdmaaten.github.io/tsne/). And following the direction pointed by [Douglas Duhaime](https://douglasduhaime.com/posts/visualizing-tsne-maps-with-three-js.html) and [Grant Custer](https://github.com/GrantCuster/umap-explorer), I was curious about applying the same approach to the Fashion MNIST dataset. ### Initial Exploration First plotted some random samples to get a general feel about the data. ${html`

`} Then tried different UMAP parameters to find a suitable setting. ${html`

`} Once satisfied I exported the result to an \`embedding.json\` file. ### Point Cloud Compression with [Draco](https://github.com/google/draco)`49:T443,md`The next challenge is moving the data images to browsers and GPUs for rendering. There are 70k 28x28 gray scale images, with each pixel taking 1 byte, which means the total data size will be about \`70000 * 28 * 28 = 52 mb\`. That is far from practical. Fortunately last year Google and Binomial released the [Basis Universal Texture Format](https://www.khronos.org/blog/google-and-binomial-contribute-basis-universal-texture-format-to-khronos-gltf-3d-transmission-open-standard) for working with GPU compressed textures. The greatness is that you can tranfer the data in compressed format over the network direct to the GPU. ${html`

Source: GST: GPU-decodable Supercompressed Textures

`} With this settled, I generated an 8192x8192 (to utilize mipmap) image to host all the 70k 28x28 images, then use Basis's encoding tool to compressed it to a ${dataTexture.byteLength/1024|0} kb \`fashion_mnist_full.basis\` file. `4a:T11c0,class Cluster { constructor(mesh, imageData) { this.texImage = createBasisTexture(imageData); this.fbo = webgl.defFBO(gl, { tex: [webgl.defTexture(gl, { width: gl.drawingBufferWidth, height: gl.drawingBufferHeight, premultiply: true, image: null })], depth: webgl.defRBO(gl, { width: gl.drawingBufferWidth, height: gl.drawingBufferHeight }) }); this.model = webgl.compileModel(gl, { attribs: { position: { data: mesh.position }, label: { data: new Float32Array(mesh.label), size: 1 }, index: { data: new Float32Array(mesh.index), size: 1 }, id: { data: new Float32Array(Array.from(mesh.index, ID.encode).flat(1)) } }, num: mesh.position.length / 3, mode: gl.POINTS, uniforms: {}, shader: webgl.defShader(gl, { vs: ` void main() { v_label = label; v_index = index; v_id = id; vec4 pos = view * model * vec4(position, 1.0); gl_Position = proj * pos; v_size = clamp(28.0 / -pos.z, 1.0, 100.0); gl_PointSize = v_size; }`, fs: ` const float cols = 292.0; const float s = 28.0 / 8192.0; float norm(float x, float a, float b) { return (x - a) / (b - a); } vec2 norm(vec2 x, vec2 a, vec2 b) { return (x - a) / (b - a); } vec2 fit(vec2 x, vec2 a, vec2 b, vec2 c, vec2 d) { return c + (d - c) * norm(x, a, b); } float aastep(float threshold, float value) { float afwidth = length(vec2(dFdx(value), dFdy(value))) * 0.70710678118654757; return smoothstep(threshold-afwidth, threshold+afwidth, value); } float frame(vec2 uv, float width) { vec2 d = vec2(dFdx(uv.x), dFdy(uv.y)); float tx = aastep(d.x * width, uv.x) * (1.0 - aastep(1.0 - d.x * width, uv.x)); float ty = aastep(d.y * width, uv.y) * (1.0 - aastep(1.0 - d.y * width, uv.y)); return tx * ty; } void main() { if (picking) { gl_FragColor = vec4(v_id, 1.0); } else { vec3 color = texture2D(cmap, vec2((v_label + .5) * .1, .5)).rgb; vec2 offset = vec2(mod(v_index, cols), floor(v_index / cols)) * s; vec2 pc = gl_PointCoord; float ft = 1.0; if (hover != vec3(0.0) && distance(hover, v_id) < 1e-3) ft = frame(vec2(pc.x, 1.0 - pc.y), 2.0); vec2 uv = fit(pc, vec2(0.0), vec2(1.0), vec2(0.0), vec2(s)) + offset; float alpha = norm(texture2D(tex, uv).r, 0.0, .7); float t = clamp(norm(v_size, 18.0, 1.0),0.0,1.0); color *= mix(alpha, 1.0, t); gl_FragColor = vec4(mix(vec3(1.0), color, ft), 1.0); } }`, declPrefixes: { v: "v_" }, attribs: { position: "vec3", label: "float", index: "float", id: "vec3" }, varying: { label: "float", index: "float", size: "float", id: "vec3" }, uniforms: { model: ["mat4", m.identity44([])], view: "mat4", proj: "mat4", picking: ["bool", 0], hover: 'vec3', cmap: ["sampler2D", 0], tex: ["sampler2D", 1] }, state: { depth: true, blend: false, }, ext: { OES_standard_derivatives: true } }), textures: [makeCmapTexture(d3.schemeCategory10)] }); Object.defineProperties(this, { _pick: { value: new Uint8Array(4), writable: true } }); } pick(x, y) { this.fbo.bind(); gl.readPixels(x, y, 1, 1, gl.RGBA, gl.UNSIGNED_BYTE, this._pick); this.fbo.unbind(); const id = ID.decode(this._pick); return id != 0xffffff ? id : undefined; } drawPick(camera) { this.fbo.bind(); gl.clearColor(1.0, 1.0, 1.0, 1); gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); this.model.uniforms.view = camera.view; this.model.uniforms.proj = camera.proj; gl.useProgram(this.model.shader.program); gl.activeTexture(gl.TEXTURE0 + 1); gl.bindTexture(gl.TEXTURE_2D, this.texImage); gl.uniform1i(this.model.shader.uniforms.tex.loc, 1); this.model.uniforms.picking = 1; webgl.draw(this.model); this.fbo.unbind(); } draw(camera, hover) { this.model.uniforms.view = camera.view; this.model.uniforms.proj = camera.proj; gl.useProgram(this.model.shader.program); gl.activeTexture(gl.TEXTURE0 + 1); gl.bindTexture(gl.TEXTURE_2D, this.texImage); gl.uniform1i(this.model.shader.uniforms.tex.loc, 1); this.model.uniforms.hover = ID.encode(hover); this.model.uniforms.picking = 0; webgl.draw(this.model); } }4b:T4ef,class Guide { constructor() { this.model = webgl.compileModel(gl, { attribs: { position: { data: new Float32Array([ 0, 0, 0, 1, 0, 0, // x 0, 0, 0, 0, 1, 0, // y 0, 0, 0, 0, 0, 1 // z ]) }, color: { data: new Float32Array([ 1, 0, 0, 1, 0, 0, // r 0, 1, 0, 0, 1, 0, // g 0, 0, 1, 0, 0, 1, // b ]), size: 3 } }, mode: webgl.DrawMode.LINES, num: 6, uniforms: {}, shader: webgl.defShader(gl, { vs: ` void main() { v_color = color; gl_Position = proj * view * model * vec4(position, 1.0); }`, fs: ` void main() { gl_FragColor = vec4(v_color, 1.0); }`, declPrefixes: { v: "v_" }, attribs: { position: 'vec3', color: 'vec3' }, varying: { color: "vec3", }, uniforms: { model: ["mat4", m.scale44([], 100)], proj: "mat4", view: "mat4", }, state: { depth: true } }) }); } draw(camera) { this.model.uniforms.view = camera.view; this.model.uniforms.proj = camera.proj; webgl.draw(this.model); } }4c:Td4a,decodeDracoGeometry = { function decodeBuffer(decoder, data) { const buffer = new draco.DecoderBuffer(); buffer.Init(new Int8Array(data), data.byteLength); const geometryType = decoder.GetEncodedGeometryType(buffer); let geometry, status; if (geometryType === draco.TRIANGULAR_MESH) { geometry = new draco.Mesh(); status = decoder.DecodeBufferToMesh(buffer, geometry); } else if (geometryType === draco.POINT_CLOUD) { geometry = new draco.PointCloud(); status = decoder.DecodeBufferToPointCloud(buffer, geometry); } else { draco.destroy(decoder); draco.destroy(geometry); draco.destroy(buffer); throw new Error('Unknown geometry type.'); } if (!status.ok() || geometry.ptr === 0) { draco.destroy(decoder); draco.destroy(geometry); draco.destroy(buffer); throw new Error('Decoding failed: ' + status.error_msg()); } geometry.type = geometryType; draco.destroy(buffer); return geometry; } const ATTRIB_ID = { position: draco.POSITION, normal: draco.NORMAL, color: draco.COLOR, uv: draco.TEX_COORD }; const DATA_TYPE = { [Float32Array]: draco.DT_FLOAT32, [Int8Array]: draco.DT_INT8, [Int16Array]: draco.DT_INT16, [Int32Array]: draco.DT_INT32, [Uint8Array]: draco.DT_UINT8, [Uint16Array]: draco.DT_UINT16, [Uint32Array]: draco.DT_UINT32 }; const HEAP = { [draco.DT_FLOAT32]: draco.HEAPF32, [draco.DT_INT8]: draco.HEAP8, [draco.DT_INT16]: draco.HEAP16, [draco.DT_INT32]: draco.HEAP32, [draco.DT_UINT8]: draco.HEAPU8, [draco.DT_UINT16]: draco.HEAPU16, [draco.DT_UINT32]: draco.HEAPU32 }; function decodeAttribute(decoder, geometry, key, type) { let attId; if (key in ATTRIB_ID) { attId = decoder.GetAttributeId(geometry, ATTRIB_ID[key]); } else { attId = decoder.GetAttributeIdByName(geometry, key); } if (attId == -1) return null; const attribute = decoder.GetAttribute(geometry, attId); const numComponents = attribute.num_components(); const numPoints = geometry.num_points(); const numValues = numPoints * numComponents; const byteLength = numValues * type.BYTES_PER_ELEMENT; const dataType = DATA_TYPE[type]; const ptr = draco._malloc( byteLength ); decoder.GetAttributeDataArrayForAllPoints(geometry, attribute, dataType, byteLength, ptr); const array = new type(HEAP[dataType].buffer, ptr, numValues).slice(); draco._free( ptr ); return array; } function decodeIndices(decoder, geometry) { const numFaces = geometry.num_faces(); const numIndices = numFaces * 3; const byteLength = numIndices * 4; const ptr = draco._malloc(byteLength); decoder.GetTrianglesUInt32Array(geometry, byteLength, ptr); const index = new Uint32Array(draco.HEAPF32.buffer, ptr, numIndices).slice(); draco._free( ptr ); return index; } return (data, attribs) => { const decoder = new draco.Decoder(); const geometry = decodeBuffer(decoder, data); const mesh = {}; for (const key in attribs) { mesh[key] = decodeAttribute(decoder, geometry, key, attribs[key]); } if (geometry.type == draco.TRIANGULAR_MESH) { mesh["indices"] = decodeIndices(decoder, geometry); } draco.destroy(geometry); draco.destroy(decoder); return mesh; } }4d:T154b,createBasisTexture = { const SUPPORTS = { astc: !!gl.getExtension('WEBGL_compressed_texture_astc'), etc: !!gl.getExtension('WEBGL_compressed_texture_etc1'), dxt: !!gl.getExtension('WEBGL_compressed_texture_s3tc'), pvrtc: !!(gl.getExtension('WEBGL_compressed_texture_pvrtc')) || !!(gl.getExtension('WEBKIT_WEBGL_compressed_texture_pvrtc')), bc7: !!gl.getExtension('EXT_texture_compression_bptc') }; const COMPRESSED = { RGBA_ASTC_4x4_KHR: 0x93B0, RGB_S3TC_DXT1_EXT: 0x83F0, RGBA_S3TC_DXT1_EXT: 0x83F1, RGBA_S3TC_DXT3_EXT: 0x83F2, RGBA_S3TC_DXT5_EXT: 0x83F3, RGBA_BPTC_UNORM: 0x8E8C, RGB_ETC1_WEBGL: 0x8D64, RGBA8_ETC2_EAC: 0x9278, RGB_PVRTC_4BPPV1_IMG: 0x8C00, RGBA_PVRTC_4BPPV1_IMG: 0x8C02, }; const BASIS_FORMAT = { ETC1: 0, ETC2: 1, BC1: 2, BC3: 3, BC4: 4, BC5: 5, BC7: 6, PVRTC1_4_RGB: 8, PVRTC1_4_RGBA: 9, ASTC_4x4: 10, ATC_RGB: 11, ATC_RGBA_INTERPOLATED_ALPHA: 12, RGBA32: 13, RGB565: 14, BGR565: 15, RGBA4444: 16, }; const createCompressedTexture = (data, width, height, format) => { const tex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, tex); gl.compressedTexImage2D(gl.TEXTURE_2D, 0, format, width, height, 0, data); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.bindTexture(gl.TEXTURE_2D, null); return tex; }; const createRgb565Texture = (data, width, height) => { const tex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, tex); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, width, height, 0, gl.RGB, gl.UNSIGNED_SHORT_5_6_5, data); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.bindTexture(gl.TEXTURE_2D, null); return tex; }; return data => { const basisFile = new BasisFile(new Uint8Array(data)); function cleanup() { basisFile.close(); basisFile.delete(); } const width = basisFile.getImageWidth(0, 0); const height = basisFile.getImageHeight(0, 0); const images = basisFile.getNumImages(); const levels = basisFile.getNumLevels(0); const has_alpha = basisFile.getHasAlpha(); if (!width || !height || !images || !levels) { cleanup(); throw new Error('Invalid .basis file'); } let format; if (SUPPORTS.astc) { format = BASIS_FORMAT.ASTC_4x4; } else if (SUPPORTS.bc7) { format = BASIS_FORMAT.BC7; } else if (SUPPORTS.dxt) { format = has_alpha ? BASIS_FORMAT.BC3 : BASIS_FORMAT.BC1; } else if (SUPPORTS.pvrtc) { format = has_alpha ? BASIS_FORMAT.PVRTC1_4_RGBA : BASIS_FORMAT.PVRTC1_4_RGB; if (((width & (width - 1)) != 0) || ((height & (height - 1)) != 0)) console.warn('ERROR: PVRTC1 requires square power of 2 textures'); if (width != height) console.warn('ERROR: PVRTC1 requires square power of 2 textures'); } else if (SUPPORTS.etc) { format = BASIS_FORMAT.ETC1; } else { format = BASIS_FORMAT.RGB565; } if (!basisFile.startTranscoding()) { cleanup(); throw new Error('startTranscoding failed'); } const dstSize = basisFile.getImageTranscodedSizeInBytes(0, 0, format); const dst = new Uint8Array(dstSize); if (!basisFile.transcodeImage(dst, 0, 0, format, 0, 0)) { cleanup(); throw new Error('transcodeImage failed'); } cleanup(); const alignedWidth = (width + 3) & ~3; const alignedHeight = (height + 3) & ~3; let tex; if (format === BASIS_FORMAT.ASTC_4x4) { tex = createCompressedTexture(dst, alignedWidth, alignedHeight, COMPRESSED.RGBA_ASTC_4x4_KHR); } else if (format == BASIS_FORMAT.BC3) { tex = createCompressedTexture(dst, alignedWidth, alignedHeight, COMPRESSED.RGBA_S3TC_DXT5_EXT); } else if (format == BASIS_FORMAT.BC1) { tex = createCompressedTexture(dst, alignedWidth, alignedHeight, COMPRESSED.RGB_S3TC_DXT1_EXT); } else if (format == BASIS_FORMAT.BC7) { tex = createCompressedTexture(dst, alignedWidth, alignedHeight, COMPRESSED.RGBA_BPTC_UNORM); } else if (format === BASIS_FORMAT.ETC1) { tex = createCompressedTexture(dst, alignedWidth, alignedHeight, COMPRESSED.RGB_ETC1_WEBGL); } else if (format === BASIS_FORMAT.PVRTC1_4_RGB) { tex = createCompressedTexture(dst, alignedWidth, alignedHeight, COMPRESSED.RGB_PVRTC_4BPPV1_IMG); } else if (format === BASIS_FORMAT.PVRTC1_4_RGBA) { tex = createCompressedTexture(dst, alignedWidth, alignedHeight, COMPRESSED.RGBA_PVRTC_4BPPV1_IMG); } else { // Create 565 texture. const dstTex = new Uint16Array(width * height); // Convert the array of bytes to an array of uint16's. var pix = 0; for (var y = 0; y < height; y++) for (var x = 0; x < width; x++, pix++) dstTex[pix] = dst[2 * pix + 0] | (dst[2 * pix + 1] << 8); tex = createRgb565Texture(dstTex, width, height); } return tex; }; }4e:T516,fetchProgress = { function concat(arrays) { let i = 0, n = 0; for (const a of arrays) n += a.length; const concat = new Uint8Array(n); for (let a of arrays) concat.set(a, i), i += a.length; return concat.buffer; } const progressText = (cur, total) => { const max = total > 0 ? `${total / 1024 | 0}` : '?'; return `${cur / 1024 | 0} / ${max} kb`; }; return (name, url, init) => { const root = html`

${name} waiting...

`; const progress = root.children[1]; const text = root.children[2]; fetch(url, init).then(async response => { const max = response.headers.get("content-length"); progress.max = max || Number.MAX_SAFE_INTEGER; text.textContent = progressText(0, max); const reader = response.body.getReader(); const values = []; while (true) { const {done, value} = await reader.read(); if (done) break; progress.value += value.length; text.textContent = progressText(progress.value, max); values.push(value); } root.value = concat(values); root.dispatchEvent(new CustomEvent("input")); }); return root; }; }4f:Tb04,class Camera { constructor(opts) { const { viewport, pos, target = v.ZERO3, up = v.Y3, fov = 45, near = 0.01, far = 50, } = opts; Object.defineProperties(this, { _viewport: { value: viewport, writable: true }, _radius: { value: Math.min(...viewport), writable: true }, _center: { value: v.mulN([], viewport, 0.5), writable: true }, _pos: { value: pos, writable: true }, _up: { value: v.copy(up), writable: true }, _target: { value: v.copy(target), writable: true }, // drag start _camUp: { value: [], writable: true }, _viewDir: { value: [], writable: true }, _down: { value: [], writable: true }, // drag _delta: { value: [], writable: true }, _side: { value: [], writable: true }, _axis: { value: [], writable: true }, _vd: { value: [], writable: true }, _rot: { value: [], writable: true }, }); this.view = m.lookAt([], this.pos, this.target, this.up); this.proj = m.perspective([], fov, viewport[0] / viewport[1], near, far); } _spherePos(out, x, y) { v.divN(null, v.setC3(out, x - this._center[0], -(y - this._center[1]), 0), this._radius); const mag = v.magSq(out); return mag > 1.0 ? v.normalize(null, out) : (out[2] = Math.sqrt(1.0 - mag), out); } _updateView() { m.lookAt(this.view, this.pos, this.target, this.up); return this; } get pos() { return this._pos; } set pos(val) { this._pos = val; return this._updateView(); } get target() { return this._target; } set target(val) { this._target = val; return this._updateView(); } get up() { return this._up; } set up(val) { this._up = val; return this._updateView(); } start(x, y) { this._spherePos(this._down, x, y); v.set(this._camUp, this.up); v.sub(this._viewDir, this.pos, this.target); return this; } drag(x, y) { v.sub(null, this._spherePos(this._delta, x, y), this._down); v.cross3(this._side, this._camUp, this._viewDir) v.mulN(null, v.normalize(null, this._side), this._delta[0]); v.mulN(this._axis, this._camUp, this._delta[1]); v.add(null, this._axis, this._side); v.cross3(null, this._axis, this._viewDir) v.normalize(null, this._axis); m.rotationAroundAxis33(this._rot, this._axis, v.magSq(this._delta) * 5); m.mulV(this.up, this._rot, this._camUp); v.add(this.pos, this.target, m.mulV(this._vd, this._rot, this._viewDir)); return this._updateView(); } zoom(delta, min = 0.1, max = 10.0) { v.sub(this._vd, this.target, this.pos); const mag = Math.max(min, Math.min(max, Math.pow(Math.E, delta) * v.mag(this._vd))); v.mulN(null, v.sub(this.pos, this.target, v.normalize(null, this._vd)), mag); return this._updateView(); } }50:T6be,class View { static create(initialize) { return new Promise(resolve => { const view = new View(); let resolved = false; const dispose = initialize(x => { view.value = x; if (!resolved) { resolved = true; resolve(Generators.disposable(view, () => { dispose && dispose(); view.dispose(); })); }; }) }); } static subs(unsubs) { return () => unsubs.forEach(unsub => unsub()); } constructor(value) { Object.defineProperties(this, { _value: { value, writable: true }, _list: { value: {}, writable: true }, _subs: { value: [], writable: true }, }); const dispatch = e => this._subs.forEach(sub => sub(e.value)); this.addEventListener('input', dispatch); this.dispose = () => this.removeEventListener("input", dispatch); }; get value() { return this._value; } set value(val) { this._value = val; this.notify(); } observe(sub) { this._subs.push(sub); sub(this.value); // give out the current value on subscription return () => this._subs = this._subs.filter(l => l !== sub); } notify() { this.dispatchEvent({ type: "input", value: this.value }); } addEventListener(type, listener) { if (!this._list[type]) this._list[type] = []; if (this._list[type].includes(listener)) return; this._list[type] = [listener].concat(this._list[type]); } removeEventListener(type, listener) { this._list[type] = this._list[type].filter(l => l !== listener); } dispatchEvent(event) { if (!this._list[event.type]) return; const p = Promise.resolve(event); this._list[event.type].forEach(l => p.then(l)); } }51:[{"id":"7314847946bc9733d95fa9f88e7f3ed6f03a7b29d1d7adc317b671720101306287f0654878039f75d4aa7d1ecc254b8af69763bfd5436780b9ab527f675257e9","href":"https://static.observableusercontent.com/files/7314847946bc9733d95fa9f88e7f3ed6f03a7b29d1d7adc317b671720101306287f0654878039f75d4aa7d1ecc254b8af69763bfd5436780b9ab527f675257e9","name":"embedding@1.drc","type":"application/octet-stream","lastModified":1675090144482.546,"size":325522},{"id":"23f330509d1eaa08f4cff62528ff4a2bb1ed42cf33d0d3b580c6e752b3f126e326ca99859da421f64a3c5124ac2d11032aa109bf475d269378fb3ac5e550400e","href":"https://static.observableusercontent.com/files/23f330509d1eaa08f4cff62528ff4a2bb1ed42cf33d0d3b580c6e752b3f126e326ca99859da421f64a3c5124ac2d11032aa109bf475d269378fb3ac5e550400e","name":"fashion_mnist_full@1.basis","type":"application/octet-stream","lastModified":1675090144482.546,"size":9458986},{"id":"99783ef5e2d8670f89eaf1b8f3ae703e04d8ead78572ac1b12590d3d3c0eb8585d3f30bf406150b337545019aa22deb3070f323d0b987e95eca7c6a8a5de2d88","href":"https://static.observableusercontent.com/files/99783ef5e2d8670f89eaf1b8f3ae703e04d8ead78572ac1b12590d3d3c0eb8585d3f30bf406150b337545019aa22deb3070f323d0b987e95eca7c6a8a5de2d88","name":"asset_flat.png","type":"image/png","lastModified":1675090144482.546,"size":359909},{"id":"0ff591ecaf96c676c84bf4a4bee11a01d61f90a46989435b32167e69ed414552489581e0db6e9fd1b695e29280e015c497b531d0cafba2c1811bf9ad42b26af5","href":"https://static.observableusercontent.com/files/0ff591ecaf96c676c84bf4a4bee11a01d61f90a46989435b32167e69ed414552489581e0db6e9fd1b695e29280e015c497b531d0cafba2c1811bf9ad42b26af5","name":"fashion_umap_3d.png","type":"image/png","lastModified":1675090144482.546,"size":140653}] 45:{"id":"197e0e21c69eefe3","slug":null,"title":"Exploring Fashion MNIST","thumbnail":"c2fb57f8287d05bd36a376be047786f4b42bee4582cd7fba4b0eeaf3246179ec","access_level":"public","create_time":"$D2023-01-30T14:49:04.482Z","update_time":"$D2023-01-30T14:49:04.482Z","view_time":null,"first_public_time":"$D2023-01-30T14:49:04.482Z","has_unpublished_edits":null,"fork":{"id":"54c6302d5ad5d228","slug":"exploring-fashion-mnist","title":"Exploring Fashion MNIST","owner":{"id":"7fad80bf128915e9","login":"stwind","name":"Zihou Ng"}},"stars":0,"starred":false,"owner":{"type":"team","id":"171a4b7d64a95d10","login":"zeitgeberh","name":"Zeta","avatar_url":"https://avatars-next.observableusercontent.com/e07d720718cbbace381b252fd42d674b7ea5d79d17ec3266c784f953ee7d2d9c","tier":"starter_2024","bio":"System neuroscientist\r\n","following":false},"version":1662,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"05b311542350cd48","login":"zeitgeberh","name":"Zeta","avatar_url":"https://avatars-next.observableusercontent.com/e07d720718cbbace381b252fd42d674b7ea5d79d17ec3266c784f953ee7d2d9c","tier":null,"bio":"System neuroscientist\r\n","following":false}],"members":[{"type":"individual","id":"05b311542350cd48","login":"zeitgeberh","name":"Zeta","avatar_url":"https://avatars-next.observableusercontent.com/e07d720718cbbace381b252fd42d674b7ea5d79d17ec3266c784f953ee7d2d9c","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Exploring Fashion MNIST","cells":[{"id":0,"value":"md`# Exploring Fashion MNIST`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":472,"value":"md`[Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist) is a MNIST-like fashion product dataset that contains 70,000 28x28 gray scale images each associated with a label from 10 classes. \n\nThis is an explorer of all the 70k images, clustered and embedded into 3 dimensions space using the [UMAP](https://umap-learn.readthedocs.io/en/latest/) algorithm. You can navigate by dragging the camera, clicking a specific image, jumping with an id or picking randomly.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1411,"value":"md`---`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":539,"value":"viewof dataEmbedding = fetchProgress(\"embedding\", await FileAttachment(\"embedding@1.drc\").url())","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":650,"value":"viewof dataTexture = fetchProgress(\"texture\", await FileAttachment(\"fashion_mnist_full@1.basis\").url())","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":36,"value":"viewof gl = {\n const canvas = DOM.canvas(width, width / 1.618);\n const gl = canvas.value = canvas.getContext('webgl');\n gl.getExtension(\"WEBGL_draw_buffers\");\n gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);\n gl.clearColor(0.2, 0.2, 0.2, 1);\n gl.clear(gl.COLOR_BUFFER_BIT);\n \n const root = html`

\n \n\n ${canvas}\n

`;\n root.value = gl;\n \n return root;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1061,"value":"$47","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1534,"value":"$48","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":988,"value":"md`Now the problem is that the generated UMAP embedding are quite large, which is about 4.5 MB. \n\nThis is where Draco plays the important role. With Draco I compressed the point cloud to an impressive ${dataEmbedding.byteLength/1024|0} kb \\`embedding.drc\\` file.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1549,"value":"md`### Data Texture Compression with [Basis Universal](https://github.com/BinomialLLC/basis_universal)`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1544,"value":"$49","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1540,"value":"md`### Rendering The Scene\n\nFinally it is time to render the scene using [thi.ng/umbrella](https://thi.ng/umbrella) and [D3](https://d3js.org/). The steps are quite straight forward, mainly involved the following:\n\n* Loading and decoding assets.\n* Draw the data as \\`gl.POINTS\\`.\n* In shaders\n * Calculate the texture offsets by point indices.\n * Colormapping based on data labels.\n * For far away points, fade to pure color.\n* Handle picking and camera transition.\n\n*All assets generation code is available in this [notebook](https://colab.research.google.com/gist/stwind/92ff73e0a599d5bc7dcdc9d15ad32abf/umap-fashion-mnist.ipynb).*`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":982,"value":"md`---\n## Code`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":42,"value":"draw = {\n gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);\n \n cluster.drawPick(camera);\n \n gl.clearColor(0.2, 0.2, 0.2, 1);\n gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);\n \n guide.draw(camera);\n cluster.draw(camera, hover);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":697,"value":"hover = Generators.observe(next => {\n const canvas = d3.select(gl.canvas);\n let cur = undefined;\n next(cur);\n \n const root = d3.select(viewof gl);\n const valId = root.select('.val-id');\n const valLabel = root.select('.val-label');\n \n canvas.on('mousemove', e => {\n const id = cluster.pick(e.offsetX, gl.drawingBufferHeight - e.offsetY);\n if (id != cur) {\n next(cur = id);\n if (cur !== undefined) {\n valId.text(id);\n valLabel.text(classnames[meta.labels[id]]);\n } else {\n valId.text(\"\");\n valLabel.text(\"\");\n }\n }\n });\n canvas.on(\"mouseout\", e => {\n next(cur = undefined);\n valId.text(\"\");\n valLabel.text(\"\");\n });\n\n return () => canvas.on('mousemove', null).on(\"mouseout\", null);\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":873,"value":"cluster = new Cluster(mesh, dataTexture)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":66,"value":"guide = new Guide()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1156,"value":"viewof camera = new class extends View {\n constructor() {\n super();\n this.camera = new Camera({\n viewport: [gl.drawingBufferWidth, gl.drawingBufferHeight],\n pos: [3.58453, -4.48565, 1.22442],\n far: 70,\n });\n }\n \n dragStart(x, y) {\n this.cancel && this.cancel();\n this.camera.start(x, y);\n }\n \n drag(x, y) {\n this.camera.drag(x, y);\n this.notify();\n }\n \n zoom(delta, min, max) {\n this.cancel && this.cancel();\n this.camera.zoom(delta, min, max);\n this.notify();\n }\n \n focus(to, speed, eps) {\n this.cancel && this.cancel();\n const from = v.copy(this.camera.pos);\n this.cancel = tween(t => {\n this.camera.pos = v.mixN(this.camera.pos, from, to, t);\n this.notify();\n }, speed, eps);\n }\n \n get value() {\n return this.camera;\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":152,"value":"mesh = decodeDracoGeometry(dataEmbedding, {\n position: Float32Array,\n label: Uint8Array,\n index: Uint32Array\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":770,"value":"meta = {\n const { position, index, label } = mesh;\n const positions = {}, labels = {};\n for (let i = 0; i < index.length; i++) {\n const pos = [position[i * 3], position[i * 3 + 1], position[i * 3 + 2]];\n positions[index[i]] = pos;\n labels[index[i]] = label[i];\n }\n return { positions, labels };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":943,"value":"classnames = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat','Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":913,"value":"initialized = {\n dataEmbedding;\n dataTexture;\n \n const root = d3.select(viewof gl);\n root.select(\".title\").style(\"display\", \"none\");\n root.select(\".info\").style(\"display\", \"block\");\n \n // initialize camera\n const cam = viewof camera;\n cam.focus(v.rotateY(null, v.rotateX(null, [0,0,30], -Math.PI/10), Math.PI/4), 0.025);\n \n const canvas = d3.select(gl.canvas);\n canvas.call(d3.drag().on('start', e => {\n cam.dragStart(e.x, e.y);\n e.on('drag', e => cam.drag(e.x, e.y));\n }));\n canvas.on('wheel', e => {\n e.preventDefault();\n cam.zoom(e.deltaY * 1e-3, 0.01, 50);\n });\n \n canvas.on('click', e => {\n const id = cluster.pick(e.offsetX, gl.drawingBufferHeight - e.offsetY);\n if (id) {\n let pos = meta.positions[id];\n pos = v.mulN(null, v.normalize([], pos), v.mag(pos) + 0.25)\n cam.focus(pos, 0.1);\n }\n });\n \n invalidation.then(() => {\n canvas.on(\"drag\", null).on(\"wheel\", null).on(\"click\", null);\n });\n \n return true;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":710,"value":"md`### Components`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":844,"value":"$4a","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":713,"value":"$4b","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":30,"value":"md`### Utils`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":662,"value":"ID = ({\n encode: val => {\n const r = ( val >> 16 & 0xff ) / 255;\n const g = ( val >> 8 & 0xff ) / 255;\n const b = ( val & 0xff ) / 255;\n return [r, g, b];\n },\n decode: ([r, g, b]) => (r << 16) | (g << 8) | b\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":148,"value":"$4c","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":228,"value":"$4d","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":515,"value":"makeCmapTexture = {\n const toRGB = c => {\n const { r, g, b } = d3.rgb(c);\n return [r, g, b];\n };\n \n return colors => webgl.defTexture(gl, {\n width: colors.length, height: 1,\n format: gl.RGB,\n image: new Uint8Array(colors.map(toRGB).flat(1))\n });\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":544,"value":"$4e","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":33,"value":"$4f","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1053,"value":"$50","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":728,"value":"tween = (f, speed = 0.08, eps = 1e-4) => {\n let t = 0;\n let handle = requestAnimationFrame(function frame() {\n f(t);\n t += (1 - t) * speed;\n if (Math.abs(1 - t) > eps) {\n handle = requestAnimationFrame(frame);\n }\n });\n\n return () => cancelAnimationFrame(handle);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":319,"value":"fps = {\n let t = performance.now(), delta = 0;\n const last = [t];\n return cell => {\n last.length >= 10 && last.shift();\n t = performance.now();\n delta = (t - last[0]) / last.length;\n last.push(t);\n return Math.round(1000 / delta);\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":13,"value":"md`### Appendix`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":19,"value":"webgl = require('https://bundle.run/@thi.ng/webgl@3.3.3')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":21,"value":"m = require('https://bundle.run/@thi.ng/matrices@0.6.33')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":23,"value":"v = require('https://bundle.run/@thi.ng/vectors@4.6.6')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":118,"value":"draco = {\n const PREFIX = \"https://www.gstatic.com/draco/v1/decoders\";\n const wasmBinary = await fetch(`${PREFIX}/draco_decoder.wasm`).then(resp => {\n if (!resp.ok)\n throw new Error(\"HTTP error, status = \" + resp.status);\n return resp.arrayBuffer();\n });\n \n const mod = await require(`${PREFIX}/draco_wasm_wrapper.js`);\n return new Promise(resolve => mod({ wasmBinary }).then(draco => {\n delete draco.then; // tell Observable not wait for this;\n resolve(draco);\n }));\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":225,"value":"BasisFile = {\n const BASIS = await require('three@0.120.1/examples/js/libs/basis/basis_transcoder.js');\n return new Promise(resolve => BASIS().then(transcoder => {\n transcoder.initializeBasis();\n resolve(transcoder.BasisFile);\n }));\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":3,"value":"md`---\n## Bibliography\n\n* [GrantCuster/umap-explorer: An interactive UMAP visualization of the MNIST data set.](https://github.com/GrantCuster/umap-explorer)\n* [YaleDHLab/pix-plot: A WebGL viewer for UMAP or TSNE-clustered images](https://github.com/YaleDHLab/pix-plot)\n* [Tensorflow Embedding Projector](https://projector.tensorflow.org/)\n* [Google Arts & Culture Experiments - t-SNE Map Experiment](https://artsexperiments.withgoogle.com/tsnemap/)\n* [The Infinite Drum Machine by Manny Tan & Kyle McDonald | Experiments with Google](https://experiments.withgoogle.com/drum-machine)\n* [Visualizing an art collection | Audun M Øygard](https://www.auduno.com/2018/10/27/visualizing-an-art-collection/)\n* [Visualizing High-Dimensional Space by Daniel Smilkov, Fernanda Viégas, Martin Wattenberg & the Big Picture team at Google | Experiments with Google](https://experiments.withgoogle.com/visualizing-high-dimensional-space)\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W51","stdlib":"1"}} 46:"$45:body"