25:["$","title",null,{"children":"Exploring Fashion MNIST | marlow deneal | Observable"}] 49:T50c,controls = { const cam = viewof camera; const form = html``; const rand = d3.randomUniform(mesh.index.length); const jump = id => { let pos = meta.positions[id]; if (pos) { pos = v.mulN(null, v.normalize([], pos), v.mag(pos) + 0.25); cam.focus(pos, 0.07); } }; const formEl = d3.select(form).on("submit", e => e.preventDefault()); const btnGo = d3.select(form.elements["go"]).on("click", e => { jump(form.elements["id"].value); }); const btnRandom = d3.select(form.elements["random"]).on("click", e => { const id = rand()|0; form.elements["id"].value = id; jump(id); }); const btnFS = d3.select(form.elements["fullscreen"]).on("click", e => { viewof gl.parentElement.requestFullscreen(); }); invalidation.then(() => { formEl.on("submit", null); btnGo.on("click", null); btnFS.on("click", null); btnRandom.on("click", null); }); return form; }4a:T4ab,md`--- ## Process Inspired by the pioneer works by [Google Arts](https://artsexperiments.withgoogle.com/tsnemap/) and [Google Creative Lab](https://experiments.withgoogle.com/drum-machine) of building tools for massive data visualization with [TSNE](https://lvdmaaten.github.io/tsne/). And following the direction pointed by [Douglas Duhaime](https://douglasduhaime.com/posts/visualizing-tsne-maps-with-three-js.html) and [Grant Custer](https://github.com/GrantCuster/umap-explorer), I was curious about applying the same approach to the Fashion MNIST dataset. ### Initial Exploration First plotted some random samples to get a general feel about the data. ${html`

`} Then tried different UMAP parameters to find a suitable setting. ${html`

`} Once satisfied I exported the result to an \`embedding.json\` file. ### Point Cloud Compression with [Draco](https://github.com/google/draco)`4b:T443,md`The next challenge is moving the data images to browsers and GPUs for rendering. There are 70k 28x28 gray scale images, with each pixel taking 1 byte, which means the total data size will be about \`70000 * 28 * 28 = 52 mb\`. That is far from practical. Fortunately last year Google and Binomial released the [Basis Universal Texture Format](https://www.khronos.org/blog/google-and-binomial-contribute-basis-universal-texture-format-to-khronos-gltf-3d-transmission-open-standard) for working with GPU compressed textures. The greatness is that you can tranfer the data in compressed format over the network direct to the GPU. ${html`

Source: GST: GPU-decodable Supercompressed Textures

`} With this settled, I generated an 8192x8192 (to utilize mipmap) image to host all the 70k 28x28 images, then use Basis's encoding tool to compressed it to a ${dataTexture.byteLength/1024|0} kb \`fashion_mnist_full.basis\` file. `4c:T11c0,class Cluster { constructor(mesh, imageData) { this.texImage = createBasisTexture(imageData); this.fbo = webgl.defFBO(gl, { tex: [webgl.defTexture(gl, { width: gl.drawingBufferWidth, height: gl.drawingBufferHeight, premultiply: true, image: null })], depth: webgl.defRBO(gl, { width: gl.drawingBufferWidth, height: gl.drawingBufferHeight }) }); this.model = webgl.compileModel(gl, { attribs: { position: { data: mesh.position }, label: { data: new Float32Array(mesh.label), size: 1 }, index: { data: new Float32Array(mesh.index), size: 1 }, id: { data: new Float32Array(Array.from(mesh.index, ID.encode).flat(1)) } }, num: mesh.position.length / 3, mode: gl.POINTS, uniforms: {}, shader: webgl.defShader(gl, { vs: ` void main() { v_label = label; v_index = index; v_id = id; vec4 pos = view * model * vec4(position, 1.0); gl_Position = proj * pos; v_size = clamp(28.0 / -pos.z, 1.0, 100.0); gl_PointSize = v_size; }`, fs: ` const float cols = 292.0; const float s = 28.0 / 8192.0; float norm(float x, float a, float b) { return (x - a) / (b - a); } vec2 norm(vec2 x, vec2 a, vec2 b) { return (x - a) / (b - a); } vec2 fit(vec2 x, vec2 a, vec2 b, vec2 c, vec2 d) { return c + (d - c) * norm(x, a, b); } float aastep(float threshold, float value) { float afwidth = length(vec2(dFdx(value), dFdy(value))) * 0.70710678118654757; return smoothstep(threshold-afwidth, threshold+afwidth, value); } float frame(vec2 uv, float width) { vec2 d = vec2(dFdx(uv.x), dFdy(uv.y)); float tx = aastep(d.x * width, uv.x) * (1.0 - aastep(1.0 - d.x * width, uv.x)); float ty = aastep(d.y * width, uv.y) * (1.0 - aastep(1.0 - d.y * width, uv.y)); return tx * ty; } void main() { if (picking) { gl_FragColor = vec4(v_id, 1.0); } else { vec3 color = texture2D(cmap, vec2((v_label + .5) * .1, .5)).rgb; vec2 offset = vec2(mod(v_index, cols), floor(v_index / cols)) * s; vec2 pc = gl_PointCoord; float ft = 1.0; if (hover != vec3(0.0) && distance(hover, v_id) < 1e-3) ft = frame(vec2(pc.x, 1.0 - pc.y), 2.0); vec2 uv = fit(pc, vec2(0.0), vec2(1.0), vec2(0.0), vec2(s)) + offset; float alpha = norm(texture2D(tex, uv).r, 0.0, .7); float t = clamp(norm(v_size, 18.0, 1.0),0.0,1.0); color *= mix(alpha, 1.0, t); gl_FragColor = vec4(mix(vec3(1.0), color, ft), 1.0); } }`, declPrefixes: { v: "v_" }, attribs: { position: "vec3", label: "float", index: "float", id: "vec3" }, varying: { label: "float", index: "float", size: "float", id: "vec3" }, uniforms: { model: ["mat4", m.identity44([])], view: "mat4", proj: "mat4", picking: ["bool", 0], hover: 'vec3', cmap: ["sampler2D", 0], tex: ["sampler2D", 1] }, state: { depth: true, blend: false, }, ext: { OES_standard_derivatives: true } }), textures: [makeCmapTexture(d3.schemeCategory10)] }); Object.defineProperties(this, { _pick: { value: new Uint8Array(4), writable: true } }); } pick(x, y) { this.fbo.bind(); gl.readPixels(x, y, 1, 1, gl.RGBA, gl.UNSIGNED_BYTE, this._pick); this.fbo.unbind(); const id = ID.decode(this._pick); return id != 0xffffff ? id : undefined; } drawPick(camera) { this.fbo.bind(); gl.clearColor(1.0, 1.0, 1.0, 1); gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); this.model.uniforms.view = camera.view; this.model.uniforms.proj = camera.proj; gl.useProgram(this.model.shader.program); gl.activeTexture(gl.TEXTURE0 + 1); gl.bindTexture(gl.TEXTURE_2D, this.texImage); gl.uniform1i(this.model.shader.uniforms.tex.loc, 1); this.model.uniforms.picking = 1; webgl.draw(this.model); this.fbo.unbind(); } draw(camera, hover) { this.model.uniforms.view = camera.view; this.model.uniforms.proj = camera.proj; gl.useProgram(this.model.shader.program); gl.activeTexture(gl.TEXTURE0 + 1); gl.bindTexture(gl.TEXTURE_2D, this.texImage); gl.uniform1i(this.model.shader.uniforms.tex.loc, 1); this.model.uniforms.hover = ID.encode(hover); this.model.uniforms.picking = 0; webgl.draw(this.model); } }4d:T4ef,class Guide { constructor() { this.model = webgl.compileModel(gl, { attribs: { position: { data: new Float32Array([ 0, 0, 0, 1, 0, 0, // x 0, 0, 0, 0, 1, 0, // y 0, 0, 0, 0, 0, 1 // z ]) }, color: { data: new Float32Array([ 1, 0, 0, 1, 0, 0, // r 0, 1, 0, 0, 1, 0, // g 0, 0, 1, 0, 0, 1, // b ]), size: 3 } }, mode: webgl.DrawMode.LINES, num: 6, uniforms: {}, shader: webgl.defShader(gl, { vs: ` void main() { v_color = color; gl_Position = proj * view * model * vec4(position, 1.0); }`, fs: ` void main() { gl_FragColor = vec4(v_color, 1.0); }`, declPrefixes: { v: "v_" }, attribs: { position: 'vec3', color: 'vec3' }, varying: { color: "vec3", }, uniforms: { model: ["mat4", m.scale44([], 100)], proj: "mat4", view: "mat4", }, state: { depth: true } }) }); } draw(camera) { this.model.uniforms.view = camera.view; this.model.uniforms.proj = camera.proj; webgl.draw(this.model); } }4e:Td4a,decodeDracoGeometry = { function decodeBuffer(decoder, data) { const buffer = new draco.DecoderBuffer(); buffer.Init(new Int8Array(data), data.byteLength); const geometryType = decoder.GetEncodedGeometryType(buffer); let geometry, status; if (geometryType === draco.TRIANGULAR_MESH) { geometry = new draco.Mesh(); status = decoder.DecodeBufferToMesh(buffer, geometry); } else if (geometryType === draco.POINT_CLOUD) { geometry = new draco.PointCloud(); status = decoder.DecodeBufferToPointCloud(buffer, geometry); } else { draco.destroy(decoder); draco.destroy(geometry); draco.destroy(buffer); throw new Error('Unknown geometry type.'); } if (!status.ok() || geometry.ptr === 0) { draco.destroy(decoder); draco.destroy(geometry); draco.destroy(buffer); throw new Error('Decoding failed: ' + status.error_msg()); } geometry.type = geometryType; draco.destroy(buffer); return geometry; } const ATTRIB_ID = { position: draco.POSITION, normal: draco.NORMAL, color: draco.COLOR, uv: draco.TEX_COORD }; const DATA_TYPE = { [Float32Array]: draco.DT_FLOAT32, [Int8Array]: draco.DT_INT8, [Int16Array]: draco.DT_INT16, [Int32Array]: draco.DT_INT32, [Uint8Array]: draco.DT_UINT8, [Uint16Array]: draco.DT_UINT16, [Uint32Array]: draco.DT_UINT32 }; const HEAP = { [draco.DT_FLOAT32]: draco.HEAPF32, [draco.DT_INT8]: draco.HEAP8, [draco.DT_INT16]: draco.HEAP16, [draco.DT_INT32]: draco.HEAP32, [draco.DT_UINT8]: draco.HEAPU8, [draco.DT_UINT16]: draco.HEAPU16, [draco.DT_UINT32]: draco.HEAPU32 }; function decodeAttribute(decoder, geometry, key, type) { let attId; if (key in ATTRIB_ID) { attId = decoder.GetAttributeId(geometry, ATTRIB_ID[key]); } else { attId = decoder.GetAttributeIdByName(geometry, key); } if (attId == -1) return null; const attribute = decoder.GetAttribute(geometry, attId); const numComponents = attribute.num_components(); const numPoints = geometry.num_points(); const numValues = numPoints * numComponents; const byteLength = numValues * type.BYTES_PER_ELEMENT; const dataType = DATA_TYPE[type]; const ptr = draco._malloc( byteLength ); decoder.GetAttributeDataArrayForAllPoints(geometry, attribute, dataType, byteLength, ptr); const array = new type(HEAP[dataType].buffer, ptr, numValues).slice(); draco._free( ptr ); return array; } function decodeIndices(decoder, geometry) { const numFaces = geometry.num_faces(); const numIndices = numFaces * 3; const byteLength = numIndices * 4; const ptr = draco._malloc(byteLength); decoder.GetTrianglesUInt32Array(geometry, byteLength, ptr); const index = new Uint32Array(draco.HEAPF32.buffer, ptr, numIndices).slice(); draco._free( ptr ); return index; } return (data, attribs) => { const decoder = new draco.Decoder(); const geometry = decodeBuffer(decoder, data); const mesh = {}; for (const key in attribs) { mesh[key] = decodeAttribute(decoder, geometry, key, attribs[key]); } if (geometry.type == draco.TRIANGULAR_MESH) { mesh["indices"] = decodeIndices(decoder, geometry); } draco.destroy(geometry); draco.destroy(decoder); return mesh; } }4f:T154b,createBasisTexture = { const SUPPORTS = { astc: !!gl.getExtension('WEBGL_compressed_texture_astc'), etc: !!gl.getExtension('WEBGL_compressed_texture_etc1'), dxt: !!gl.getExtension('WEBGL_compressed_texture_s3tc'), pvrtc: !!(gl.getExtension('WEBGL_compressed_texture_pvrtc')) || !!(gl.getExtension('WEBKIT_WEBGL_compressed_texture_pvrtc')), bc7: !!gl.getExtension('EXT_texture_compression_bptc') }; const COMPRESSED = { RGBA_ASTC_4x4_KHR: 0x93B0, RGB_S3TC_DXT1_EXT: 0x83F0, RGBA_S3TC_DXT1_EXT: 0x83F1, RGBA_S3TC_DXT3_EXT: 0x83F2, RGBA_S3TC_DXT5_EXT: 0x83F3, RGBA_BPTC_UNORM: 0x8E8C, RGB_ETC1_WEBGL: 0x8D64, RGBA8_ETC2_EAC: 0x9278, RGB_PVRTC_4BPPV1_IMG: 0x8C00, RGBA_PVRTC_4BPPV1_IMG: 0x8C02, }; const BASIS_FORMAT = { ETC1: 0, ETC2: 1, BC1: 2, BC3: 3, BC4: 4, BC5: 5, BC7: 6, PVRTC1_4_RGB: 8, PVRTC1_4_RGBA: 9, ASTC_4x4: 10, ATC_RGB: 11, ATC_RGBA_INTERPOLATED_ALPHA: 12, RGBA32: 13, RGB565: 14, BGR565: 15, RGBA4444: 16, }; const createCompressedTexture = (data, width, height, format) => { const tex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, tex); gl.compressedTexImage2D(gl.TEXTURE_2D, 0, format, width, height, 0, data); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.bindTexture(gl.TEXTURE_2D, null); return tex; }; const createRgb565Texture = (data, width, height) => { const tex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, tex); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, width, height, 0, gl.RGB, gl.UNSIGNED_SHORT_5_6_5, data); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.bindTexture(gl.TEXTURE_2D, null); return tex; }; return data => { const basisFile = new BasisFile(new Uint8Array(data)); function cleanup() { basisFile.close(); basisFile.delete(); } const width = basisFile.getImageWidth(0, 0); const height = basisFile.getImageHeight(0, 0); const images = basisFile.getNumImages(); const levels = basisFile.getNumLevels(0); const has_alpha = basisFile.getHasAlpha(); if (!width || !height || !images || !levels) { cleanup(); throw new Error('Invalid .basis file'); } let format; if (SUPPORTS.astc) { format = BASIS_FORMAT.ASTC_4x4; } else if (SUPPORTS.bc7) { format = BASIS_FORMAT.BC7; } else if (SUPPORTS.dxt) { format = has_alpha ? BASIS_FORMAT.BC3 : BASIS_FORMAT.BC1; } else if (SUPPORTS.pvrtc) { format = has_alpha ? BASIS_FORMAT.PVRTC1_4_RGBA : BASIS_FORMAT.PVRTC1_4_RGB; if (((width & (width - 1)) != 0) || ((height & (height - 1)) != 0)) console.warn('ERROR: PVRTC1 requires square power of 2 textures'); if (width != height) console.warn('ERROR: PVRTC1 requires square power of 2 textures'); } else if (SUPPORTS.etc) { format = BASIS_FORMAT.ETC1; } else { format = BASIS_FORMAT.RGB565; } if (!basisFile.startTranscoding()) { cleanup(); throw new Error('startTranscoding failed'); } const dstSize = basisFile.getImageTranscodedSizeInBytes(0, 0, format); const dst = new Uint8Array(dstSize); if (!basisFile.transcodeImage(dst, 0, 0, format, 0, 0)) { cleanup(); throw new Error('transcodeImage failed'); } cleanup(); const alignedWidth = (width + 3) & ~3; const alignedHeight = (height + 3) & ~3; let tex; if (format === BASIS_FORMAT.ASTC_4x4) { tex = createCompressedTexture(dst, alignedWidth, alignedHeight, COMPRESSED.RGBA_ASTC_4x4_KHR); } else if (format == BASIS_FORMAT.BC3) { tex = createCompressedTexture(dst, alignedWidth, alignedHeight, COMPRESSED.RGBA_S3TC_DXT5_EXT); } else if (format == BASIS_FORMAT.BC1) { tex = createCompressedTexture(dst, alignedWidth, alignedHeight, COMPRESSED.RGB_S3TC_DXT1_EXT); } else if (format == BASIS_FORMAT.BC7) { tex = createCompressedTexture(dst, alignedWidth, alignedHeight, COMPRESSED.RGBA_BPTC_UNORM); } else if (format === BASIS_FORMAT.ETC1) { tex = createCompressedTexture(dst, alignedWidth, alignedHeight, COMPRESSED.RGB_ETC1_WEBGL); } else if (format === BASIS_FORMAT.PVRTC1_4_RGB) { tex = createCompressedTexture(dst, alignedWidth, alignedHeight, COMPRESSED.RGB_PVRTC_4BPPV1_IMG); } else if (format === BASIS_FORMAT.PVRTC1_4_RGBA) { tex = createCompressedTexture(dst, alignedWidth, alignedHeight, COMPRESSED.RGBA_PVRTC_4BPPV1_IMG); } else { // Create 565 texture. const dstTex = new Uint16Array(width * height); // Convert the array of bytes to an array of uint16's. var pix = 0; for (var y = 0; y < height; y++) for (var x = 0; x < width; x++, pix++) dstTex[pix] = dst[2 * pix + 0] | (dst[2 * pix + 1] << 8); tex = createRgb565Texture(dstTex, width, height); } return tex; }; }50:T516,fetchProgress = { function concat(arrays) { let i = 0, n = 0; for (const a of arrays) n += a.length; const concat = new Uint8Array(n); for (let a of arrays) concat.set(a, i), i += a.length; return concat.buffer; } const progressText = (cur, total) => { const max = total > 0 ? `${total / 1024 | 0}` : '?'; return `${cur / 1024 | 0} / ${max} kb`; }; return (name, url, init) => { const root = html`

${name} waiting...

`; const progress = root.children[1]; const text = root.children[2]; fetch(url, init).then(async response => { const max = response.headers.get("content-length"); progress.max = max || Number.MAX_SAFE_INTEGER; text.textContent = progressText(0, max); const reader = response.body.getReader(); const values = []; while (true) { const {done, value} = await reader.read(); if (done) break; progress.value += value.length; text.textContent = progressText(progress.value, max); values.push(value); } root.value = concat(values); root.dispatchEvent(new CustomEvent("input")); }); return root; }; }51:Tb04,class Camera { constructor(opts) { const { viewport, pos, target = v.ZERO3, up = v.Y3, fov = 45, near = 0.01, far = 50, } = opts; Object.defineProperties(this, { _viewport: { value: viewport, writable: true }, _radius: { value: Math.min(...viewport), writable: true }, _center: { value: v.mulN([], viewport, 0.5), writable: true }, _pos: { value: pos, writable: true }, _up: { value: v.copy(up), writable: true }, _target: { value: v.copy(target), writable: true }, // drag start _camUp: { value: [], writable: true }, _viewDir: { value: [], writable: true }, _down: { value: [], writable: true }, // drag _delta: { value: [], writable: true }, _side: { value: [], writable: true }, _axis: { value: [], writable: true }, _vd: { value: [], writable: true }, _rot: { value: [], writable: true }, }); this.view = m.lookAt([], this.pos, this.target, this.up); this.proj = m.perspective([], fov, viewport[0] / viewport[1], near, far); } _spherePos(out, x, y) { v.divN(null, v.setC3(out, x - this._center[0], -(y - this._center[1]), 0), this._radius); const mag = v.magSq(out); return mag > 1.0 ? v.normalize(null, out) : (out[2] = Math.sqrt(1.0 - mag), out); } _updateView() { m.lookAt(this.view, this.pos, this.target, this.up); return this; } get pos() { return this._pos; } set pos(val) { this._pos = val; return this._updateView(); } get target() { return this._target; } set target(val) { this._target = val; return this._updateView(); } get up() { return this._up; } set up(val) { this._up = val; return this._updateView(); } start(x, y) { this._spherePos(this._down, x, y); v.set(this._camUp, this.up); v.sub(this._viewDir, this.pos, this.target); return this; } drag(x, y) { v.sub(null, this._spherePos(this._delta, x, y), this._down); v.cross3(this._side, this._camUp, this._viewDir) v.mulN(null, v.normalize(null, this._side), this._delta[0]); v.mulN(this._axis, this._camUp, this._delta[1]); v.add(null, this._axis, this._side); v.cross3(null, this._axis, this._viewDir) v.normalize(null, this._axis); m.rotationAroundAxis33(this._rot, this._axis, v.magSq(this._delta) * 5); m.mulV(this.up, this._rot, this._camUp); v.add(this.pos, this.target, m.mulV(this._vd, this._rot, this._viewDir)); return this._updateView(); } zoom(delta, min = 0.1, max = 10.0) { v.sub(this._vd, this.target, this.pos); const mag = Math.max(min, Math.min(max, Math.pow(Math.E, delta) * v.mag(this._vd))); v.mulN(null, v.sub(this.pos, this.target, v.normalize(null, this._vd)), mag); return this._updateView(); } }52:T6be,class View { static create(initialize) { return new Promise(resolve => { const view = new View(); let resolved = false; const dispose = initialize(x => { view.value = x; if (!resolved) { resolved = true; resolve(Generators.disposable(view, () => { dispose && dispose(); view.dispose(); })); }; }) }); } static subs(unsubs) { return () => unsubs.forEach(unsub => unsub()); } constructor(value) { Object.defineProperties(this, { _value: { value, writable: true }, _list: { value: {}, writable: true }, _subs: { value: [], writable: true }, }); const dispatch = e => this._subs.forEach(sub => sub(e.value)); this.addEventListener('input', dispatch); this.dispose = () => this.removeEventListener("input", dispatch); }; get value() { return this._value; } set value(val) { this._value = val; this.notify(); } observe(sub) { this._subs.push(sub); sub(this.value); // give out the current value on subscription return () => this._subs = this._subs.filter(l => l !== sub); } notify() { this.dispatchEvent({ type: "input", value: this.value }); } addEventListener(type, listener) { if (!this._list[type]) this._list[type] = []; if (this._list[type].includes(listener)) return; this._list[type] = [listener].concat(this._list[type]); } removeEventListener(type, listener) { this._list[type] = this._list[type].filter(l => l !== listener); } dispatchEvent(event) { if (!this._list[event.type]) return; const p = Promise.resolve(event); this._list[event.type].forEach(l => p.then(l)); } }53:[{"id":"7314847946bc9733d95fa9f88e7f3ed6f03a7b29d1d7adc317b671720101306287f0654878039f75d4aa7d1ecc254b8af69763bfd5436780b9ab527f675257e9","href":"https://static.observableusercontent.com/files/7314847946bc9733d95fa9f88e7f3ed6f03a7b29d1d7adc317b671720101306287f0654878039f75d4aa7d1ecc254b8af69763bfd5436780b9ab527f675257e9","name":"embedding@1.drc","type":"application/octet-stream","lastModified":1738888257320.493,"size":325522},{"id":"23f330509d1eaa08f4cff62528ff4a2bb1ed42cf33d0d3b580c6e752b3f126e326ca99859da421f64a3c5124ac2d11032aa109bf475d269378fb3ac5e550400e","href":"https://static.observableusercontent.com/files/23f330509d1eaa08f4cff62528ff4a2bb1ed42cf33d0d3b580c6e752b3f126e326ca99859da421f64a3c5124ac2d11032aa109bf475d269378fb3ac5e550400e","name":"fashion_mnist_full@1.basis","type":"application/octet-stream","lastModified":1738888257320.493,"size":9458986},{"id":"99783ef5e2d8670f89eaf1b8f3ae703e04d8ead78572ac1b12590d3d3c0eb8585d3f30bf406150b337545019aa22deb3070f323d0b987e95eca7c6a8a5de2d88","href":"https://static.observableusercontent.com/files/99783ef5e2d8670f89eaf1b8f3ae703e04d8ead78572ac1b12590d3d3c0eb8585d3f30bf406150b337545019aa22deb3070f323d0b987e95eca7c6a8a5de2d88","name":"asset_flat.png","type":"image/png","lastModified":1738888257320.493,"size":359909},{"id":"0ff591ecaf96c676c84bf4a4bee11a01d61f90a46989435b32167e69ed414552489581e0db6e9fd1b695e29280e015c497b531d0cafba2c1811bf9ad42b26af5","href":"https://static.observableusercontent.com/files/0ff591ecaf96c676c84bf4a4bee11a01d61f90a46989435b32167e69ed414552489581e0db6e9fd1b695e29280e015c497b531d0cafba2c1811bf9ad42b26af5","name":"fashion_umap_3d.png","type":"image/png","lastModified":1738888257320.493,"size":140653}] 29:{"id":"0cb6d5fea5b76de9","slug":null,"title":"Exploring Fashion MNIST","thumbnail":"e58b2998837e37a7c2ba0ec38a65fe21089daa52d1f7f989cf9abbeaf06ca378","access_level":"public","create_time":"$D2025-02-07T00:30:57.320Z","update_time":"$D2025-02-07T00:30:57.320Z","view_time":null,"first_public_time":"$D2025-02-07T00:30:57.320Z","has_unpublished_edits":null,"fork":{"id":"54c6302d5ad5d228","slug":"exploring-fashion-mnist","title":"Exploring Fashion MNIST","owner":{"id":"7fad80bf128915e9","login":"stwind","name":"Zihou Ng"}},"stars":0,"starred":false,"owner":{"type":"team","id":"1dd8a7f5cbbe3892","login":"marlow","name":"marlow deneal","avatar_url":"https://avatars-next.observableusercontent.com/74c4b6c1dec20f433604141173d89ecbcc00b42c4df6539eb41e5efb2940bb7d","tier":"starter_2024","bio":"","followed_by_you":false,"follows_you":false},"version":1662,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"0fd8cc0ebc06ad87","login":"marlow","name":"marlow deneal","avatar_url":"https://avatars-next.observableusercontent.com/74c4b6c1dec20f433604141173d89ecbcc00b42c4df6539eb41e5efb2940bb7d","tier":null,"bio":"Traditional artist digital artist exploring the wonderful world of today through data visualization art ai exploring creating exploring creating more thank you d3 observable you help me to explore new possibilities 😀 ","followed_by_you":false,"follows_you":false}],"members":[{"type":"individual","id":"0fd8cc0ebc06ad87","login":"marlow","name":"marlow deneal","avatar_url":"https://avatars-next.observableusercontent.com/74c4b6c1dec20f433604141173d89ecbcc00b42c4df6539eb41e5efb2940bb7d","tier":null,"role":"editor"},{"type":"team","id":"1dd8a7f5cbbe3892","login":"marlow","name":"marlow deneal","avatar_url":"https://avatars-next.observableusercontent.com/74c4b6c1dec20f433604141173d89ecbcc00b42c4df6539eb41e5efb2940bb7d","tier":"starter_2024","role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Exploring Fashion MNIST","cells":[{"id":0,"value":"md`# Exploring Fashion MNIST`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":472,"value":"md`[Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist) is a MNIST-like fashion product dataset that contains 70,000 28x28 gray scale images each associated with a label from 10 classes. \n\nThis is an explorer of all the 70k images, clustered and embedded into 3 dimensions space using the [UMAP](https://umap-learn.readthedocs.io/en/latest/) algorithm. You can navigate by dragging the camera, clicking a specific image, jumping with an id or picking randomly.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1411,"value":"md`---`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":539,"value":"viewof dataEmbedding = fetchProgress(\"embedding\", await FileAttachment(\"embedding@1.drc\").url())","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":650,"value":"viewof dataTexture = fetchProgress(\"texture\", await FileAttachment(\"fashion_mnist_full@1.basis\").url())","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":36,"value":"viewof gl = {\n const canvas = DOM.canvas(width, width / 1.618);\n const gl = canvas.value = canvas.getContext('webgl');\n gl.getExtension(\"WEBGL_draw_buffers\");\n gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);\n gl.clearColor(0.2, 0.2, 0.2, 1);\n gl.clear(gl.COLOR_BUFFER_BIT);\n \n const root = html`

\n \n\n ${canvas}\n

`;\n root.value = gl;\n \n return root;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1061,"value":"$49","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1534,"value":"$4a","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":988,"value":"md`Now the problem is that the generated UMAP embedding are quite large, which is about 4.5 MB. \n\nThis is where Draco plays the important role. With Draco I compressed the point cloud to an impressive ${dataEmbedding.byteLength/1024|0} kb \\`embedding.drc\\` file.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1549,"value":"md`### Data Texture Compression with [Basis Universal](https://github.com/BinomialLLC/basis_universal)`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1544,"value":"$4b","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1540,"value":"md`### Rendering The Scene\n\nFinally it is time to render the scene using [thi.ng/umbrella](https://thi.ng/umbrella) and [D3](https://d3js.org/). The steps are quite straight forward, mainly involved the following:\n\n* Loading and decoding assets.\n* Draw the data as \\`gl.POINTS\\`.\n* In shaders\n * Calculate the texture offsets by point indices.\n * Colormapping based on data labels.\n * For far away points, fade to pure color.\n* Handle picking and camera transition.\n\n*All assets generation code is available in this [notebook](https://colab.research.google.com/gist/stwind/92ff73e0a599d5bc7dcdc9d15ad32abf/umap-fashion-mnist.ipynb).*`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":982,"value":"md`---\n## Code`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":42,"value":"draw = {\n gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);\n \n cluster.drawPick(camera);\n \n gl.clearColor(0.2, 0.2, 0.2, 1);\n gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);\n \n guide.draw(camera);\n cluster.draw(camera, hover);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":697,"value":"hover = Generators.observe(next => {\n const canvas = d3.select(gl.canvas);\n let cur = undefined;\n next(cur);\n \n const root = d3.select(viewof gl);\n const valId = root.select('.val-id');\n const valLabel = root.select('.val-label');\n \n canvas.on('mousemove', e => {\n const id = cluster.pick(e.offsetX, gl.drawingBufferHeight - e.offsetY);\n if (id != cur) {\n next(cur = id);\n if (cur !== undefined) {\n valId.text(id);\n valLabel.text(classnames[meta.labels[id]]);\n } else {\n valId.text(\"\");\n valLabel.text(\"\");\n }\n }\n });\n canvas.on(\"mouseout\", e => {\n next(cur = undefined);\n valId.text(\"\");\n valLabel.text(\"\");\n });\n\n return () => canvas.on('mousemove', null).on(\"mouseout\", null);\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":873,"value":"cluster = new Cluster(mesh, dataTexture)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":66,"value":"guide = new Guide()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1156,"value":"viewof camera = new class extends View {\n constructor() {\n super();\n this.camera = new Camera({\n viewport: [gl.drawingBufferWidth, gl.drawingBufferHeight],\n pos: [3.58453, -4.48565, 1.22442],\n far: 70,\n });\n }\n \n dragStart(x, y) {\n this.cancel && this.cancel();\n this.camera.start(x, y);\n }\n \n drag(x, y) {\n this.camera.drag(x, y);\n this.notify();\n }\n \n zoom(delta, min, max) {\n this.cancel && this.cancel();\n this.camera.zoom(delta, min, max);\n this.notify();\n }\n \n focus(to, speed, eps) {\n this.cancel && this.cancel();\n const from = v.copy(this.camera.pos);\n this.cancel = tween(t => {\n this.camera.pos = v.mixN(this.camera.pos, from, to, t);\n this.notify();\n }, speed, eps);\n }\n \n get value() {\n return this.camera;\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":152,"value":"mesh = decodeDracoGeometry(dataEmbedding, {\n position: Float32Array,\n label: Uint8Array,\n index: Uint32Array\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":770,"value":"meta = {\n const { position, index, label } = mesh;\n const positions = {}, labels = {};\n for (let i = 0; i < index.length; i++) {\n const pos = [position[i * 3], position[i * 3 + 1], position[i * 3 + 2]];\n positions[index[i]] = pos;\n labels[index[i]] = label[i];\n }\n return { positions, labels };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":943,"value":"classnames = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat','Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":913,"value":"initialized = {\n dataEmbedding;\n dataTexture;\n \n const root = d3.select(viewof gl);\n root.select(\".title\").style(\"display\", \"none\");\n root.select(\".info\").style(\"display\", \"block\");\n \n // initialize camera\n const cam = viewof camera;\n cam.focus(v.rotateY(null, v.rotateX(null, [0,0,30], -Math.PI/10), Math.PI/4), 0.025);\n \n const canvas = d3.select(gl.canvas);\n canvas.call(d3.drag().on('start', e => {\n cam.dragStart(e.x, e.y);\n e.on('drag', e => cam.drag(e.x, e.y));\n }));\n canvas.on('wheel', e => {\n e.preventDefault();\n cam.zoom(e.deltaY * 1e-3, 0.01, 50);\n });\n \n canvas.on('click', e => {\n const id = cluster.pick(e.offsetX, gl.drawingBufferHeight - e.offsetY);\n if (id) {\n let pos = meta.positions[id];\n pos = v.mulN(null, v.normalize([], pos), v.mag(pos) + 0.25)\n cam.focus(pos, 0.1);\n }\n });\n \n invalidation.then(() => {\n canvas.on(\"drag\", null).on(\"wheel\", null).on(\"click\", null);\n });\n \n return true;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":710,"value":"md`### Components`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":844,"value":"$4c","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":713,"value":"$4d","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":30,"value":"md`### Utils`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":662,"value":"ID = ({\n encode: val => {\n const r = ( val >> 16 & 0xff ) / 255;\n const g = ( val >> 8 & 0xff ) / 255;\n const b = ( val & 0xff ) / 255;\n return [r, g, b];\n },\n decode: ([r, g, b]) => (r << 16) | (g << 8) | b\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":148,"value":"$4e","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":228,"value":"$4f","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":515,"value":"makeCmapTexture = {\n const toRGB = c => {\n const { r, g, b } = d3.rgb(c);\n return [r, g, b];\n };\n \n return colors => webgl.defTexture(gl, {\n width: colors.length, height: 1,\n format: gl.RGB,\n image: new Uint8Array(colors.map(toRGB).flat(1))\n });\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":544,"value":"$50","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":33,"value":"$51","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1053,"value":"$52","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":728,"value":"tween = (f, speed = 0.08, eps = 1e-4) => {\n let t = 0;\n let handle = requestAnimationFrame(function frame() {\n f(t);\n t += (1 - t) * speed;\n if (Math.abs(1 - t) > eps) {\n handle = requestAnimationFrame(frame);\n }\n });\n\n return () => cancelAnimationFrame(handle);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":319,"value":"fps = {\n let t = performance.now(), delta = 0;\n const last = [t];\n return cell => {\n last.length >= 10 && last.shift();\n t = performance.now();\n delta = (t - last[0]) / last.length;\n last.push(t);\n return Math.round(1000 / delta);\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":13,"value":"md`### Appendix`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":19,"value":"webgl = require('https://bundle.run/@thi.ng/webgl@3.3.3')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":21,"value":"m = require('https://bundle.run/@thi.ng/matrices@0.6.33')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":23,"value":"v = require('https://bundle.run/@thi.ng/vectors@4.6.6')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":118,"value":"draco = {\n const PREFIX = \"https://www.gstatic.com/draco/v1/decoders\";\n const wasmBinary = await fetch(`${PREFIX}/draco_decoder.wasm`).then(resp => {\n if (!resp.ok)\n throw new Error(\"HTTP error, status = \" + resp.status);\n return resp.arrayBuffer();\n });\n \n const mod = await require(`${PREFIX}/draco_wasm_wrapper.js`);\n return new Promise(resolve => mod({ wasmBinary }).then(draco => {\n delete draco.then; // tell Observable not wait for this;\n resolve(draco);\n }));\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":225,"value":"BasisFile = {\n const BASIS = await require('three@0.120.1/examples/js/libs/basis/basis_transcoder.js');\n return new Promise(resolve => BASIS().then(transcoder => {\n transcoder.initializeBasis();\n resolve(transcoder.BasisFile);\n }));\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":3,"value":"md`---\n## Bibliography\n\n* [GrantCuster/umap-explorer: An interactive UMAP visualization of the MNIST data set.](https://github.com/GrantCuster/umap-explorer)\n* [YaleDHLab/pix-plot: A WebGL viewer for UMAP or TSNE-clustered images](https://github.com/YaleDHLab/pix-plot)\n* [Tensorflow Embedding Projector](https://projector.tensorflow.org/)\n* [Google Arts & Culture Experiments - t-SNE Map Experiment](https://artsexperiments.withgoogle.com/tsnemap/)\n* [The Infinite Drum Machine by Manny Tan & Kyle McDonald | Experiments with Google](https://experiments.withgoogle.com/drum-machine)\n* [Visualizing an art collection | Audun M Øygard](https://www.auduno.com/2018/10/27/visualizing-an-art-collection/)\n* [Visualizing High-Dimensional Space by Daniel Smilkov, Fernanda Viégas, Martin Wattenberg & the Big Picture team at Google | Experiments with Google](https://experiments.withgoogle.com/visualizing-high-dimensional-space)\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W53","stdlib":"1"}} 24:null