// adapted from https://github.com/milhidaka/chainer-image-caption/blob/master/webdnn/index.js // licensed MIT { const baseUrl = 'https://milhidaka.github.io/chainer-image-caption/'; let cap_gen; async function run() { runButton.innerText = 'Captioning...'; runButton.disabled = true; try { await run_generation(); } catch (ex) { alert('Failed: ' + ex); throw ex; } finally { runButton.innerText = 'Generate caption'; runButton.disabled = false; } } function loadLocalImage(e) { let file = this.files[0]; if (file) { let image_blob = window.URL.createObjectURL(file); sample_image.src = image_blob; } } async function run_generation() { console.log('start running'); let image_data = await WebDNN.Image.getImageArray(canvas, { order: WebDNN.Image.Order.CHW, color: WebDNN.Image.Color.BGR, bias: [123.68, 116.779, 103.939] }); let sentences = await cap_gen.generate_caption(image_data); sentencesEl.textContent = sentences.join('\n'); } async function load_models() { let word_data = await (await fetch( `${baseUrl}image-caption-model/word_data.json` )).json(); let runner_image = await WebDNN.load( `${baseUrl}image-caption-model/image-feature`, { backendOrder: ["webgpu", "webassembly"] } ); let runner_caption = await WebDNN.load( `${baseUrl}image-caption-model/caption-generation`, { backendOrder: ["webassembly"] } ); cap_gen = new ImageCaptionGenerator( runner_image, runner_caption, word_data ); } // based on http://phiary.me/html5-canvas-drag-and-drop-image-draw/ // allow drag-and-drop image file on canvas const canvas = html`

`; const runButton = html``; const loadLocalImageInput = html``; const progressItems = [ html`Load models`, html`Analyze image`, html`Generate text` ]; const sentencesEl = html`

Generated caption will be shown here.

`; const ui = html`

Input image (can drag-drop image file):

${canvas}
${loadLocalImageInput}
${runButton}
${sentencesEl}

`; yield ui; var ctx = canvas.getContext('2d'); var render = function(image) { ctx.drawImage(image, 0, 0, 224, 224); }; var cancelEvent = function(e) { e.preventDefault(); e.stopPropagation(); return false; }; document.addEventListener("dragover", cancelEvent, false); document.addEventListener("dragenter", cancelEvent, false); document.addEventListener( "drop", function(e) { e.preventDefault(); var file = e.dataTransfer.files[0]; var image = new Image(); image.onload = function() { render(this); }; var reader = new FileReader(); reader.onload = function(e) { image.src = e.target.result; }; reader.readAsDataURL(file); }, false ); // show initial sample image var sample_image = new Image(); sample_image.onload = function() { ctx.drawImage(sample_image, 0, 0, 224, 224); }; sample_image.crossOrigin = "Anonymous"; sample_image.src = `${baseUrl}asakusa.jpg`; runButton.disabled = true; runButton.innerText = 'Loading...'; try { await load_models(); runButton.disabled = false; runButton.innerText = 'Generate caption'; } catch { runButton.innerText = 'Failed to load'; } }