Public
Edited
Oct 18, 2023
sdk = await import("https://aaronge-2020.github.io/JSTransformer/sdk.js")
tf = await import("https://esm.sh/@tensorflow/tfjs@4.10.0");
tokenizer = await(await fetch("https://aaronge-2020.github.io/JSTransformer/tokenizer.json")).json()
function translate(sentence, tokenizers, transformer, MAX_TOKENS) {
// Tokenize the English input sentence
const encoderInput = sdk.wordsToIntTokens(sentence,tokenizers.English, MAX_TOKENS);
// console.log(encoderInput);
// Initialize the Spanish output with the [START] token
const startEnd = [1,2];
const start = startEnd[0];
const end = startEnd[1];
let outputArray = [start].concat(Array(MAX_TOKENS-1).fill(0));
for (let i = 0; i < MAX_TOKENS; i++) {
// Prepare encoder and decoder inputs
const encoderInputTensor = tf.tensor([encoderInput]);
const outputTensor = tf.tensor([outputArray]);
// Get predictions
const predictions = transformer.predict([encoderInputTensor, outputTensor]);
// console.log(predictions.shape);
const lastPrediction = predictions.slice([0, i, 0], [1, 1, predictions.shape[2]]);
// Get the ID of the predicted token
const predictedId = lastPrediction.argMax(-1).dataSync()[0];
console.log(predictedId);
// Replace the placeholder 0 with the predicted token to output
outputArray[i+1] = predictedId;
// Check for [END] token
if (predictedId === end) {
break;
}
}
// Detokenize the output to get the translated sentence
const translatedSentence = sdk.detokenizeSentence(outputArray, tokenizers.Spanish);
return translatedSentence;
}
model = tf.loadLayersModel("https://aaronge-2020.github.io/JSTransformer/my-model-epoch-70-batch-40.json")
model.summary()
translate("We'll be home by sunset.", tokenizer, model, 13)
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
