25:["$","title",null,{"children":"Crypto Analysis Unscented Kalman Filter | Henrique Recidive | Observable"}] 49:T443,trades = { const start = timeRange.start.getTime() const end = timeRange.end.getTime() const firstTradeId = (await ( // https://binance-docs.github.io/apidocs/spot/en/#compressed-aggregate-trades-list await fetch(`https://api.binance.com/api/v3/aggTrades?symbol=${symbol}&startTime=${start}&endTime=${start + 3600000}&limit=1`) ).json())[0].a let result = new Map() let lastTradeId = firstTradeId infinitLoop: while (true) { // https://binance-docs.github.io/apidocs/spot/en/#compressed-aggregate-trades-list const endpoint = `https://api.binance.com/api/v3/aggTrades?symbol=${symbol}&fromId=${lastTradeId}&limit=1000` const response = await fetch(endpoint) const tradesPage = await response.json() if (tradesPage.length === 0) break for (const trade of tradesPage) { if (trade.T > end) break infinitLoop result.set(trade.a, trade) lastTradeId = trade.a } } return Array.from(result.values()).sort((a, b) => a.T - b.T).map(t => ({ timestamp: t.T, price: parseFloat(t.p), volume: parseFloat(t.q), })) }4a:T5ae,function resampleTradesData(tradesData) { const startTime = tradesData[0].timestamp; const endTime = tradesData[tradesData.length - 1].timestamp; const numIntervals = Math.ceil((endTime - startTime) / 1000); const resampledData = new Array(numIntervals); let intervalStartIndex = 0; let lastPrice = null; for (let i = 0; i < numIntervals; i++) { const intervalStartTime = startTime + i * 1000; const intervalEndTime = intervalStartTime + 1000; let sumPriceVolume = 0; let sumVolume = 0; let numTrades = 0; // Iterate over trades in the current interval and calculate volume-weighted average price while (intervalStartIndex < tradesData.length && tradesData[intervalStartIndex].timestamp < intervalEndTime) { const trade = tradesData[intervalStartIndex]; const tradeWeight = trade.volume * trade.price; sumPriceVolume += tradeWeight; sumVolume += trade.volume; numTrades++; lastPrice = trade.price; intervalStartIndex++; } if (sumVolume > 0) { const intervalPrice = sumPriceVolume / sumVolume; resampledData[i] = { timestamp: intervalStartTime, price: intervalPrice, volume: sumVolume }; } else { // No trades in this interval, fill in the last price resampledData[i] = { timestamp: intervalStartTime, price: lastPrice, volume: 0 }; } } return resampledData; } 4b:T1398,class UnscentedKalmanFilter { constructor(n, initialState, initialCovariance, alpha, beta, kappa) { this.n = n this.x = initialState this.P = initialCovariance this.alpha = alpha this.beta = beta this.kappa = kappa const lambda = Math.pow(this.alpha, 2) * (this.n + this.kappa) - this.n; this.Wm = tf.concat( [ tf.scalar(lambda / (this.n + lambda)).reshape([1, 1]), tf.ones([1, 2 * this.n]).mul(1 / (2 * (this.n + lambda))), ], 1, ); this.Wc = this.Wm.clone().add( tf.scalar(this.beta - 1).mul(tf.square(this.alpha)), ); } sigmaPoints() { const lambda = Math.pow(this.alpha, 2) * (this.n + this.kappa) - this.n // Calculate square root of (n + lambda) * P const sqrtScaledP = this.choleskyDecomposition(tf.mul(this.P, this.n + lambda)) // Calculate positive and negative sigma points const positiveSigmaPoints = tf.add(this.x, sqrtScaledP) const negativeSigmaPoints = tf.sub(this.x, sqrtScaledP) // Concatenate mean state, positive sigma points, and negative sigma points const sigmaPoints = tf.concat([this.x, positiveSigmaPoints, negativeSigmaPoints], 1) return sigmaPoints } unscentedTransform(sigmaPoints, Wm, Wc) { // Calculate mean const mean = tf.sum(tf.mul(sigmaPoints, Wm), 1, true); // Calculate covariance const residual = tf.sub(sigmaPoints, tf.expandDims(mean, 1)); const weightedResiduals = tf.mul(residual, tf.sqrt(Wc)); const covariance = tf.matMul(weightedResiduals, weightedResiduals.transpose()); return [mean, covariance]; } predict(f, Q) { const lambda = Math.pow(this.alpha, 2) * (this.n + this.kappa) - this.n const sigmaPoints = this.sigmaPoints() // Predict sigma points const predictedSigmaPoints = f(sigmaPoints) // Perform unscented transform const [predictedMean, predictedCovariance] = this.unscentedTransform( predictedSigmaPoints, this.Wm, this.Wc ) // Add process noise this.x = predictedMean this.P = tf.add(predictedCovariance, Q) } update(h, z, R) { const lambda = Math.pow(this.alpha, 2) * (this.n + this.kappa) - this.n const sigmaPoints = this.sigmaPoints() // Transform sigma points into measurement space const predictedMeasurements = h(sigmaPoints) // Perform unscented transform in measurement space const [predictedMean, predictedCovariance] = this.unscentedTransform( predictedMeasurements, this.Wm, this.Wc ) // Add measurement noise const predictedCovarianceWithNoise = tf.add(predictedCovariance, R) // Calculate cross-covariance between state and measurement const residualState = tf.sub(sigmaPoints, this.x) const residualMeasurement = tf.sub(predictedMeasurements, predictedMean) const crossCovariance = tf.matMul(tf.mul(residualState, this.Wc), residualMeasurement.transpose()) // Calculate Kalman gain const K = tf.matMul( crossCovariance, this.matrixInverse(predictedCovarianceWithNoise) ) // Update state and covariance const measurementResidual = tf.sub(z, predictedMean) this.x = tf.add(this.x, tf.matMul(K, measurementResidual)) this.P = tf.sub( this.P, tf.matMul(tf.matMul(K, predictedCovarianceWithNoise), K.transpose()) ) } choleskyDecomposition(A) { const n = A.shape[0] const L = tf.buffer([n, n], 'float32') const Abuf = A.bufferSync() for (let i = 0; i < n; i++) { Abuf.set(Abuf.get(i, i) + 1e-6, i, i) // Add a small positive value to the diagonal for (let j = 0; j < i + 1; j++) { let sum = 0 for (let k = 0; k < j; k++) { sum += L.get(i, k) * L.get(j, k) } if (i === j) { L.set(Math.sqrt(Abuf.get(i, i) - sum), i, j) } else { L.set((1.0 / L.get(j, j)) * (Abuf.get(i, j) - sum), i, j) } } } return L.toTensor() } matrixInverse(A) { const n = A.shape[0] const AI = tf.concat([A, tf.eye(n)], 1) const rref = tf.buffer(AI.shape, "float32") const AIArray = AI.arraySync() for (let i = 0; i < n; i++) { let maxIndex = i for (let j = i + 1; j < n; j++) { if (Math.abs(AIArray[j][i]) > Math.abs(AIArray[maxIndex][i])) { maxIndex = j } } if (maxIndex !== i) { ;[AIArray[i], AIArray[maxIndex]] = [AIArray[maxIndex], AIArray[i]] } const pivot = AIArray[i][i] for (let j = i; j < 2 * n; j++) { AIArray[i][j] /= pivot } for (let j = 0; j < n; j++) { if (j !== i) { const factor = AIArray[j][i] for (let k = i; k < 2 * n; k++) { AIArray[j][k] -= factor * AIArray[i][k] } } } } for (let i = 0; i < n; i++) { for (let j = 0; j < 2 * n; j++) { rref.set(AIArray[i][j], i, j) } } return rref.toTensor().slice([0, n], [n, n]) } }4c:T666,filteredStates = { const data = resampledTrades.slice(-300) // Define the state transition function function f(x) { const dt = 1 // Time step const F = tf.tensor([ [1, dt, 0.5 * dt * dt], [0, 1, dt], [0, 0, 1] ]) return tf.matMul(F, x) } // Define the measurement function function h(x) { const H = tf.tensor([[1, 0, 0]]) return tf.matMul(H, x) } // Define the initial state (position, velocity, and acceleration) const initialState = tf.tensor([[data[0].normalizedPrice], [0.1], [0.1]]) // Define the initial covariance matrix // const initialCovariance = tf.eye(3).mul(10) const initialCovariance = tf.tensor([ [10, 0, 0], [0, 10, 0], [0, 0, 10], ]); // Create the Unscented Kalman Filter const ukf = new UnscentedKalmanFilter( 3, initialState, initialCovariance, 1e-3, 1, 0 ) // Define the process noise covariance matrix (Q) const q = tf.tensor([ [0.1, 0, 0], [0, 0.1, 0], [0, 0, 0.1] ]) // Define the measurement noise covariance matrix (R) const r = tf.tensor([[1]]) const results = [] data.forEach((trade) => { // Predict the next state and covariance ukf.predict(f, q) // Update the state and covariance with the new measurement const z = tf.tensor([[trade.normalizedPrice]]) ukf.update(h, z, r) const result = ukf.x.arraySync() // Log the estimated state results.push({ ...trade, ...result, position: result[0][0], velocity: result[1][0], acceleration: result[2][0], }) }) return results }4d:[] 29:{"id":"b513da87086bea5b","slug":null,"title":"Crypto Analysis Unscented Kalman Filter","thumbnail":"c947dd983d4ef893827dfac222f2848f85ccf441a930ae86b7f1ab8355701e23","access_level":"public","create_time":"$D2023-03-15T17:55:40.589Z","update_time":"$D2023-03-16T14:11:55.226Z","view_time":null,"first_public_time":"$D2023-03-15T17:55:40.589Z","has_unpublished_edits":null,"fork":{"id":"aedfa7d20d28af6c","slug":null,"title":"Crypto Analysis log returns/kalman filter","owner":{"id":"4f214c0cf299c92f","login":"recidive","name":"Henrique Recidive"}},"stars":0,"starred":false,"owner":{"type":"team","id":"4f214c0cf299c92f","login":"recidive","name":"Henrique Recidive","avatar_url":"https://avatars-next.observableusercontent.com/9e4d20eba1a2abfb49bd7d421160379fa6253cdd9e2ef77a003fa7f0db749e69","tier":"starter_2024","bio":"","following":false},"version":828,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"6c0f68e32d904202","login":"recidive","name":"Henrique Recidive","avatar_url":"https://avatars-next.observableusercontent.com/9e4d20eba1a2abfb49bd7d421160379fa6253cdd9e2ef77a003fa7f0db749e69","tier":null,"bio":"","following":false}],"members":[{"type":"individual","id":"6c0f68e32d904202","login":"recidive","name":"Henrique Recidive","avatar_url":"https://avatars-next.observableusercontent.com/9e4d20eba1a2abfb49bd7d421160379fa6253cdd9e2ef77a003fa7f0db749e69","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Crypto Analysis Unscented Kalman Filter","cells":[{"id":0,"value":"# Crypto Analysis Unscented Kalman Filter","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":512,"value":"symbol = 'INJUSDT'","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":515,"value":"timeRange = ({\n start: new Date('2023-03-13T18:50:00Z'),\n end: new Date('2023-03-13T19:10:00Z'),\n})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"$49","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":40,"value":"class CryptoPriceIndicator {\n constructor() {\n this.previousPrice = null;\n this.normalizedPrice = 100;\n }\n\n update(price) {\n let logReturn\n if (this.previousPrice !== null) {\n logReturn = Math.log(price / this.previousPrice);\n this.normalizedPrice *= Math.exp(logReturn);\n }\n\n this.previousPrice = price;\n return {\n normalizedPrice: Math.round(this.normalizedPrice * 10),\n logReturn: logReturn || 0,\n }\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":799,"value":"class CryptoPriceIndicatorAlt {\n constructor() {\n this.initialPrice = null;\n }\n\n update(price) {\n if (!this.initialPrice) {\n this.initialPrice = price;\n return {\n normalizedPrice: 1\n }\n }\n\n const returns = price / this.initialPrice\n return {\n normalizedPrice: returns\n }\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":4,"value":"$4a","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":18,"value":"resampledTradesData = resampleTradesData(trades).map(t => ({...t, timestamp: new Date(t.timestamp)}))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":51,"value":"resampledTrades = {\n const normalizer = new CryptoPriceIndicatorAlt();\n return resampledTradesData.map(t => {\n const normalizerResult = normalizer.update(t.price)\n return {\n ...t,\n ...normalizerResult,\n }\n })\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":345,"value":"secondsToPlot = 300","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":20,"value":"Plot.plot({\n marks: [\n Plot.lineY(resampledTrades.slice(-secondsToPlot), {x: 'timestamp', y: 'normalizedPrice'})\n ],\n grid: true,\n style: {\n width: 1024,\n }\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":762,"value":"tf = require('@tensorflow/tfjs@4.2.0')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":821,"value":"tf.setBackend('cpu');","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":760,"value":"$4b","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":85,"value":"$4c","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":88,"value":"Plot.plot({\n marks: [\n // Plot.lineY(filteredStates.slice(-secondsToPlot), {x: 'timestamp', y: 'price', stroke: 'grey'}),\n Plot.lineY(filteredStates, {x: 'timestamp', y: 'position', stroke: 'purple'}),\n // Plot.lineY(filteredStates.slice(-100), {x: 'timestamp', y: 'price', stroke: 'lightgrey'}),\n // Plot.dot(filteredStates.slice(499).slice(-300), {x: 'timestamp', y: 'predictedPrice', stroke: 'grey'}),\n ],\n grid: true,\n style: {\n width: 1024,\n }\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":96,"value":"Plot.plot({\n marks: [\n Plot.lineY(filteredStates.slice(-secondsToPlot), {x: 'timestamp', y: 'velocity', stroke: 'green'}),\n Plot.lineY(filteredStates.slice(-secondsToPlot), {x: 'timestamp', y: 'acceleration', stroke: 'red'}),\n // Plot.lineY(filteredStates.slice(60, 240), {x: 'timestamp', y: 'predictedVelocity', stroke: 'lightgreen'}),\n // Plot.lineY(filteredStates.slice(60, 240), {x: 'timestamp', y: 'predictedAcceleration', stroke: 'pink'}),\n ],\n grid: true,\n style: {\n width: 1024,\n }\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":334,"value":"kineticEnergy = {\n function calculateKineticEnergy(mass, velocity) {\n // Calculate kinetic energy using the formula KE = 0.5 * m * v^2\n const kineticEnergy = 0.5 * mass * velocity ** 2;\n \n return kineticEnergy;\n }\n\n return filteredStates.map(t => ({\n ...t,\n kineticEnergy: calculateKineticEnergy(t.volume, t.velocity)\n }))\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":341,"value":"Plot.plot({\n marks: [\n Plot.lineY(kineticEnergy.slice(-secondsToPlot), {x: 'timestamp', y: 'velocity', stroke: 'green'}),\n Plot.lineY(kineticEnergy.slice(-secondsToPlot), {x: 'timestamp', y: 'kineticEnergy', stroke: 'blue'}),\n ],\n grid: true,\n style: {\n width: 1024,\n }\n})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W4d","stdlib":"1"}} 24:null