Partner and Head of Research at @data-desk
Public
Edited
Dec 1, 2021
Importers
6 stars
cache = new SQLiteHTTPCache
cache.load(FileAttachment("SQLiteHTTPCache@2.db.gz"))
cache
.fetch("https://httpbin.org/get")
.then(r => r.status)
cache
.fetch(
"https://httpbin.org/post",
{ method: "POST", body: JSON.stringify({ "testing": true }) })
.then(r => r.status)
DOM.download(async () => { return await cache.export()} , "SQLiteHTTPCache.db.gz")
class SQLiteHTTPCache {
constructor(db) {
this.cache = SQLiteDatabaseClient.open(new Blob());
}
async load(file) {
const buffer = await file.arrayBuffer();
const array = new Uint8Array(buffer);
const db = fflate.gunzipSync(array);
this.cache = await SQLiteDatabaseClient.open(db);
const cache = this.cache;
const requests = await cache.queryRow(`select count(*) as count from cache`);
return md`*Loaded ${requests.count} requests from cache*`;
}
async export() {
const cache = await this.cache;
const exported = await cache._db.export();
const gzipped = fflate.gzipSync(exported);
return new Blob([gzipped], {type: "application/gzip"});
}
async fetch(...a) {
const key = this.key(new Request(...a));
let cache = await this.cache;
await cache.query(`create table if not exists cache(id, data)`);
await cache.query(`create index if not exists cache_id_idx on cache(id)`);
this.cache = cache;
const result = await cache.queryRow(`select * from cache where id = ?`, [key]);
if (result) {
console.log(`Getting ${[...a][0]} from cache`);
const response = JSON.parse(result["data"]);
return new Response(response.body, response.init);
} else {
const response = await fetch(...a);
if (response.ok) {
const text = await response.clone().text();
const headers = Object.fromEntries(response.headers.entries());
const data = JSON.stringify({
body: text,
init: {
ok: response.ok,
status: response.status,
headers: this.sanitise_headers(headers)
}
});
await cache.query(`insert into cache(id, data) values (?, ?)`, [key, data]);
this.cache = cache;
}
return response;
}
}
key(request) {
let headers = Object.fromEntries(request.headers.entries());
return hash.sha1({
url: request.url,
mode: request.mode,
method: request.method,
headers: this.sanitise_headers(headers),
body: request.body
});
}
sanitise_headers(headers) {
delete headers["User-Agent"];
delete headers["Authorization"];
delete headers["X-Amzn-Trace-Id"];
return headers;
}
}
hash = import("https://cdn.skypack.dev/object-hash@2.2.0?min")
fflate = import("https://cdn.skypack.dev/fflate@0.7.1?min")
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
