Published
Edited
Jun 10, 2022
Importers
1 star
function newCorsFetch(proxyUrl) {
// See https://developer.mozilla.org/en-US/docs/Web/API/Request/cache
return async function* (
method,
{ url, login, password, body, onResponse, force } = {}
) {
const headers = new Headers();
if (login) {
headers.set("Authorization", `Basic ${btoa(`${login}:${password}`)}`);
}
headers.set("Access-Control-Allow-Origin", "*");
headers.set("Access-Control-Allow-Credentials", "true");
const fullUrl = `${proxyUrl}${encodeURIComponent(url)}${
force ? "?force=true" : ""
}`;
yield* fetchContent(fullUrl, {
// mode: "cors",
// credentials: "include",
method,
headers,
body,
onResponse
});
};
}
async function* fetchContent(url, { onResponse, ...options } = {}) {
const controller = new AbortController();
try {
const response = await fetch(url, {
...options,
signal: controller.signal
});
let done, value;
const reader = await response.body.getReader();
onResponse && onResponse(response);
while (({ done, value } = await reader.read())) {
if (done) break;
yield value;
}
} finally {
controller.abort();
}
}
async function fileExists(url, options = {}) {
let result = false;
for await (let block of fetchContent(url, {
mode: "cors",
...options,
method: "GET",
onResponse: (res) => (result = res.ok)
})) {
break;
}
return result;
}
async function loadTextFile(url) {
const res = await fetch(url);
return await res.text();
}
async function loadDataFile(url) {
const res = await fetch(url, { method: "GET", mode: "cors" });
return res.json();
}
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
function joinUrlSegments(...segments) {
return segments
.map((str) => String(str).replace(/^\.\//, "").replace(/\/$/gim, ""))
.join("/");
}
function toBlob(content) {
if (
typeof content === "object" &&
!(content instanceof Blob) &&
!(content instanceof ArrayBuffer) &&
!(content.buffer && content.buffer instanceof ArrayBuffer)
) {
content = JSON.stringify(content);
}
if (typeof content === "string") {
content = new TextEncoder().encode(content);
}
if (!(content instanceof Blob)) {
content = new Blob(content ? [content] : []);
}
return content;
}
async function sendFiles(url, files, options = {}) {
url = url.replace(/\/$/gim, "") + "/";
const formData = new FormData();
const paths = {};
let idx = 0;
for await (const file of files) {
let content = file.content;
content = toBlob(typeof content === "function" ? await content() : content);
const fileKey = `file-${idx++}`;
paths[fileKey] = file.path;
const f = new File([content], fileKey);
formData.append("files", f);
}
formData.append("paths", JSON.stringify(paths));
const headers = new Headers();
if (options && options.username) {
headers.set(
"Authorization",
"Basic " + btoa(options.username + ":" + (options.password || ""))
);
}
const response = await fetch(url, {
method: "POST",
body: formData,
headers
});
const result = await response.json();
const map = new Map();
if (result.files) {
const list = result.files;
for (let i = 0; i < list.length; i++) {
const filePath = list[i];
const fileUrl = joinUrlSegments(url, filePath);
map.set(filePath, fileUrl);
}
}
return map;
}
async function* loadTarFiles(url) {
const res = await fetch(url);
const buf = await res.arrayBuffer();
yield* unpackTarFile(buf);
}
async function* unpackTarFile(buf) {
const inflated = await pako.inflate(buf);
yield* await untar(inflated.buffer);
}
pako = import("pako")
untar = require("js-untar")
asyncUtils = {
// https://github.com/mkotelnikov/agen/blob/master/packages/agen-utils/src/lines.js
/**
* Transforms sequence of strings (containing '\r' and '\n' symbols) to a sequence
* of individual lines.
*/
async function* lines(provider) {
let buffer;
for await (let chunk of provider) {
buffer = buffer || "";
buffer += chunk;
const lines = buffer.split(/\r?\n/);
buffer = lines.pop();
yield* lines;
// while (lines.length) yield lines.shift();
}
if (buffer !== undefined) yield buffer;
}
// See https://github.com/mkotelnikov/agen/blob/master/packages/agen-utils/src/decoder.js
/**
* Transforms sequence of buffers or int arrays to a sequence of decoded strings.
* @param provider async generator providing buffers
* @param {String} enc encoding for the text
* @return an async generator providing decoded strings
*/
async function* decoder(provider, enc = "UTF-8") {
const decoder = new TextDecoder(enc);
for await (let chunk of provider) {
yield decoder.decode(chunk, { stream: true });
}
const lastChunk = decoder.decode();
if (lastChunk) yield lastChunk; // finish the stream
}
// See https://github.com/mkotelnikov/agen/blob/master/packages/agen-utils/src/encoder.js
/**
* Transforms sequence of strings to UTF-8 encoded int arrays / buffers.
* @param provider async generator providing strings
* @return an async generator providing decoded strings
*/
async function* encoder(provider) {
const encoder = new TextEncoder("UTF-8");
for await (let chunk of provider) {
yield encoder.encode(chunk);
}
}
return { lines, encoder, decoder };
}
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
