The avatar for @kimmolinna
Kimmo Linna
Public
Edited
Jan 26, 2023
Fork of DuckDB - Parquet and Scaleway Object Storage
•
1 fork
Importers
4 stars
Insert cell
Insert cell
Insert cell
region = 'eu-north-1'
Insert cell
Insert cell
access_key_id='AKIAST5RBRTIXT47I6O7'
Insert cell
secret_access_key='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'
Insert cell
Insert cell
s3_duckdb = {
const con = await DuckDBClient.of();
await con.query(`SET s3_region='${region}'`);
await con.query(`SET s3_access_key_id='${access_key_id}'`);
await con.query(`SET s3_secret_access_key='${secret_access_key}'`);
return con
}
Insert cell
bucket = 'linna'
Insert cell
folder = "parquet"
Insert cell
s3_duckdb
PRAGMA version
Insert cell
Insert cell
s3_duckdb
SELECT
count(*) AS n
FROM read_parquet(${`s3://${bucket}/${folder}/electricity_2018.parquet`})
Insert cell
Insert cell
s3 = await new AWS.S3Client({
region: region,
credentials:{
accessKeyId: access_key_id,
secretAccessKey: secret_access_key}
});
Insert cell
files = (await s3.send(await new AWS.ListObjectsV2Command({Bucket: bucket, Prefix: 'parquet', MaxKeys: 100}))).Contents.slice(1).map(d=>"s3://"+bucket+"/"+d.Key)
Insert cell
file = await s3.send(new AWS.GetObjectCommand({Bucket: bucket, Key: 'parquet/electricity_2022.parquet'}))
Insert cell
s3_duckdb
SELECT
count(*) as N
FROM read_parquet(str_split(${files.join(",")},','))
Insert cell
AWS = await import("https://cdn.skypack.dev/@aws-sdk/client-s3@3.212")
Insert cell

Purpose-built for displays of data

Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
Learn more