Pack and unpack tabular data
data
packed = pack(data)
initial = unpack(packed)
data1 = d3.csv( "https://raw.githubusercontent.com/bumbeishvili/sample-data/main/org.csv")
pack(data1)
data
sizeFriendly = d3.csvFormat(replaced)
v = pack(data, { drop: ["employee_id"] })
unpack(v)
[{ "": 1 }]
Object.keys({ "": 1 })
class NumberPacker {
static symbols =
"0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz+ԱԲԳԴԵԶԷԸԹԺԻԼԽԾԿՀՁՂՃՄՅՆՇՈՉՊՋՌՍՎՏՐՑՒՓՔևՕაბგდევზთიკლმნოპჟრსტუფქღყშჩცძწჭხჯჰабвгдеёжзийклмноГМНО\tпрстуфхцчшщъыьэюяПРСХЦЯΑΒΓΔΕΖΗΘΙΚΛАБВДЕЖЗИКЛТУФЧШΜΝΞΟΠΡΣΤΥΦΧΨΩαβγδεζηθικλμνξοπρστυφχψωΆΈΉΊΌΎΏάέήίόύώ一二三四五六七八九十百千万億あいうえおかきくけこさしすせそたちつてとなにぬねのはひふへほまみむめもやゆよらりるれろわをんアイウエオカキクケコサシスセソタチツテトナニヌネノハヒフヘホマミムメモヤユヨラリルレロワヲンঅ আকািীউুঊূঋৃএেঐৈওোঔৌ্তংঃঁখগঘঙচছজঝঞটঠডঢণথদধনপফবভমযরৰলৱশষসহ়০১২৩৪৫৬৭৮৯અઆઇઈઉઊઋઌઍએઐઑઓઔકખગઘઙચછજઝઞટઠડઢણતથદધનપફબભમયરલળવશષસહૠૡૢૣཀཁགངཅཆཇཉཏཐདནཔཕབམཙཚཛཝཞཟའཡརལཤསཨאកខគឃងចឆជឈញដឋឌឍណតថទធនបផពភមសហយរលឡអវ្ែกขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะา฿เแโใไๅๆ๏๐๑๒๓๔๕๖๗๘๙๚ⴱⴲⴳⴴⴵⴶⴷⴸⴹⴺⴻⴼⴽⴾⴿⵀⵁⵂⵃⵄⵅⵆⵇⵈⵉⵊⵋⵌⵍⵎⵐⵑⵒⵓⵔⵕⵖⵗⵘⵙⵚⵛⵜⵝⵞⵠⵡⵢⵣⵤⵥĐÀẢÃÁẠĂẰẲẴẮẶÂẦẨẪẤẬÈẺẼÉẸÊỀỂỄẾỆÌỈĨÍỊÒỎÕÓỌÔỒỔỖỐỘƠỜỞỠỚỢÙỦŨÚỤƯỪỬỮỨỰỲỶỸÝỴàäèéëïijöüáêíîôóúûÆØÅæøåẂẀÄËÖÜẄŸẃýìòùẁỳẅÿĈĜĤĴŜŬĉĝĥĵŝŭÎȘȚăâșțÇŞÛçşÐÞðþŐŰőűÏŒœãõÑ̃ẽĩũỹñĄ́ĘĮŁŃĎĚŇŘŤŮďěňřťůĽĹŔľĺŕĀĒĢĪĶĻŅŌŖŪāēģīķļņōŗūĖŲąęėįųđảạằẳẵắặầẩẫấậẻẹềểễếệỉịỏọồổỗốơờởỡớợủụưừửữứựỷỵअआइईउऊऋॠऌॡऍऎएऐऑऒओकखगघङचछजझञटठडढणतथदधनपफबभमयरलळवशषसह०१२३४५६७८९्ँंः़ऽਅਆਇਈਉਊਏਐਓਔਕਖਗਘਙਚਛਜਝਞਟਠਡਢਣਤਥਦਧਨਪਫਬਭਮਯਰਲ਼ਵਸਹ";
static indexes = new Map(this.symbols.split("").map((d, i) => [d, i]));
static base = this.symbols.length;
static pack(num) {
let base = this.base;
var mod = num % base;
let remaining = Math.floor(num / base);
let chars = NumberPacker.symbols.charAt(mod);
if (remaining <= 0) {
return chars;
}
while (remaining > 0) {
mod = remaining % base;
remaining = Math.floor(remaining / base);
chars = NumberPacker.symbols.charAt(mod) + chars;
}
return chars;
}
static unpack(value) {
let base = this.base;
return value
.split("")
.reverse()
.reduce(function (prev, cur, i) {
return prev + NumberPacker.indexes.get(cur) * Math.pow(base, i);
}, 0);
}
}
function unpack(packedData) {
const parsedCsv = d3.csvParse(packedData.data);
const metaKeys = Object.keys(packedData.metadata);
parsedCsv.forEach((row) => {
metaKeys.forEach((k) => {
const index = row[k];
if (index == "") {
row[k] = packedData.metadata[k][0];
} else {
row[k] = packedData.metadata[k][parseInt(index, 36)];
}
});
});
return parsedCsv;
}
function pack(dataInitial, args) {
const drop = args?.drop || [];
let data = dataInitial;
if (drop.length > 0) {
data = dataInitial.map((d) => {
const copy = Object.assign({}, d);
drop.forEach((key) => {
delete copy[key];
});
return copy;
});
}
const props = Object.keys(data[0]);
const groups = [];
props.forEach((prop) => {
groups.push([prop, d3.groups(data, (d) => d[prop])]);
});
const dataGrouped = groups;
const filtered = dataGrouped.filter((d) => d[1].length < data.length / 1.3);
filtered.forEach((d) => {
let maxIndex = 0;
let maxVal = 0;
d[1].forEach((arr, j) => {
if (arr[1].length > maxVal) {
maxIndex = j;
maxVal = arr[1].length;
}
});
const temp = d[1][maxIndex];
d[1][maxIndex] = d[1][0];
d[1][0] = temp;
});
const jsonTables = {};
filtered.forEach((k) => {
jsonTables[k[0]] = k[1].map((d) => d[0]);
});
const indexMap = {};
filtered.forEach((k) => {
indexMap[k[0]] = {};
k[1].forEach((d, i) => {
indexMap[k[0]][d[0]] = !i ? "" : i.toString(36);
});
});
const replaced = data.map((d) => Object.assign({}, d));
replaced.forEach((row) => {
filtered.forEach((key) => {
const val = row[key[0]];
row[key[0]] = indexMap[key[0]][val];
});
});
const columns = Object.keys(replaced[0]);
const sizeFriendly = d3.csvFormat(replaced);
const finalData = Object.assign({
data: sizeFriendly,
metadata: jsonTables,
version: "36 base pack"
});
return finalData;
}
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.