Lawyer/Designer/Developer
Published
Edited
Jul 7, 2022
Data Analysis in Javascript
SimpleData = {
const sda = await require("simple-data-analysis@0.9.18");
return sda.SimpleData;
}
function showTable(sd) {
return Inputs.table(sd.getData());
}
function showChart(htmlOrSvg) {
return html`${htmlOrSvg}`;
}
{
let someData = [
{firstName: "Nael", lastName: "Shiab", job: "Computational journalist"},
{firstName: "Isabelle", lastName: "Bouchard"}
];
let simpleData = new SimpleData({
data: someData,
fillMissingKeys: true
});
return showTable(simpleData);
}
data from url
// Here, we load a CSV file from our Github repository
simpleData = new SimpleData()
.loadDataFromUrl({
url:
"https://raw.githubusercontent.com/nshiab/simple-data-analysis/main/data/employees.csv"
})
// you can also load data in nodejs using localDataFromLocalFile
showTable(simpleData)
// Department or Unit" becomes departmentOrUnit and End-of_year-Bonus becomes endOfYearBonus
simpleDataCleanKeys = simpleData.clone().formatAllKeys();
showTable(simpleDataCleanKeys);
renameKey()
you can also rename keys individually.
simpleDataShortKeys = simpleDataCleanKeys
.clone()
.renameKey({oldKey:"departementOrUnit", newKey: "unit"})
.renameKey({oldKey: "endOfYearBonus", newKey: "bonus"});
showTable(simpleDataShortKeys);
checkValues()
we must check what is in our values
checkValues tells us how many and what percentage (count | percentage ) of the values are of a specific type
valuesChecked = simpleDataShortKeys.clone().checkValues();
showTable(valuesChecked);
// Noticed we have a typo for the vice-president Neena Kocchar. Instead of Number, there is string "&6%" for its salary. Let's add it to the missingValues options, so the VP is also excluded.
simpleDataNoMissingValues = simpleDataShortKeys
.clone()
.excludeMissingValues({missingValues: [null, NaN, undefined, " ", "&6%"]});
//if you want to exclude items based on the values of a specific key only, you would write .excludeMissingValues({key: "nameOfTheKey"})
showTable(simpleDataNoMissingValues);
simpleDataString = simpleDataNoMissingValues
.clone()
.valuesToString({key: "unit"});
showTable(simpleDataString);
simpleDataDates = simpleDataString
.clone()
.valuesToDate({key: "hireDate", format: "%d-%b-%y"});
showTable(simpleDataDates);
dateToString()
You can convert a date into a string with a specific format. Let's rewrite our dates in hireDate to match a more familiar syntax.
simpleDataDatesString = simpleDataDates
.clone()
.datesToString({key: "hireDate", format: "%A, %B %e, %Y"});
showTable(simpleDataDatesString);
replaceValues()
Sometimes, we want to replace specific things in the text values of our data.
By default, replaceValues will make the replacement only if the entire string matches the oldValue provided. You can change the method option to partialString to replace only a part of the string. That's what we do here.
The values must be strings to use this method
simpleDataReplacedValues = simpleDataDatesString
.clone()
//Bonuses have a "%" at the end. We remove it.
.replaceValues({
key: "bonus",
oldValue: "%",
newValue: "",
method: "partialString"
})
// The comma must be replaced by a point for the decimals.
.replaceValues({
key: "bonus",
oldValue: ",",
newValue: ".",
method: "partialString"
})
// Since bonus should be a number instead of a string, we convert it to float while we are at it. It wasn't possible before removing the "%" and replacing the ",".
.valuesToFloat({
key: "bonus"
});
modifyValues()
When you need to do more complex operations on a value, modifyValues is the method to use. You can provide a function to valueGenerator that will be applied to values.
modifyValues gives you access to a specified value only. If you need values stored in multiple keys, check modifyItems.
// Let's change the order of the first and last name.
simpleDataValuesModified = simpleDataRounded
.clone()
.modifyValues({
key: "name",
valueGenerator: (name) => {
const nameSpilt = name.split(",");
return `${nameSpilt[1].trim()} ${nameSpilt[0].trim()}`;
}
});
showTable(simpleDataValuesModified);
modifyItems()
modifyItems lets us generate new values for a specific key by using any values from any keys. All you need is to provide itemGenerator option.
simpleDataItemsModified = simpleDataValuesModified
.clone()
.modifyItems({
key: "bonus",
itemGenerator: (employee) => {
return Math.round((employee.bonus/100) * employee.salary);
}
});
showTable(simpleDataItemsModified);
simpleDataKeyAdded = simpleDataItemsModified
.clone()
.addKey({
key: "firstName",
itemGenerator: (employee) => employee.name.split(" ")[0]
});
showTable(simpleDataKeyAdded);
simpleDataKeyRemoved = simpleDataKeyAdded
.clone()
.removeKey({key: "firstName"});
showTable(simpleDataKeyRemoved);
simpleDataItemsAdded = {
//our array of objects
const newEmployees = [{
hireDate:"Thursday, April 6, 2006",
name: "kaketo Paulo",
job: "Data Analyst",
salary: 1000000,
unit: "60",
bonus: 50.0,
}
];
return simpleDataKeyRemoved.clone().addItems({
dataToBeAdded: newEmployees
});
};
showTable(simpleDataItemsAdded);
simpleDataMerged = {
// Heres the name of each unit as an array of objects.
const unitsNames = [
{unit: "20", unitName: "Marketing"},
{unit: "30", unitName: "Administration"},
{unit: "40", unitName: "Sales"},
{unit: "50", unitName: "Human Resources"},
{unit: "60", unitName: "Engineering"},
{unit: "100", unitName: "Accounting"},
{unit: "110", unitName: "R&N"},
];
return simpleDataItemsAdded
.clone()
.mergeItems({dataToBeMerged: unitsNames, commonKey: "unit"});
};
showTable(simpleDataMerged);
Selecting Data
While looking for answers in your data, you often need to focus on only a part of your data sets.
simpleDataSelected = simpleDataMerged
.clone()
.selectKeys({keys: ["job", "salary", "unit"]});
showTable(simpleDataSelected);
// let us keep employees with a salary of more than 7000
simpleDataFilteredValues = simpleDataSelected
.clone()
.filterValues({key: "salary", valueComparator: (salary) => salary > 7000});
showTable(simpleDataFilteredValues);
// We keep employees that are not in the unit 100 and with a salary more than 7000
simpleDataFilteredItems = simpleDataSelected
.clone()
.filterItems({
itemComparator: (employee) => employee.unit != "100" && employee.salary > 7000
});
showTable(simpleDataFilteredItems);
showTable(simpleDataSelected.clone().describe());
showTable(simpleDataSelected.clone().sortValues({
key: "salary", order: "descending"
}));
simpleDataQuantiles = simpleDataSelected
.clone()
.addQuantiles({
key: "salary",
newKey: "decile",
nbQuantiles: 10
});
showTable(simpleDataQuantiles);
simpleDataBins = simpleDataSelected
.clone()
.addBins({
key: "salary",
newKey: "bin",
nbBins: 1000
});
showTable(simpleDataBins);
addOutliers()
Identifying outliers can be useful when your data values cover a wide range. addOutliers create a new key indicating if the value is an outlier(true or not false).
The outlier definition of this method is the same as the one used in Box plots
simpleDataOutliers = simpleDataSelected
.clone()
.addOutliers({
key: "salary",
newKey: "outlier"
})
// Let's reorder the data to see outliers first
.sortValues({key: "outlier", order: "descending"});
showTable(simpleDataOutliers);
showTable(simpleDataSelectedNoOutliers);
simpleDataSales = new SimpleData({
data: [
{temperature: 2, iceCreamSales: 0, hotChocholateSales: 15},
{temperature: 0, iceCreamSales: 0, hotChocholateSales: 9},
{temperature: 7, iceCreamSales: 0, hotChocholateSales: 3},
{temperature: 15, iceCreamSales: 1, hotChocholateSales: 0},
{temperature: 12, iceCreamSales: 4, hotChocholateSales: 1},
{temperature: 25, iceCreamSales: 10, hotChocholateSales: 0},
{temperature: 31, iceCreamSales: 25, hotChocholateSales: 0},
]
});
showTable(
simpleDataSales
.clone()
.correlation({
key1: "temperature",
key2: "iceCreamSales"
})
);
showTable(simpleDataSales.clone().correlation());
showTable(simpleDataSelected);
showTable(simpleDataSelected.clone().summarize());
showTable(
simpleDataSelected.clone().summarize({
value: "Salary",
keyCategory: "job",
summary: ["mean", "median"]
})
);
chart = simpleDataSelectedNoOutliers.getChart({
x: "job",
y: "salary",
type: "dot",
color: "job",
marginLeft: 75,
marginBottom: 50
});
showChart(chart);
plotOptions = ({
grid: true,
facet: {
marginLeft: 100,
data: simpleDataSelectedNoOutliers.getData(),
y: "job"
},
marks: [
Plot.dotX(simpleDataSelectedNoOutliers.getData(), {
x: "salary",
fill: "job"
})
]
});
Plot.plot(plotOptions);
simpleDataSelected.getKeys();
simpleDataSelected.getArray({key: "job"});
simpleDataSelected.getUniqueValues({key: "job"});
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
