Quietly subversive
#PhD @anucybernetics #opensource #conversationalAI #MLops #knitting #corgi
Ex @mycroft_ai @deakin @linuxaustralia @mozilla @nvidia
Public
Edited
Mar 22, 2024
chartSplitsByAgeRange = StackedBarChart(splitsByAge, {
x: (d) => d.utterances,
y: (d) => d.locale,
z: (d) => d.ageKey,
xLabel: "# of utterances by age group",
yLabel: "language-locale",
yDomain: d3.groupSort(
splitsByAge,
(D) => d3.sum(D, (d) => d.utterances),
(d) => d.locale
), // sort y by x
zDomain: ages,
color: "golden yellowrod"
})
chartSplitsByAge100Percent = StackedBarChart(splitsByAge, {
x: (d) => d.agePercentage,
y: (d) => d.locale,
z: (d) => d.ageKey,
xLabel: "% of utterances by age group",
yLabel: "language-locale",
yDomain: d3.groupSort(
splitsByAge,
(D) => d3.sum(D, (d) => d.utterances),
(d) => d.locale
), // sort y by x
zDomain: ages,
colors: d3.schemeTableau10
})
chartSplitsByGender100Percent = StackedBarChart(splitsByGender, {
x: (d) => d.genderPercentage,
y: (d) => d.locale,
z: (d) => d.genderKey,
xLabel: "% of utterances by gender",
yLabel: "language-locale",
yDomain: d3.groupSort(
splitsByGender,
(D) => d3.sum(D, (d) => d.utterances),
(d) => d.locale
), // sort y by x
zDomain: genders,
colors: d3.schemeTableau10
})
Plot.plot({
grid: true,
inset: 10,
width: 1200,
height: 1000,
color: {
legend: true
},
x: { label: "↑ Average utterance duration (secs)" },
y: { label: "locale" },
// facet: {
// data: anscombe,
//x: "series"
//},
marks: [
Plot.frame(),
Plot.dot(localeData, {
x: (d) => d[1].avgDurationSecs,
y: (d) => d[0],
sort: { y: "x", reverse: true }
})
]
})
totalHoursValidated100Percent = Plot.plot({
grid: true,
width: 1200,
height: 2500,
legend: true,
x: { label: "↑ % of utterances validated" },
y: { label: "locale" },
// facet: {
// data: anscombe,
//x: "series"
//},
marks: [
Plot.barX(localeData, {
x: (d) => d[1].validHrs / d[1].totalHrs,
y: (d) => d[0],
sort: { y: "x", reverse: true },
fill: "#000000aa"
}),
Plot.ruleY([0])
]
})
chartClipsByValidationStatus = StackedBarChart(splitsByStatus, {
x: (d) => d.validationClips,
y: (d) => d.locale,
z: (d) => d.validationKey,
xLabel: "# of utterances by validation status",
yLabel: "language-locale",
yDomain: d3.groupSort(
splitsByStatus,
(D) => d3.sum(D, (d) => d.clips),
(d) => d.locale
), // sort y by x
zDomain: splitStatusesValidation,
colors: d3.schemeSet1
})
chartClipsByValidationStatus100Percent = StackedBarChart(splitsByStatus, {
x: (d) => d.validationPercentage,
y: (d) => d.locale,
z: (d) => d.validationKey,
xLabel: "% of utterances by validation status",
yLabel: "language-locale",
yDomain: d3.groupSort(
splitsByStatus,
(D) => d3.sum(D, (d) => d.clips),
(d) => d.locale
), // sort y by x
zDomain: splitStatusesValidation,
colors: d3.schemeSet1
})
chartClipsBySplitStatus = StackedBarChart(splitsByStatus, {
x: (d) => d.validationClips,
y: (d) => d.locale,
z: (d) => d.validationKey,
xLabel: "# of utterances by split status",
yLabel: "language-locale",
yDomain: d3.groupSort(
splitsByStatus,
(D) => d3.sum(D, (d) => d.clips),
(d) => d.locale
), // sort y by x
zDomain: splitStatusesDevTrainTest,
colors: d3.schemeDark2
})
Note that percentage here is calculated as a % of the total number of clips for the language. Not all the clips in the language will be included in the dev-test-train splits, for example clips that are invalidated.
An unknown
One of the things I will need to follow up with Gabe is whether duplicated sentences are removed from the splits. I am not sure.
I know that the CV project is planning to revise how splits are calculated, so this graph will need to be updated in the future.
chartClipsBySplitStatus100Percent = StackedBarChart(splitsByStatus, {
x: (d) => d.validationPercentage,
y: (d) => d.locale,
z: (d) => d.validationKey,
xLabel: "% of utterances by dev test train split status",
yLabel: "language-locale",
yDomain: d3.groupSort(
splitsByStatus,
(D) => d3.sum(D, (d) => d.clips),
(d) => d.locale
), // sort y by x
zDomain: splitStatusesDevTrainTest,
colors: d3.schemeDark2
})
avgClipsPerContributor = Plot.plot({
grid: true,
inset: 10,
width: 1200,
height: 1000,
color: {
legend: true
},
x: { label: "↑ Average number of clips per contributoer" },
y: { label: "locale" },
// facet: {
// data: anscombe,
//x: "series"
//},
marks: [
Plot.frame(),
Plot.dot(localeData, {
x: (d) => parseInt(d[1].clips / d[1].users),
y: (d) => d[0],
sort: { y: "x", reverse: false }
})
]
})
cvdata = FileAttachment("cv-corpus-17.0-2024-03-15.json").json()
splitStatusesValidation = ["invalidated", "other", "validated"]
splitStatusesDevTrainTest = ["dev", "test", "train"]
Imports
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
