async function getDataAggregateWithConfidenceIntervals(channelId, legend, numQuestionsToGet=70, dateBeginning="2019-10-9", dateEnd= "2019-11-20", fromStatic = true){
let dataframe
if(fromStatic){
dataframe = channelId == controlGroup ? EV2Comparison : EV2
}else{
dataframe = await getEvolutionOfQuestionsInChannel(channelId, numQuestionsToGet, "time", dateBeginning, dateEnd)
}
let initialScores = []
let lastQuestionName=""
let numQuestions =0;
dataframe.forEach(element => {
if(element.questionName != lastQuestionName && !element.questionName.includes("Meta")){
initialScores.push(element.score)
lastQuestionName = element.questionName
numQuestions = numQuestions+1
}
})
let initialScore = math.mean(initialScores)
let initialScoreStd = math.std(initialScores)
let timestamps = getTimeStamps(dataframe)
let dfNew = []
let counter = 0;
for(let i=0; i<dataframe.length; i++){if(!dataframe[i].questionName.includes("Meta")){
if(i!=(dataframe.length-1) && dataframe[i].questionName == dataframe[i+1].questionName){
let tempDataPoint = dataframe[i]
let originalTimeStamp = dataframe[i].timestamp
let nextTimeStamp = dataframe[i+1].timestamp
while(originalTimeStamp >= timestamps[counter]){
tempDataPoint.timestamp = timestamps[counter];
dfNew.push({
score: tempDataPoint.score,
timestamp: timestamps[counter],
questionName: tempDataPoint.questionName
})
counter++
}
}else if(i==(dataframe.length-1) || (dataframe[i].questionName != dataframe[i+1].questionName)){
let tempDataPoint = dataframe[i] // Note that this is *not* a copy
let lim = timestamps.length
let originalTimeStamp = dataframe[i].timestamp
while(counter < lim){
tempDataPoint.timestamp = timestamps[counter];
dfNew.push({
score: tempDataPoint.score,
timestamp: timestamps[counter],
questionName: tempDataPoint.questionName
})
counter++
}
counter = 0
}
}}
// At every point t, we have a copy of all questions. We use that to get the average, standard deviation, etc. at every point.
// We also prepare to plot the Prior and its standard deviations, but we later don't use it.
let dfMean = []
let dfPrior = []
for(let i=0; i<timestamps.length; i++){
let scores = []
for(let j=0; j< dfNew.length; j++){
if(dfNew[j].timestamp == timestamps[i]){
scores.push(dfNew[j].score)
}
}
let mean = math.mean(scores)
let std = math.std(scores)
if (mean+std*1.96/math.sqrt(numQuestions)<0 || legend == "Online crowdworkers" || legend.includes("Control")){ // This is a hack to solve a bug in foretold. See the spike which happens if you remove this line.
dfMean.push({
score: mean,
lowerConfidenceInterval: mean-std/math.sqrt(numQuestions), // *1.96
upperConfidenceInterval: mean+std/math.sqrt(numQuestions),
std: std,
timestamp: i*100/(timestamps.length -1),
initialScore: initialScore,
color: legend
})
dfPrior.push({
timestamp: i*100/timestamps.length,
initialScore: initialScore,
lowerConfidenceIntervalPrior: initialScore-1.96*initialScoreStd/math.sqrt(numQuestions),
upperConfidenceIntervalPrior: initialScore+1.96*initialScoreStd/math.sqrt(numQuestions),
color: legend+"' Prior",
})
}
}
return {dfMean: dfMean} // If one wishes, one could add dfPrior
}