2d:null 2e:["$","title",null,{"children":"Plots for the Foretold Amplification Experiment | Nuño Sempere | Observable"}] 33:Ta16,md `You may import the functions used to produce the following two charts with: ~~~ js import {plotAggregateAfterEnd,plotAggregateEvolution} from "@nunosempere/plots-for-the-amplification-experiment" ~~~ Arguments: - the channel id as a string. Defaults to "f19015f5-55d8-4fd6-8621-df79ac072e15" (amplification experiment) - The title of the plot. Defaults to "Amplification Experiment: Aggregate score at the end of the experiment". - Number of questions to fetch. Defaults to 50. These questions have to include, but don't have to be limited to the questions you want to analyze - dateBeginning, as a string "Year-Month-Day", for example "2019-10-9"[^]. Any questions created before this date will be ignored. - dateEnd, as a string "Year-Month-Day", for example "2019-11-20"[^]. Any questions created after this date will be ignored. - plotMarginalImprovement has an additional argument: startAt. If you plot the marginal improvement of all predictions, the first ~three swamp all the rest, so it might be useful to start at, say, 4 - fromStatic: Is an API call made, or are the observable attachments used? [^]: Or, more generally, anything that is understood as a date by JS, see: https://javascript.info/date#date-parse-from-a-string Examples below: \`\`\` title = "Amplification Experiment: " foretoldPredictors = "f19015f5-55d8-4fd6-8621-df79ac072e15" controlGroup = "9ba34894-8fb7-40d8-b5c0-e7863ade924a" plotAggregateAfterEnd(foretoldPredictors,title+"Aggregate score at the end of the experiment", 50, "2019-10-9", "2019-11-10") plotAggregateAfterEnd(controlGroup,title+"Aggregate score at the end of the experiment "+"(Control)", 50, "2019-10-9", "2019-11-30") plotAggregateAfterEndComparison(foretoldPredictors, controlGroup) plotAggregateEvolution(foretoldPredictors,title+"Aggregate score across time", 50, "2019-10-9", "2019-11-30") plotAggregateEvolution(controlGroup,title+"Aggregate score across time (Control)", 50, "2019-10-9", "2019-11-30") plotAggregateWithConfidenceIntervals(title + "Average Question Across time, with standard error", foretoldPredictors, controlGroup, 50,"2019-10-9","2019-10-9", "2019-11-20","2019-11-30") plotMarginalImprovementForetoldAndControl(foretoldPredictors, controlGroup, title + "Marginal improvement of going from n => n+1 predictions, with standard error", 50, "2019-10-9", "2019-11-30", 1) plotMarginalImprovementForetoldAndControl(foretoldPredictors, controlGroup, title + "Marginal improvement of going from n => n+1 predictions, with standard error", 50, "2019-10-9", "2019-11-20", 4) \`\`\` `34:T45e,async function getEvolutionOfQuestion(questionId, questionName="", orderBy = "predictionNumber"){ let data = await getQuestionData(questionId) let comparison = findInitial(data) let resolution = findResolution(data) let result = [] let counter = 1; let bugSkipped = false for(let i=0; i !question.questionName.includes("Meta")) // And this is the same as above const line = vl.markLine().data(df).encode( vl.y().field('score').type('quantitative'), vl.x().field('timestamp').type('quantitative'), vl.color().field('questionName').type('nominal').legend({orient: 'bottom', direction: "vertical", labelLimit:1000}), ); const point = vl.markCircle().data(df).encode( vl.y().field('score').type('quantitative').scale({type: 'log'}), vl.x().field('timestamp').type('quantitative').title("Prediction #"), vl.color().field('questionName').type('nominal') ); return vl.layer(line, point) .width(500) .title(title) .render(); } 37:T8ee,async function plotAggregateAfterEnd(channelId="f19015f5-55d8-4fd6-8621-df79ac072e15", title = "Amplification Experiment: Aggregate score at the end of the experiment", numberOfQuestionsToFetch = 50,dateBeginning="2019-10-9", dateEnd= "2019-11-20", fromStatic = true){ // This is just to avoid having to fetch the data from the API every time async function getData(fromStatic){ if(fromStatic){ return (channelId == "f19015f5-55d8-4fd6-8621-df79ac072e15")? EV: EVControl; }else{ return getEvolutionOfQuestionsInChannel(channelId, numberOfQuestionsToFetch, "predictionNumber", dateBeginning, dateEnd); } }; let df = await getData(fromStatic) // For each question, we get the last aggregate. function getTheLastAggregate(df){ let result = [] for(let i=0;i a.score - b.score) let counter = 1; df2.forEach(object => { object.questionName = counter + ". "+ object.questionName counter++ }) // Plotting specifics const line = vl.markBar().data(df2).encode( vl.y().field('score').type('quantitative').scale({type: 'log'}).title("Score"), vl.x().field('questionName').type('nominal').axis(null).sort(null), vl.color().field('questionName').type('nominal').legend({orient: 'bottom', direction: "vertical", labelLimit:1000}).title("Question Names"), ); return line .width(500) .title(title) .render(); }38:Tb2e,async function plotAggregateAfterEndComparison(channelIdPredictors="f19015f5-55d8-4fd6-8621-df79ac072e15", channelIdComparison, title = "Comparison of final aggregate scores", numberOfQuestionsToFetch = 50,dateBeginning="2019-10-9", dateEnd= "2019-11-30", fromStatic=true){ // This function is much the same as the one before it, with the exception of the last thing at the end: layering. // Predictors async function getData(fromStatic, channelId){ if(fromStatic){ return (channelId == "f19015f5-55d8-4fd6-8621-df79ac072e15")? EV: EVControl; }else{ return getEvolutionOfQuestionsInChannel(channelId, numberOfQuestionsToFetch, "predictionNumber", dateBeginning, dateEnd); } }; let df = await getData(fromStatic, channelIdPredictors) function transformData(df){ let result = [] for(let i=0;i a.score - b.score) let counter = 1; df2.forEach(object => { object.questionName = counter + ". "+ object.questionName counter++ }) const line = vl.markBar({opacity: 0.35}).data(df2).encode( vl.y().fieldQ('score').scale({type: 'log'}).sort(null), vl.x().fieldN('questionName').axis(null), vl.color().field('questionName').type('nominal').legend({orient: 'bottom', direction: "vertical", labelLimit:1000}), ); // Comparison let dfComparison = await getData(fromStatic, channelIdComparison) let df2Comparison = transformData(dfComparison); function sortFunction(object){ for(let i=0; i (sortFunction(a)-sortFunction(b))) df2Comparison.forEach(object => { object.questionName = sortFunction(object) + ". "+ object.questionName }) const lineComparison = vl.markBar({opacity: 1}).data(df2Comparison).encode( vl.y().field('score').type('quantitative').scale({type: 'log'}).sort(null), vl.x().fieldN('questionName').axis(null), vl.color().field('questionName').type('nominal').legend({orient: 'bottom', direction: "vertical", labelLimit:1000}), ); // This layer is responsible for we needing to use alphabetical sorting (by appending 1., 2. before the questions. Otherwise, it doesn't respect sortings return(vl.layer(line, lineComparison)//, line) .width(500) .title(title) .render() ) }39:Tf67,async function getDataAggregateWithConfidenceIntervals(channelId, legend, numQuestionsToGet=70, dateBeginning="2019-10-9", dateEnd= "2019-11-20", fromStatic = true){ // Avoid making an API call let dataframe if(fromStatic){ dataframe = channelId == controlGroup ? EV2Comparison : EV2 }else{ dataframe = await getEvolutionOfQuestionsInChannel(channelId, numQuestionsToGet, "time", dateBeginning, dateEnd) } // get the initial Score; the score of the prior let initialScores = [] let lastQuestionName="" let numQuestions =0; dataframe.forEach(element => { if(element.questionName != lastQuestionName && !element.questionName.includes("Meta")){ initialScores.push(element.score) lastQuestionName = element.questionName numQuestions = numQuestions+1 } }) let initialScore = math.mean(initialScores) let initialScoreStd = math.std(initialScores) // What happens to a question while other questions are updated? // Here, at every event at which a timestamp exists, we take a snapshot of every question at that moment. let timestamps = getTimeStamps(dataframe) let dfNew = [] let counter = 0; for(let i=0; i= timestamps[counter]){ tempDataPoint.timestamp = timestamps[counter]; dfNew.push({ score: tempDataPoint.score, timestamp: timestamps[counter], questionName: tempDataPoint.questionName }) counter++ } }else if(i==(dataframe.length-1) || (dataframe[i].questionName != dataframe[i+1].questionName)){ let tempDataPoint = dataframe[i] // Note that this is *not* a copy let lim = timestamps.length let originalTimeStamp = dataframe[i].timestamp while(counter < lim){ tempDataPoint.timestamp = timestamps[counter]; dfNew.push({ score: tempDataPoint.score, timestamp: timestamps[counter], questionName: tempDataPoint.questionName }) counter++ } counter = 0 } }} // At every point t, we have a copy of all questions. We use that to get the average, standard deviation, etc. at every point. // We also prepare to plot the Prior and its standard deviations, but we later don't use it. let dfMean = [] let dfPrior = [] for(let i=0; i object.timestamp)) let marginalImprovementInScore = Array(maxPredictionNumber-1) marginalImprovementInScore.fill([]) // Get the proportional marginal improvement of the aggregate after every prediction. let lastTimeStamp = Infinity; let lastScore = Infinity // Exclude question which was only answered by one of the two groups in the experiment data.forEach(element => {if(!element.questionName.includes("Meta")){ if(element.timestamp>lastTimeStamp){ // Calculate marginal improvement as how much closer to a perfect score each prediction moved the aggregate, as a percentage of the distance between the aggregate and the perfect score. let functionOfScore = (element.score - lastScore) / math.abs(lastScore) // Push to array for storing the values marginalImprovementInScore[element.timestamp -2] = marginalImprovementInScore[element.timestamp-2].concat([functionOfScore]) } lastTimeStamp = element.timestamp lastScore = element.score }}) // Average it and push it to a dataframe suitable for plotting let counter = 0; let dataframe = [] // Iterate over prediction # marginalImprovementInScore.forEach(element => { counter++ if(counter>=startAt){ // Compute errorbar width let mean = math.mean(element) let std = math.std(element) let length = element.length let errorbar = std/math.sqrt(length) // Multiply by 1.96 to get 95% confidence intervals // Push to dataframe dataframe.push({ predictionNumber: counter, marginalImprovementInScore: mean, lowerConfidenceInterval: mean-errorbar, upperConfidenceInterval: mean+errorbar, // 1.96 std: std, color: legend, }) } }) return dataframe }3d:T78b,async function plotMarginalImprovement(channelId, color, color2, legend, numQuestionsToGet=50, dateBeginning="2019-10-9", dateEnd= "2019-11-20", startAt=1, constant, fromStatic=true){ // Avoid making unnecessary calls let dataframe if(fromStatic == true){ dataframe = legend.includes("crowdworkers") || legend.includes("Control") ? marginalImprovementControl : marginalImprovementForecasters }else{ dataframe = getMarginalImprovementData(channelId, legend, numQuestionsToGet, dateBeginning, dateEnd, startAt) } // Define the graphic const line = vl.mark({type: 'line', color: color}).data(dataframe).encode( vl.y().field('marginalImprovementInScore').type('quantitative'),//.scale({type: 'symlog', constant: constant}), vl.x().field('predictionNumber').type('quantitative'), vl.color().title("Legend").scale({"range": ["blue", "green"]}).field("color") // This, is a hack ); const point = vl.mark({type: 'circle', color: color}).data(dataframe).encode( vl.y().field('marginalImprovementInScore').type('quantitative').title("Percentage move towards perfect score"), vl.x().field('predictionNumber').type('quantitative').title("Prediction #"), ); const lineMin = vl.mark({type: 'line', color: "light"+color}).data(dataframe).encode( vl.y().field('lowerConfidenceInterval').type('quantitative'), vl.x().field('predictionNumber').type('quantitative'), ); const lineMax = vl.mark({type: 'line', color: "light"+color}).data(dataframe).encode( vl.y().field('upperConfidenceInterval').type('quantitative'), vl.x().field('predictionNumber').type('quantitative'), ); const area = vl.markArea({opacity : 0.35, color: "light"+color}) .data(dataframe) .encode( vl.x().fieldQ('predictionNumber'), vl.y().fieldQ('lowerConfidenceInterval'), vl.y2().fieldQ('upperConfidenceInterval') ); return([line, point, lineMin, lineMax, area]) }3e:[{"id":"9647c2ad70c56bba90bc601bad7499e8807c0f2f5604711436954e65dd15f073414c6a3ae4c1199f278e2d5f5ea86bf1c0c3dfbb137bfe744161f6f5959c6fb2","href":"https://static.observableusercontent.com/files/9647c2ad70c56bba90bc601bad7499e8807c0f2f5604711436954e65dd15f073414c6a3ae4c1199f278e2d5f5ea86bf1c0c3dfbb137bfe744161f6f5959c6fb2","name":"plots-for-the-amplification-experiment (6).png","type":"image/png","lastModified":1574940845165.997,"size":86111},{"id":"cf6b169ac6436cd0a3f5bdb1046f16c06f5ccd7e952f9890ff1db46e98b4481d82272cefc78ca86f26a221e8197e2078c5bbb621f7b2234e9b908eacdbaedaec","href":"https://static.observableusercontent.com/files/cf6b169ac6436cd0a3f5bdb1046f16c06f5ccd7e952f9890ff1db46e98b4481d82272cefc78ca86f26a221e8197e2078c5bbb621f7b2234e9b908eacdbaedaec","name":"plots-for-the-amplification-experiment.png","type":"image/png","lastModified":1574940844695.893,"size":85853},{"id":"ee1658d7542471aa9164303861b149dc53137fd3ed860afeccc9e5c502cfc75fab97535a2eb4b381db9f1ac5b13b1e09712a38de2356f41f5bba0808f3962568","href":"https://static.observableusercontent.com/files/ee1658d7542471aa9164303861b149dc53137fd3ed860afeccc9e5c502cfc75fab97535a2eb4b381db9f1ac5b13b1e09712a38de2356f41f5bba0808f3962568","name":"plots-for-the-amplification-experiment (8).png","type":"image/png","lastModified":1574940844635.498,"size":93647},{"id":"cf6b169ac6436cd0a3f5bdb1046f16c06f5ccd7e952f9890ff1db46e98b4481d82272cefc78ca86f26a221e8197e2078c5bbb621f7b2234e9b908eacdbaedaec","href":"https://static.observableusercontent.com/files/cf6b169ac6436cd0a3f5bdb1046f16c06f5ccd7e952f9890ff1db46e98b4481d82272cefc78ca86f26a221e8197e2078c5bbb621f7b2234e9b908eacdbaedaec","name":"plots-for-the-amplification-experiment (2).png","type":"image/png","lastModified":1574940844778.288,"size":85853},{"id":"22a7a66755b9605c4447774f04cf3964d1187ad21bcb62dfd790784721971001b04779f55c00ccc29d93acbe73564b12e532aa2a64227ba47504262168db80f8","href":"https://static.observableusercontent.com/files/22a7a66755b9605c4447774f04cf3964d1187ad21bcb62dfd790784721971001b04779f55c00ccc29d93acbe73564b12e532aa2a64227ba47504262168db80f8","name":"plots-for-the-amplification-experiment (4).png","type":"image/png","lastModified":1574940845167.648,"size":166834},{"id":"55530ac4f4f3ed469033c8c96377aa27873d5ce4e135b5e6fb9b5c062c6bb9c21f5aa873d2981b27e05d7f9ff12b58b0fde5d81b022018e224dd4ed468ab95c6","href":"https://static.observableusercontent.com/files/55530ac4f4f3ed469033c8c96377aa27873d5ce4e135b5e6fb9b5c062c6bb9c21f5aa873d2981b27e05d7f9ff12b58b0fde5d81b022018e224dd4ed468ab95c6","name":"plots-for-the-amplification-experiment (3).png","type":"image/png","lastModified":1574940845125.831,"size":89361},{"id":"2b6c439fb9aa0ce5f36276570c8096b59d12cdfa1501bbf32333ee894cc7dc769517967a45581d5573afd09b682da2447b2bec78d28b2a551b7e86fa04de2bd4","href":"https://static.observableusercontent.com/files/2b6c439fb9aa0ce5f36276570c8096b59d12cdfa1501bbf32333ee894cc7dc769517967a45581d5573afd09b682da2447b2bec78d28b2a551b7e86fa04de2bd4","name":"plots-for-the-amplification-experiment (9)@1.png","type":"image/png","lastModified":1574950087413.031,"size":112859},{"id":"c5396ed488afffb4716e68cc9414533694f8c3d4e78f0efc1fca8224c24676b88c142526fcb843d31418d355eb85db923a1eeca5c563502e40aeba6fcf6d764d","href":"https://static.observableusercontent.com/files/c5396ed488afffb4716e68cc9414533694f8c3d4e78f0efc1fca8224c24676b88c142526fcb843d31418d355eb85db923a1eeca5c563502e40aeba6fcf6d764d","name":"EV (1).json","type":"application/json","lastModified":1574949294670.474,"size":14821},{"id":"da3e03946c2e628c3004671ceaacc373a82d85224d3883c5b91009100aa39a7a7fc0d588e3fcc33c57f2d21ca37f87ff5b057ccaa152898e4ac4f43ad3a83a1b","href":"https://static.observableusercontent.com/files/da3e03946c2e628c3004671ceaacc373a82d85224d3883c5b91009100aa39a7a7fc0d588e3fcc33c57f2d21ca37f87ff5b057ccaa152898e4ac4f43ad3a83a1b","name":"EVControl (1).json","type":"application/json","lastModified":1574949294650.825,"size":10380},{"id":"e10c1c36e6de35514f52701c6f84c65124d30379847261dc32f44c3f5b8362d3a47aad4a0d13e5db7e0f88915b453a74b494f78b444a2f20cb30a5e1620a5504","href":"https://static.observableusercontent.com/files/e10c1c36e6de35514f52701c6f84c65124d30379847261dc32f44c3f5b8362d3a47aad4a0d13e5db7e0f88915b453a74b494f78b444a2f20cb30a5e1620a5504","name":"dataframeForecasters.json","type":"application/json","lastModified":1574948660000.19,"size":3252},{"id":"0b63b88910e69ad0da721f65bc1693607caac1dce26f0582ec6af65afa1806df99568fc2f122586ddb0e4c84c0fef4e13cd56717ccf52415d6bf36ca034fb9af","href":"https://static.observableusercontent.com/files/0b63b88910e69ad0da721f65bc1693607caac1dce26f0582ec6af65afa1806df99568fc2f122586ddb0e4c84c0fef4e13cd56717ccf52415d6bf36ca034fb9af","name":"dataframeControl.json","type":"application/json","lastModified":1574948659988.061,"size":2365},{"id":"7ee9f8d5f1b7a5363877df2ddb668a768df5923a328af46e91efa41d3bccb3394bc98ce43ca656bbf056bc311b62e96407e79f7d2b146e03c5b3a51a7660e50e","href":"https://static.observableusercontent.com/files/7ee9f8d5f1b7a5363877df2ddb668a768df5923a328af46e91efa41d3bccb3394bc98ce43ca656bbf056bc311b62e96407e79f7d2b146e03c5b3a51a7660e50e","name":"CG.json","type":"application/json","lastModified":1574948575092.267,"size":14811},{"id":"656deae17700a7005e91b60b0919f156528c10bc8fe570b5fad605f4dd4873e52c94f76b52e9fe7314f679a220505781a71e9f75557a8d7df2d37cd434afb500","href":"https://static.observableusercontent.com/files/656deae17700a7005e91b60b0919f156528c10bc8fe570b5fad605f4dd4873e52c94f76b52e9fe7314f679a220505781a71e9f75557a8d7df2d37cd434afb500","name":"FP.json","type":"application/json","lastModified":1574948569618.579,"size":20209},{"id":"322b276e3bef248289e7353a53ff28e10e6263f1b94c6a25aa3af399425aee2233d793b0b3b148ecda5267322634d29c0a7d53a2f78d5db058484075ff6a266a","href":"https://static.observableusercontent.com/files/322b276e3bef248289e7353a53ff28e10e6263f1b94c6a25aa3af399425aee2233d793b0b3b148ecda5267322634d29c0a7d53a2f78d5db058484075ff6a266a","name":"EV2.json","type":"application/json","lastModified":1574948430351.528,"size":16221},{"id":"8d16e9d1649487b038b9ce5c0c89113205fcab1fb855fe922ba9009bb9d14f729d982fb9238e3b1f53212e2e632b273072bb92e26671141a1765299088b4337e","href":"https://static.observableusercontent.com/files/8d16e9d1649487b038b9ce5c0c89113205fcab1fb855fe922ba9009bb9d14f729d982fb9238e3b1f53212e2e632b273072bb92e26671141a1765299088b4337e","name":"EV2Comparison.json","type":"application/json","lastModified":1574948430359.566,"size":11491},{"id":"c7127cac8e309daa741ccf4140585bbe0199666f2a5b9cc075bd88d4c9f60b35f8a3ed79abf73deb8dd7c1abed9a47e587911c6212d23f63aab69a4edd35a691","href":"https://static.observableusercontent.com/files/c7127cac8e309daa741ccf4140585bbe0199666f2a5b9cc075bd88d4c9f60b35f8a3ed79abf73deb8dd7c1abed9a47e587911c6212d23f63aab69a4edd35a691","name":"plots-for-the-amplification-experiment (1).png","type":"image/png","lastModified":1574940845141.539,"size":103670},{"id":"c4b01c9f2549c14d8281dd7ba569498e6d8601c4507b56afc942ebc229cdfb9bf98956039764e371f6ae1fd3bda83657458d24ae73104081a756c41f3c109d99","href":"https://static.observableusercontent.com/files/c4b01c9f2549c14d8281dd7ba569498e6d8601c4507b56afc942ebc229cdfb9bf98956039764e371f6ae1fd3bda83657458d24ae73104081a756c41f3c109d99","name":"plots-for-the-amplification-experiment (5).png","type":"image/png","lastModified":1574940845149.518,"size":128396},{"id":"334f538cd157835b67bb8f0605c514e3e54d9451c9ff546e4915009c49f0b135b95672926192d44e74e83e158b3adeaedef15b65e889a04381d2e2c4d855e87a","href":"https://static.observableusercontent.com/files/334f538cd157835b67bb8f0605c514e3e54d9451c9ff546e4915009c49f0b135b95672926192d44e74e83e158b3adeaedef15b65e889a04381d2e2c4d855e87a","name":"plots-for-the-amplification-experiment (7).png","type":"image/png","lastModified":1574940845122.287,"size":91077},{"id":"c5396ed488afffb4716e68cc9414533694f8c3d4e78f0efc1fca8224c24676b88c142526fcb843d31418d355eb85db923a1eeca5c563502e40aeba6fcf6d764d","href":"https://static.observableusercontent.com/files/c5396ed488afffb4716e68cc9414533694f8c3d4e78f0efc1fca8224c24676b88c142526fcb843d31418d355eb85db923a1eeca5c563502e40aeba6fcf6d764d","name":"EV (2).json","type":"application/json","lastModified":1574961914625.236,"size":14821},{"id":"6d58959f852b4594ebe9f2ec93d0f2d0114f782a581c47e529aed43dd91a8093ca328985558fd24571106e0ddf5a270992bc315c7f3cbd93f08de8c713a12b3d","href":"https://static.observableusercontent.com/files/6d58959f852b4594ebe9f2ec93d0f2d0114f782a581c47e529aed43dd91a8093ca328985558fd24571106e0ddf5a270992bc315c7f3cbd93f08de8c713a12b3d","name":"marginalImprovementForecasters.json","type":"application/json","lastModified":1575029876289.268,"size":3377},{"id":"f3a0e6beb77a8af5334b1ec63200d8f6f90809bad138c154127120bec9592f96a8bb36e40a5408d8a4b11d81414f3098feacbc520463c2e9897e06b868cd5b28","href":"https://static.observableusercontent.com/files/f3a0e6beb77a8af5334b1ec63200d8f6f90809bad138c154127120bec9592f96a8bb36e40a5408d8a4b11d81414f3098feacbc520463c2e9897e06b868cd5b28","name":"marginalImprovementControl (1).json","type":"application/json","lastModified":1575029876290.73,"size":2431},{"id":"da3e03946c2e628c3004671ceaacc373a82d85224d3883c5b91009100aa39a7a7fc0d588e3fcc33c57f2d21ca37f87ff5b057ccaa152898e4ac4f43ad3a83a1b","href":"https://static.observableusercontent.com/files/da3e03946c2e628c3004671ceaacc373a82d85224d3883c5b91009100aa39a7a7fc0d588e3fcc33c57f2d21ca37f87ff5b057ccaa152898e4ac4f43ad3a83a1b","name":"EVControl (2).json","type":"application/json","lastModified":1574961914603.527,"size":10380},{"id":"e9f051c023c966db7b2749d6e1b627eba894f53f8706e6fcbd868d50ec5d87b145fb9e88a9af1d263eded1467ed173fb47597a81a355583c87150263348b2c06","href":"https://static.observableusercontent.com/files/e9f051c023c966db7b2749d6e1b627eba894f53f8706e6fcbd868d50ec5d87b145fb9e88a9af1d263eded1467ed173fb47597a81a355583c87150263348b2c06","name":"EV (3).json","type":"application/json","lastModified":1575632741643.564,"size":18595},{"id":"349780cdb114f1766d605d7eea96e72699162fca0038ac93cf9c3520a2ae8599554abe16447782befb8929ed4766e1643b73d0e53daf5a51cfe54676b4edc4df","href":"https://static.observableusercontent.com/files/349780cdb114f1766d605d7eea96e72699162fca0038ac93cf9c3520a2ae8599554abe16447782befb8929ed4766e1643b73d0e53daf5a51cfe54676b4edc4df","name":"EVControl (3).json","type":"application/json","lastModified":1575632741644.534,"size":13433},{"id":"f4f83ecc6714242689132e9ff819b301de8e27e266c1ecb7fc0b98f61b8035082a6a904589d19478170199fa56f1c9d1a0365908ce6336a1c74f8ef34beea225","href":"https://static.observableusercontent.com/files/f4f83ecc6714242689132e9ff819b301de8e27e266c1ecb7fc0b98f61b8035082a6a904589d19478170199fa56f1c9d1a0365908ce6336a1c74f8ef34beea225","name":"EV2Comparison (1).json","type":"application/json","lastModified":1575632830206.395,"size":14542},{"id":"96d7196801147c9a496ede864d8fa923982563f1d09a1ad9ee01bf9dfdd8a2a8ef0c21aa2bfa7b08d1a8760ee97774322d676c2689f09c424022af3e67bb8be0","href":"https://static.observableusercontent.com/files/96d7196801147c9a496ede864d8fa923982563f1d09a1ad9ee01bf9dfdd8a2a8ef0c21aa2bfa7b08d1a8760ee97774322d676c2689f09c424022af3e67bb8be0","name":"EV2 (1).json","type":"application/json","lastModified":1575632831919.214,"size":19982}] 31:{"id":"59ca63065a73e94f","slug":"plots-for-the-amplification-experiment","title":"Plots for the Foretold Amplification Experiment","thumbnail":"d4ff9d1b7ba39b9a24789f813fc0090748a99a7c9e62a08d555baf067e4da6ec","access_level":"public","create_time":"$D2019-11-21T14:28:07.717Z","update_time":"$D2019-12-18T12:33:30.694Z","view_time":null,"first_public_time":"$D2019-11-21T20:59:49.308Z","has_unpublished_edits":null,"fork":false,"stars":0,"starred":false,"owner":{"type":"team","id":"07a813a05afbeeaf","login":"nunosempere","name":"Nuño Sempere","avatar_url":"https://avatars-next.observableusercontent.com/512eae7d32fc25bba414910bc3c00fac559004bc68c3e14fa07fa7c6544b630f","tier":"starter_2024"},"version":2601,"publish_version":2601,"role":null,"authored":false,"authors":[{"type":"individual","id":"cee02f8e9acb9aee","login":"nunosempere","name":"Nuño Sempere","avatar_url":"https://avatars-next.observableusercontent.com/bc840e103afe5117def6829a8a8476725da032e7bb7cc3cb8c49257be439a233","tier":null}],"members":[{"type":"individual","id":"cee02f8e9acb9aee","login":"nunosempere","name":"Nuño Sempere","avatar_url":"https://avatars-next.observableusercontent.com/bc840e103afe5117def6829a8a8476725da032e7bb7cc3cb8c49257be439a233","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":4,"collection":null,"type":"notebook","body":{"title":"Plots for the Foretold Amplification Experiment","cells":[{"id":0,"value":"md`# Plots for the Foretold Amplification Experiment`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1717,"value":"md `## Images, generated from static data`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1163,"value":"foretoldPredictors = \"f19015f5-55d8-4fd6-8621-df79ac072e15\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1169,"value":"controlGroup = \"9ba34894-8fb7-40d8-b5c0-e7863ade924a\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2415,"value":"group1= \" (Network-adjacent forecasters)\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2417,"value":"group2 = \" (Online crowdworkers)\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":482,"value":"plotAggregateAfterEnd(foretoldPredictors,\"Aggregate score at the end of the experiment\"+group1, 50, \"2019-10-9\", \"2019-11-10\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1175,"value":"plotAggregateAfterEnd(controlGroup,\"Aggregate score at the end of the experiment \"+group2, 50, \"2019-10-9\", \"2019-11-30\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1657,"value":"plotAggregateAfterEndComparison(foretoldPredictors, controlGroup) ","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":486,"value":"plotAggregateEvolution(foretoldPredictors,\"Aggregate score across time\"+group1, 50, \"2019-10-9\", \"2019-11-30\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1180,"value":"plotAggregateEvolution(controlGroup,\"Aggregate score across time\"+group2, 50, \"2019-10-9\", \"2019-11-20\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":860,"value":"plotAggregateWithConfidenceIntervals(\"Evolution of aggregate score across time (± standard error)\", foretoldPredictors, controlGroup,50,\"2019-10-9\",\"2019-10-9\", \"2019-11-20\",\"2019-11-30\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1445,"value":"plotMarginalImprovementForetoldAndControl(foretoldPredictors, controlGroup, \"Marginal improvement of n:th prediction after prior (± standard error)\", 50, \"2019-10-9\", \"2019-11-30\", 1)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2138,"value":"md`___`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":501,"value":"md `# Behind the scenes`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2493,"value":"md `### How to import the plots in another observable notebook`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":489,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":33,"value":"md `### Get the data`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":6,"value":"import {getQuestionData} from \"@nunosempere/shapley-value-experiments-part-i\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":16,"value":"data = getQuestionData(\"5be0fba1-5701-4a83-816b-0ee4ab71073a\") // gets data ordered by creation time.","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2283,"value":"md `This data comes from a foretold GraphQL query, and we'll manipulate it in various ways to create the plots we want. \n\nIn Observable, one can see what an object contains by clicking on the black triangle before it, and how it's defined by clicking on the three lines which appear when one hovers the mouse to the left of the definition, and clicking \"Edit\". The reader may want to do this right now, and throughout the notebook`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":36,"value":"md `### Get the score`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":22,"value":"predictionAnalysis = require('@foretold/prediction-analysis@0.0.7/dist/index.js')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":20,"value":"function getScore(comparison, resolution, floatCdf){\n let contextLocation = ({\n agentPrediction: {\n data: { xs: floatCdf.xs, ys: floatCdf.ys },\n dataType: \"floatCdf\"\n },\n //marketPrediction: comparison.measurement, // findInitial(data)\n resolution: resolution.measurement // findInitial(data).measurement\n\n })\n //console.log(contextLocation)\n\n let PA = new predictionAnalysis.PredictionResolutionGroup(contextLocation)\n if(PA != undefined && PA.pointScore(\"NonMarketScore\") != undefined){\n return(PA.pointScore(\"NonMarketScore\").data)\n\n }else{\n return(undefined)\n }\n}\n\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":25,"value":"function findInitial(data){\n\n return({\n time: 0,// (new Date(data.creationTimes[0]))/(1000*3600*24), // number of days\n measurement: {\n data: {\n xs: data.values[0].floatCdf.xs, //0\n ys: data.values[0].floatCdf.ys //0\n },\n dataType: \"floatCdf\"\n }\n })\n\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":28,"value":"function findResolution(data){\n let result = []\n let InitialTime = new Date(data.creationTimes[0])\n for(let i = 0; i {\n if(element.questionName != lastQuestionName && !element.questionName.includes(\"Meta\")){\n initialScores.push(element.score)\n lastQuestionName = element.questionName\n numQuestions = numQuestions+1\n }\n })\n return \n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":726,"value":"function getTimeStamps(df){\n let timestamps = []\n for(let i=0; i (a-b)).filter((v, i, a) => a.indexOf(v) === i);\n return timestamps\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":625,"value":"$39","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1615,"value":"$3a","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1914,"value":"plotAggregateWithConfidenceIntervals(\"Average Question Across time, with standard deviation\", foretoldPredictors, controlGroup, 50, \"2019-10-9\",\"2019-10-9\", \"2019-11-20\", \"2019-11-30\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1130,"value":"md `## Get the marginal improvements`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1689,"value":"$3b","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1694,"value":"async function getEvolutionOfQuestionsInChannel2(channelId=\"f19015f5-55d8-4fd6-8621-df79ac072e15\", numberOfQuestionsToFetch = 50, orderBy = \"predictionNumber\", dateBeginning=\"2019-10-9\", dateEnd= \"2019-11-20\"){\n let questions = await getQuestions(channelId, numberOfQuestionsToFetch, dateBeginning, dateEnd)\n let result = []\n for(let question of questions){\n if(question.state ==\"JUDGED\"){\n let name = question.name //.split(\": \")[1]\n let questionScoresOverTime = await getEvolutionOfQuestion2(question.id,name, orderBy)\n result = result.concat(\n //name: question.name,\n //id: question.id,\n //scoresOverTime: questionScoresOverTime\n questionScoresOverTime\n )\n }\n }\n return result\n \n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1045,"value":"$3c","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2594,"value":"md`The below two functions generates data structures by querying the API. If you want to analyse the data yourself, uncomment them, and instead comment out the file imports right below.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1805,"value":"// marginalImprovementForecasters = getMarginalImprovementData(foretoldPredictors, \"Network-adjacent forecasters\", 50, \"2019-10-9\", \"2019-11-20\", 1)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1944,"value":"marginalImprovementForecasters = FileAttachment(\"marginalImprovementForecasters.json\").json()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1813,"value":"// marginalImprovementControl = getMarginalImprovementData(controlGroup, \"Online crowdworkers\", 50, \"2019-10-9\", \"2019-11-20\", 1)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1947,"value":"marginalImprovementControl = FileAttachment(\"marginalImprovementControl (1).json\").json()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2591,"value":"md`This function returns plots from the data structures.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1802,"value":"$3d","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1380,"value":"vl.layer(...F2)\n .width(500)\n .title(\"Marginal Improvement of going from n=> n+1 predictions ÷ Previous Score\")\n .render();","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1093,"value":"F2 = plotMarginalImprovement(foretoldPredictors,\"blue\",\"green\", \"Network-adjacent forecasters\", 50,\"2019-10-9\", \"2019-11-20\", 1, 0.05)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1385,"value":"/* vl.layer(...C2)\n .width(500)\n .title(\"Marginal Improvement of going from n=> n+1 predictions ÷ Previous Score\")\n .render(); \n*/","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1363,"value":"C2 = plotMarginalImprovement(controlGroup, \"green\",\"blue\", \"Online crowdworkers\",50,\"2019-10-9\", \"2019-11-20\", 1, 0.05)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1376,"value":"vl.layer(...C2, ...F2)\n .width(500)\n .title(\"Amplification Experiment: Marginal Improvement of going from n=> n+1 predictions ÷ Previous Score\")\n .render();","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1409,"value":"async function plotMarginalImprovementForetoldAndControl(foretoldPredictorsChannel, controlGroupChannel, title = \"Amplification Experiment: Marginal improvement of going from n-1 => n predictions\", numQuestionsToGet=50, dateBeginning=\"2019-10-9\", dateEnd= \"2019-11-20\", startAt=1){\n \n // The next two lines can be omitted, given that F2 and C2 are already defined\n let F2 = await plotMarginalImprovement(foretoldPredictorsChannel,\"blue\",\"blue\", \"Network-adjacent forecasters\", numQuestionsToGet ,dateBeginning, dateEnd, startAt)\n let C2 = await plotMarginalImprovement(controlGroupChannel,\"green\",\"blue\", \"Online crowdworkers\", numQuestionsToGet ,dateBeginning, dateEnd, startAt) \n \n return(\n vl.layer(...C2, ...F2)\n .width(500)\n .title(title)\n .render()\n )\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1426,"value":"// Uncomment to reveal\n\nplotMarginalImprovementForetoldAndControl(foretoldPredictors, controlGroup, \"Amplification Experiment: Marginal improvement of going from n-1 => n predictions\", 50, \"2019-10-9\", \"2019-11-20\", 1)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1443,"value":"// plotMarginalImprovementForetoldAndControl(foretoldPredictors, controlGroup, \"Amplification Experiment: Marginal improvement of going from n-1 => n predictions\", 50, \"2019-10-9\", \"2019-11-20\", 4) \n\n// The difference is that now the Prediction # starts at 4","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":957,"value":"md `## Imports`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":777,"value":"math = require('https://cdnjs.cloudflare.com/ajax/libs/mathjs/6.2.2/math.js')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2390,"value":"import {vl} from '@vega/vega-lite-api'","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W3e","stdlib":"1","titling":"auto"}} 32:"$31:body"