31:T974,viewof confusion_matrix = { function makeConfusion(single_result) { let transformed_confusion_matrix = []; for ( var i = 0; i < single_result.aggregates.confusion_matrix.labels.length; i++ ) { for ( var j = 0; j < single_result.aggregates.confusion_matrix.matrix[i].length; j++ ) { transformed_confusion_matrix.push({ target: single_result.aggregates.confusion_matrix.labels[i], prediction: single_result.aggregates.confusion_matrix.labels[j], value: single_result.aggregates.confusion_matrix.matrix[i][j] }); } } return { $schema: "https://vega.github.io/schema/vega-lite/v4.json", width: 300, height: 300, title: single_result.title, data: { values: transformed_confusion_matrix, format: { type: "json" } }, layer: [ { mark: "rect", encoding: { y: { field: "target", type: "ordinal" }, x: { field: "prediction", type: "ordinal" }, color: { field: "value", type: "quantitative", title: "Count of Records", legend: { direction: "vertical", gradientLength: 300 } } } }, { mark: "text", encoding: { y: { field: "target", type: "ordinal" }, x: { field: "prediction", type: "ordinal" }, text: { field: "value", type: "quantitative" }, color: { condition: { test: "datum['value'] < 40", value: "black" }, value: "white" } } } ], config: { axis: { grid: true, tickBand: "extent" } } }; } function getMetrics(results_index) { const outputs = evaluations[results_index].outputs; let metrics_index = -1; outputs.forEach((output, i) => { if (output.interface.slug == "metrics-json-file") { metrics_index = i; } }); return evaluations[results_index].outputs[metrics_index].value; } let graphs = []; evaluations.forEach((result, i) => { graphs.push(makeConfusion(getMetrics(i))); }); return embed({ $schema: "https://vega.github.io/schema/vega-lite/v4.json", data: { values: evaluations, format: { type: "json" } }, concat: graphs, columns: 2 }); }32:Tbbf,viewof roc_graph = { function getMetrics(results_index) { const outputs = evaluations[results_index].outputs; let metrics_index = -1; outputs.forEach((output, i) => { if (output.interface.slug == "metrics-json-file") { metrics_index = i; } }); const result = evaluations[results_index].outputs[metrics_index].value; result.title = evaluations[results_index].title; return result; } let old_style_metrics = []; evaluations.forEach((result, i) => { old_style_metrics.push(getMetrics(i)); }); return embed( { $schema: "https://vega.github.io/schema/vega-lite/v4.json", description: "Google's stock price over time.", width: 600, height: 600, data: { values: old_style_metrics }, transform: [ { flatten: [ "aggregates.roc_curve.fpr_vals", "aggregates.roc_curve.low_tpr_vals", "aggregates.roc_curve.high_tpr_vals", "aggregates.roc_curve.mean_tpr_vals" ], as: ["x_vals", "low_vals", "high_vals", "mean_vals"] } ], layer: [ { selection: { legend: { type: "multi", fields: ["title"], bind: "legend" }, grid: { type: "interval", bind: "scales" } }, encoding: { x: { field: "x_vals", type: "quantitative" }, y: { field: "mean_vals", type: "quantitative" }, color: { condition: { selection: "legend", field: "title", type: "nominal" }, value: "grey" }, opacity: { condition: { selection: "legend", value: 1 }, value: 0.2 } }, mark: { type: "line", clip: true } }, { mark: { type: "errorband", clip: true }, selection: { grid: { type: "interval", bind: "scales" } }, encoding: { x: { field: "x_vals", type: "quantitative", title: "1 - Specificity" }, y: { field: "low_vals", type: "quantitative", title: "Sensitivity" }, y2: { field: "high_vals" }, color: { condition: { selection: "legend", field: "title", type: "nominal" }, value: "grey" }, opacity: { condition: { selection: "legend", value: 0.6 }, value: 0.2 } } } ], resolve: { scale: { x: "shared", y: "shared" } } }, { actions: { export: true, source: true, compiled: false, editor: true } } ); }33:T7ae,viewof summary_stats_table = { function getMetrics(results_index) { const outputs = evaluations[results_index].outputs; let metrics_index = -1; outputs.forEach((output, i) => { if (output.interface.slug == "metrics-json-file") { metrics_index = i; } }); const result = evaluations[results_index].outputs[metrics_index].value; result.title = evaluations[results_index].title; result.pk = evaluations[results_index].pk; return result; } let old_style_metrics = []; evaluations.forEach((result, i) => { old_style_metrics.push(getMetrics(i)); }); // Convert to list const submissions = Object.values(old_style_metrics); // Collect stats in '2D', stat-major object const summaryStats = [ { title: "AUC", path: "az_ci_mean" }, { path: "low_az_val", title: "AUC 95%CI-" }, { path: "high_az_val", title: "AUC 95%CI+" } ]; const stats = {}; for (const submission of submissions) { const submission_id = submission.pk; summaryStats.forEach(key => { if (!(key.path in stats)) { stats[key.path] = { title: key.title }; } stats[key.path][submission_id] = Number.parseFloat( submission.aggregates[key.path] ).toPrecision(4); }); } console.log(stats); // Construct header const headerCells = [ html`Statistic`, ...submissions.map(submission => html`${submission.title}`) ]; // Construct table rows const rows = []; for (let stat_id in stats) { const stat = stats[stat_id]; const valueCells = [ html`${stat.title}`, ...submissions.map(submission => html`${stat[submission.pk]}`) ]; rows.push(html`${valueCells}`); } return html`${headerCells}${rows}

`; }34:[] 2f:{"title":"COVID19 Comparison","cells":[{"id":0,"value":"md`# COVID19 Comparison`","pinned":true,"hidden":false,"mode":"ojs"},{"id":5,"value":"import { evaluations } from \"@grand-challenge/data-fetch-example\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":237,"value":"function getMetrics(results_index) {\n const outputs = evaluations[results_index].outputs;\n let metrics_index = -1;\n outputs.forEach((output, i) => {\n if (output.interface.slug == \"metrics-json-file\") {\n metrics_index = i;\n }\n });\n\n return evaluations[results_index].outputs[metrics_index].value;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":293,"value":"md `\n# Confusion matrix\nThe confusion matrix view shows the difference between predictions and targets `","pinned":true,"hidden":false,"mode":"ojs"},{"id":13,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":150,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":154,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":70,"value":"embed = require(\"vega-embed@6.13\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":171,"value":"import {vl} from '@vega/vega-lite-api'","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W34","stdlib":"1","titling":"auto"}