31:T4b0,NAMs offer simple visualization through line plots for networks trained on a single feature, while networks trained on two features can use heatmaps. Existing ML techniques introduce feature combinations for pairwise and higher order combinations, but this can lead to an explosion of feature maps. These techniques address this by pruning and selecting specific feature combinations during training already^{[[1](https://arxiv.org/abs/2106.01613),[2](https://dl.acm.org/doi/abs/10.1145/2487575.2487579?casa_token=Z3-qKEzzXgwAAAAA:Vx276JMhKgac4JmPdS_LWrPuilwPmCeann4tWu0pHmxVv-_2MgfXavYQ-rFon5zK7Oy9GQM93irfFA),[3](https://www.sciencedirect.com/science/article/pii/S0031320321003484?casa_token=3AWoaSZ1V-sAAAAA:CzAf3EiY3oqArb1xOta3k-zIGqWJuJBqINnv5Tde8DdMQ3cOkfjn-JsxNk0kFz_66tAs-ulaEUI)]}. However, modern visualization methods can further help to find important feature maps. We implement the scoring of feature maps and an interactive sorting and filtering method to achieve this. In order to show whether a learned feature map can be trusted, we further enhance the data density visualization by adding individual heatmaps per class to the line charts and using scatter plots in heatmaps.32:T466,**Columns:** The dashboard is organized into columns of the feature maps per class. **Rows:** The first row displays the bar chart of importances while the following rows show the feature maps visible in the current filtering ordered by importance. **Text input:** A text input field allows users to filter displayed features by typing in substrings of feature names, facilitating a targeted analysis. **Bar charts:** Interactive bar charts depict feature importance scores. Users can interact with the bar charts to create a brush selection to filter the feature maps. **Line plots:** Line plots of 1-D feature maps illustrate submodule outputs against feature values, with data distributions per class shown in the background. The y-value represents submodule output, while the x-axis displays the feature. For distributions, classes are separated along the y-axis to avoid overplotting and assigned distinct colors. **Heatmaps:** Heatmaps of 2-D feature maps display feature interactions, with color intensity signifying submodule output levels. Data sample distribution is shown as a scatter plot within the heatmaps.33:T42c,viewof featureSelection = { const featureSelection = html``; featureSelection.addEventListener("input", () => { // First, we need to transform the data into a format that can be used by Vega-Lite const transformedData = transformData(classIndexMap); // Calculate the maximum importance value across all classes const maxImportance = Math.max( ...transformedData.map((classData) => Math.max(...classData.data.map((d) => d.importance)) ) ); const barCharts = transformedData.map((classData, index) => { console.log("classData", classData); // Change data color given featureSelection const add = classData?.data.map((item) => { if (!item.feature.includes(featureSelection.value)) { return { ...item, color: "lightgray" }; } return item; }); // Call vega streaming data update on barCharts update(barChartObject.barCharts[index], "data", add, () => true); }); }); return featureSelection; }34:T41e,function getMinMaxPerClass(data) { let minMax = {}; for (let dimension in data) { minMax[dimension] = {}; for (let feature in data[dimension]) { let currentClass = data[dimension][feature].class; let currentData = data[dimension][feature].data; let currentMin, currentMax; if (dimension === "1D") { currentMin = Math.min(...currentData); currentMax = Math.max(...currentData); } else { currentMin = Math.min(...currentData.map((arr) => Math.min(...arr))); currentMax = Math.max(...currentData.map((arr) => Math.max(...arr))); } if (!minMax[dimension][currentClass]) { minMax[dimension][currentClass] = { min: currentMin, max: currentMax }; } else { if (currentMin < minMax[dimension][currentClass].min) { minMax[dimension][currentClass].min = currentMin; } if (currentMax > minMax[dimension][currentClass].max) { minMax[dimension][currentClass].max = currentMax; } } } } return minMax; }35:T46c,function getAlternateColor(data, target) { return "#000000"; // const uniqueTargets = Array.from(new Set(data.map((d) => d.target))); // if (uniqueTargets.length === 2) { // const otherTarget = uniqueTargets.find((t) => t !== target); // return getColor(otherTarget); // } else { // const hexColor = getColor(target); // // Remove the hash symbol if it's present // let hex = hexColor.startsWith("#") ? hexColor.slice(1) : hexColor; // // Ensure the input hex color string is 6 characters long // if (hex.length !== 6) { // throw new Error("Invalid hex color string"); // } // // Convert the hex color string to a number // const colorNumber = parseInt(hex, 16); // // Invert the color number by XOR with the maximum value (0xFFFFFF) // const invertedColorNumber = colorNumber ^ 0xffffff; // // Convert the inverted color number back to a hex string and pad with zeros if needed // const invertedHexColor = invertedColorNumber.toString(16).padStart(6, "0"); // // Add the hash symbol and return the result // return "#" + invertedHexColor; // } }36:T530,function extractAndSortFeatureMaps(featureMaps) { const sortedFeatureMapsByClass = {}; for (const dimension in featureMaps) { for (const key in featureMaps[dimension]) { const featureMap = featureMaps[dimension][key]; const className = featureMap.class; if (!sortedFeatureMapsByClass[className]) { sortedFeatureMapsByClass[className] = []; } // Create a new object to store feature map information let newFeatureMapObject = { key, dimension, importance: featureMap.importance }; // If dimension is "1D", add feature to the newFeatureMapObject if (dimension === "1D") { newFeatureMapObject.feature = featureMap.feature; } // If dimension is "2D", add feature_1 and feature_2 to the newFeatureMapObject else if (dimension === "2D") { newFeatureMapObject.feature_1 = featureMap.feature_1; newFeatureMapObject.feature_2 = featureMap.feature_2; } // Push the newFeatureMapObject to the corresponding class array sortedFeatureMapsByClass[className].push(newFeatureMapObject); } } for (const className in sortedFeatureMapsByClass) { sortedFeatureMapsByClass[className].sort( (a, b) => b.importance - a.importance ); } return sortedFeatureMapsByClass; }37:T513,async function prepareHtmlElements(originalData, featureMaps) { const sortedFeatureMapsByClass = extractAndSortFeatureMaps(featureMaps); const plotsContainer = document.getElementById("plots-container"); // Remove existing elements while (plotsContainer.firstChild) { plotsContainer.removeChild(plotsContainer.firstChild); } const classIndexMap = getClassIndexMap(featureMaps); for (const className in sortedFeatureMapsByClass) { const classColumn = document.createElement("div"); classColumn.classList.add("class-column"); plotsContainer.appendChild(classColumn); const classTitle = document.createElement("h3"); classTitle.textContent = className; // Set the color of the classTitle using the getColor function const classColor = getColor(className, classIndexMap); classTitle.style.color = classColor; classColumn.appendChild(classTitle); classColumn.dataset.class = className; // Create a container element for the bar chart const barChartContainer = document.createElement("div"); barChartContainer.classList.add("bar-chart-container"); classColumn.appendChild(barChartContainer); // Create a separator element const separator = document.createElement("hr"); classColumn.appendChild(separator); } }38:T416,featureMapsByFeature = { const sortedFeatureMapsByClass = extractAndSortFeatureMaps(featureMaps); const featureMapsByFeature = {}; for (const className in sortedFeatureMapsByClass) { // Create an object to store the feature maps by their feature or feature key sortedFeatureMapsByClass[className].forEach((featureMap) => { if (featureMap.dimension === "1D") { // If the featureMap dimension is "1D", use its feature as the key const feature = featureMap.feature; if (!featureMapsByFeature[feature]) { featureMapsByFeature[feature] = []; } featureMapsByFeature[feature].push(featureMap); } else { // Otherwise, create a composite feature key and use it as the key const featureKey = `${featureMap.feature_1} & ${featureMap.feature_2}`; if (!featureMapsByFeature[featureKey]) { featureMapsByFeature[featureKey] = []; } featureMapsByFeature[featureKey].push(featureMap); } }); } return featureMapsByFeature; }39:T829,function generateBarChartObject(classIndexMap) { // First, we need to transform the data into a format that can be used by Vega-Lite const transformedData = transformData(classIndexMap); // Calculate the maximum and minimum importance value across all classes const maxImportance = Math.max( ...transformedData.map((classData) => Math.max(...classData.data.map((d) => d.importance)) ) ); const minImportance = Math.min( ...transformedData.map((classData) => Math.min(...classData.data.map((d) => d.importance)) ) ); // Create an array to store the brush selections for each class const brushSelections = transformedData.map(() => []); // Finally, we create the bar charts const barCharts = transformedData.map((classData, index) => { // Create a selection object that will allow us to capture the user's brush selection const brushName = `brush_${index}`; const brush = vl.selectInterval(brushName).encodings("y"); const chart = vl .markBar() .data({ name: "data", values: classData.data }) .encode( vl.y().fieldN("feature").sort("-x"), vl.x().fieldQ("importance"), vl.color().fieldN("color").scale(null), vl.tooltip([vl.fieldN("feature"), vl.fieldQ("importance")]) ) .select(brush) .width(100); // Render the chart and get the Vega view object const view = chart.toView(); // Listen for changes in the brush selection view.addSignalListener(brushName, function (name, value) { // Update the brush selection for this class brushSelections[index] = value; }); // Append the chart to the appropriate container element const plotsContainer = document.getElementById("plots-container"); const barChartContainer = plotsContainer.querySelector( `[data-class="${classData.className}"] .bar-chart-container` ); view.initialize(barChartContainer); return view; }); // Return the container element and brush selections to render them in Observable return { barCharts, brushSelections }; }3a:T410,async function precomputePlots(originalData, featureMaps, minMaxPerClass) { const sortedFeatureMapsByClass = extractAndSortFeatureMaps(featureMaps); // Precompute and store the plots in an object const precomputedPlots = {}; for (const className in sortedFeatureMapsByClass) { precomputedPlots[className] = {}; for (const featureMap of sortedFeatureMapsByClass[className]) { const { key, dimension } = featureMap; console.log(key); const plot = dimension === "1D" ? await createLinePlot( featureMaps, key, originalData, minMaxPerClass["1D"][className].min, minMaxPerClass["1D"][className].max ) : await createHeatmap( originalData, featureMaps, key, minMaxPerClass["2D"][className].min, minMaxPerClass["2D"][className].max ); precomputedPlots[className][key] = plot; } } return precomputedPlots; }3b:Ta9c,async function renderPlotsInOrderOfImportance( originalData, featureMaps, minMaxPerClass, brushSelections, precomputedPlots // Pass in the precomputedPlots object as an argument ) { const sortedFeatureMapsByClass = extractAndSortFeatureMaps(featureMaps); const plotsContainer = document.getElementById("plots-container"); for (const className in sortedFeatureMapsByClass) { const classColumn = plotsContainer.querySelector( `[data-class="${className}"]` ); // Remove the existing plots const existingPlots = classColumn.querySelectorAll(".cell"); existingPlots.forEach((plot) => plot.remove()); // Get the brush selection for this class const classIndex = classIndexMap[className]; const classBrushSelection = brushSelections[classIndex]; // Filter the feature maps based on the featureSelection let selectedFeatureMaps = sortedFeatureMapsByClass[className].filter( (featureMap) => { if (featureMap.dimension === "1D") { return featureMap.feature.includes(featureSelection); } else { const featureKey = `${featureMap.feature_1} & ${featureMap.feature_2}`; return featureKey.includes(featureSelection); } } ); // If there's a brush selection for this class and it has a feature property, filter the selectedFeatureMaps if (classBrushSelection && classBrushSelection.feature) { // Use the filter function to select the relevant feature maps for the class selectedFeatureMaps = selectedFeatureMaps.filter((featureMap) => { // If the featureMap dimension is "1D", check if its key is included in the brush selection if (featureMap.dimension === "1D") { return classBrushSelection.feature.includes(featureMap.feature); } else { // Otherwise, create a composite feature key and check if it is included in the brush selection const featureKey = `${featureMap.feature_1} & ${featureMap.feature_2}`; return classBrushSelection.feature.includes(featureKey); } }); } for (const featureMap of selectedFeatureMaps) { const { key } = featureMap; // Retrieve the precomputed plot from the stored object instead of calling createLinePlot or createHeatmap const plot = precomputedPlots[className][key]; // Create a new div element for the cell and add the 'cell' class const cellDiv = document.createElement("div"); cellDiv.classList.add("cell"); // Append the Vega-Lite chart container to the cell div cellDiv.appendChild(plot); // Append the cell div to the classColumn element classColumn.appendChild(cellDiv); } } }3c:T10f4,function createLinePlot( featureMaps, featureMapKey, originalData, yMin = null, yMax = null, width = 150, height = 150 ) { const { plotData, histogramData } = preprocess1DData( featureMaps, featureMapKey, originalData ); const featureName = featureMaps["1D"][featureMapKey].feature; const className = featureMaps["1D"][featureMapKey].class; const classIndex = featureMaps["1D"][featureMapKey].classIndex; const score = featureMaps["1D"][featureMapKey].importance; const xValues = data.map((d) => d[featureName]); const minXValue = Math.min(...xValues); const maxXValue = Math.max(...xValues); // Calculate yMin and yMax if not provided if (yMin === null || yMax === null) { const yValues = plotData.map((d) => d.y); yMin = Math.min(...yValues); yMax = Math.max(...yValues); } const linePlot = vl .markLine() // Make a line chart .data(plotData) // Using the provided data .encode( vl.color().value("black"), vl.x().fieldQ("x").title(featureName), // For x, use the x field and set the title vl .y() .fieldQ("y") .title("Output") .scale({ domain: [yMin, yMax] }), // For y, use the y field and set the scale vl.tooltip().fieldN("class") // For tooltips, show the class field ); const numBins = 20; // Calculate the bin size const binSize = (maxXValue - minXValue) / numBins; // Create an array of bin edges const binEdges = Array.from( { length: numBins + 1 }, (_, i) => minXValue + i * binSize ); // Determine the number of target classes const targetClasses = [...new Set(originalData.map((d) => d.target))].sort( (a, b) => a - b ); // Create an object to store the bins for each class const bins = targetClasses.reduce((acc, target) => { acc[target] = binEdges.slice(0, -1).map((binStart, i) => ({ binStart, binEnd: binEdges[i + 1], count: 0, class: target // Encode the target class in the bins' data })); return acc; }, {}); // Iterate over the data and increment the count for the appropriate bin and class originalData.forEach((d) => { const binIndex = Math.floor((d[featureName] - minXValue) / binSize); if (binIndex >= 0 && binIndex < numBins) { bins[d.target][binIndex].count += 1; } }); // Find the maximum bin size let maxBinSize = 0; targetClasses.forEach((target) => { bins[target].forEach((bin) => { maxBinSize = Math.max(maxBinSize, bin.count); }); }); // Dynamically create histograms for each target class const histograms = targetClasses.map((target, index) => { const startY = (index * height) / targetClasses.length; const endY = ((index + 1) * height) / targetClasses.length; return vl .markBar() .data( bins[target].map((bin) => ({ ...bin, target // Add the target field to the bins' data })) ) .encode( vl.y().value(startY), // Set the starting y-value based on the current index vl.y2().value(endY), // Set the ending y-value based on the current index vl.x().fieldQ("binStart"), vl.x2().fieldQ("binEnd"), vl .opacity() .fieldQ("count") .scale({ domain: [0, maxBinSize], range: [0, 1] }), // .legend({ // title: "Count", // values: [ // roundToNearestBase(maxBinSize * 0.1, 5), // roundToNearestBase(maxBinSize / 2, 5), // roundToNearestBase(maxBinSize * 0.9, 5) // ] // }), vl .color() .fieldN("target") .scale({ domain: targetClasses.map((target) => target), // Map target indices to class names range: targetClasses.map((target) => getColor(target)) }) .legend(null) ); }); const zeroLine = vl .markRule({ strokeDash: [5, 5] }) // Make a dashed line .data([{ y: 0 }]) // At y=0 .encode(vl.y().fieldQ("y")); return vl .layer(...histograms, linePlot, zeroLine) .height(height) .width(width) .title({ text: `${featureName} for ${className}, Score: ${score.toFixed(2)}`, color: getColor(classIndex) }) // Set the overall chart title .render(); // Combine the charts using layer }3d:Tc9b,function createHeatmap( originalData, featureMaps, featureMapKey, colorMin = null, colorMax = null, width = 150, height = 150 ) { const { data, feature_1, feature_2 } = preprocess2DData( featureMaps, featureMapKey, originalData ); const featureName1 = featureMaps["2D"][featureMapKey].feature_1; const featureName2 = featureMaps["2D"][featureMapKey].feature_2; const className = featureMaps["2D"][featureMapKey].class; const classIndex = featureMaps["2D"][featureMapKey].classIndex; const score = featureMaps["2D"][featureMapKey].importance; if (colorMin === null || colorMax === null) { const values = data.map((d) => d.value); colorMin = Math.min(...values); colorMax = Math.max(...values); } const rowIndexValues = Array.from( new Set(data.map((d) => parseFloat(d.rowIndex))) ).sort((a, b) => a - b); const columnIndexValues = Array.from( new Set(data.map((d) => parseFloat(d.columnIndex))) ).sort((a, b) => a - b); const colorEnd = getColor(classIndex); const colorStart = getAlternateColor(originalData, classIndex); // const colorMidpoint = (colorMin + colorMax) / 2; const colorMidpoint = Math.max(0, colorMin); const xBinSize = (rowIndexValues[rowIndexValues.length - 1] - rowIndexValues[0]) / (rowIndexValues.length - 1); const yBinSize = (columnIndexValues[columnIndexValues.length - 1] - columnIndexValues[0]) / (columnIndexValues.length - 1); // const shiftedData = data.map((d) => ({ // ...d, // rowIndex: d.rowIndex + (0 * xBinSize) / 2, // columnIndex: d.columnIndex - (0 * yBinSize) / 2 // })); const heatmap = vl .markRect() .data(data) .encode( vl .x() .fieldQ("rowIndex") .bin({ maxbins: rowIndexValues.length }) .title(featureName1), vl .y() .fieldQ("columnIndex") .bin({ maxbins: columnIndexValues.length }) .title(featureName2) .scale({ reverse: false }), vl .color() .fieldQ("value") .title("Output") .scale({ domain: [colorMin, colorMidpoint, colorMax], range: [colorStart, "white", colorEnd] }) .legend({ gradientLength: height * 0.75, // Adjust the length of the gradient bar gradientThickness: 10 // Adjust the thickness of the gradient bar }) ); const scatterplot = vl .markPoint() .data(originalData) .encode( vl.x().fieldQ(featureName1).title(featureName1), vl.y().fieldQ(featureName2).title(featureName2).scale({ reverse: false }), vl .color() .fieldN("target") .title("Class") .scale({ domain: Array.from(new Set(originalData.map((d) => d.target))), // Map the domain to class names range: Array.from( new Set(originalData.map((d) => getColor(d.target))) ) }) .legend(null) ); return vl .layer(heatmap, scatterplot) .width(width) .height(height) .title({ text: `${featureName1} & ${featureName2} for ${className}, Score: ${score.toFixed( 2 )}`, color: getColor(classIndex) }) .render(); }3e:[{"id":"6e1d1cc2e35561e296da6ac7d3bad8ebb9cfaa0b211c5387ab4148ef897491cccf57d149e26320f6557b438de71a1d03fedf244c5b385ac2c296249b26407407","href":"https://static.observableusercontent.com/files/6e1d1cc2e35561e296da6ac7d3bad8ebb9cfaa0b211c5387ab4148ef897491cccf57d149e26320f6557b438de71a1d03fedf244c5b385ac2c296249b26407407","name":"iris_0.csv","type":"text/csv","lastModified":1681979118281.808,"size":3705},{"id":"383afbaa4ec8176c22bd31f41edadf46da3e8eed56cb8d90b038c5a5ccf0e8848dcb7e708457de845720cb974044a0c6be28a57f2511cb2524d362b79373fe73","href":"https://static.observableusercontent.com/files/383afbaa4ec8176c22bd31f41edadf46da3e8eed56cb8d90b038c5a5ccf0e8848dcb7e708457de845720cb974044a0c6be28a57f2511cb2524d362b79373fe73","name":"iris_0_feature_maps_ranges.json","type":"application/json","lastModified":1681979119086.867,"size":325735},{"id":"b2af363629d133cecdfa8d6d3a8ecfae1d20b3cff1f08f90f6bb35d5819c6ff97d3a4d0b0fa29d3e5e3dc493c11ed58bd743e14d3df98a6efe9be4c07733b332","href":"https://static.observableusercontent.com/files/b2af363629d133cecdfa8d6d3a8ecfae1d20b3cff1f08f90f6bb35d5819c6ff97d3a4d0b0fa29d3e5e3dc493c11ed58bd743e14d3df98a6efe9be4c07733b332","name":"nam_feature_map_version.png","type":"image/png","lastModified":1681979301478.447,"size":1261880},{"id":"51f9e8725ceee9b78ed3c5ea5aa6aad1a7fa15527e384e5efd186a4d5abc6cfeaa591e4c6c3d80342b442b4e8d137159df960f03e549a68d78e15eb6afea2d36","href":"https://static.observableusercontent.com/files/51f9e8725ceee9b78ed3c5ea5aa6aad1a7fa15527e384e5efd186a4d5abc6cfeaa591e4c6c3d80342b442b4e8d137159df960f03e549a68d78e15eb6afea2d36","name":"nam_image.jpeg","type":"image/jpeg","lastModified":1681979290204.396,"size":468481},{"id":"339b20c77497cd13ea825e79a254d442aa1fb567fc5c868e923df1f6b5eda5f500bd36abcac8f04cacc2a3dd989d35d565c6dc7bfd067f7dede99c445d8da74b","href":"https://static.observableusercontent.com/files/339b20c77497cd13ea825e79a254d442aa1fb567fc5c868e923df1f6b5eda5f500bd36abcac8f04cacc2a3dd989d35d565c6dc7bfd067f7dede99c445d8da74b","name":"xor_0_feature_maps_ranges.json","type":"application/json","lastModified":1681979134601.396,"size":38503},{"id":"be9c6ccfe81a4b55d0bec1e897b531875aceeff462ed024dc9be6cafa73cc3a33843a4d08d66beb9e8384da51133b64d6d0c4b68a6444bddcf02649cbf4ab3e4","href":"https://static.observableusercontent.com/files/be9c6ccfe81a4b55d0bec1e897b531875aceeff462ed024dc9be6cafa73cc3a33843a4d08d66beb9e8384da51133b64d6d0c4b68a6444bddcf02649cbf4ab3e4","name":"xor_0.csv","type":"text/csv","lastModified":1681979134556.259,"size":14212},{"id":"c26bda2d3b4199c8738d7a411027c9c4b5928c57f8da6074335fe590436b5ef95af436ff061c180b6808d71cb0bfb77885f786d16c1cf88b76b51e4dfe6ff5c4","href":"https://static.observableusercontent.com/files/c26bda2d3b4199c8738d7a411027c9c4b5928c57f8da6074335fe590436b5ef95af436ff061c180b6808d71cb0bfb77885f786d16c1cf88b76b51e4dfe6ff5c4","name":"xor_0_feature_maps_permutation.json","type":"application/json","lastModified":1681979134568.361,"size":38503},{"id":"175f22bf9c68d2867c6ce1d2507aa05ce133b4a47f93ece8991cf7002884b45680a4d076f2d6855d5c25bb73ddbb919f9c2c25bb4923057487bd67e89a7316e9","href":"https://static.observableusercontent.com/files/175f22bf9c68d2867c6ce1d2507aa05ce133b4a47f93ece8991cf7002884b45680a4d076f2d6855d5c25bb73ddbb919f9c2c25bb4923057487bd67e89a7316e9","name":"penguins_0_feature_maps_ranges.json","type":"application/json","lastModified":1681979128555.498,"size":882429},{"id":"f7bd5aac575c32407cf2057d496309ee613d0e93094f33b78f4e808eb69c622a4100ab1a068a773e326520c065ff0e6de475b991c7a5412aa885cd7e43d5d252","href":"https://static.observableusercontent.com/files/f7bd5aac575c32407cf2057d496309ee613d0e93094f33b78f4e808eb69c622a4100ab1a068a773e326520c065ff0e6de475b991c7a5412aa885cd7e43d5d252","name":"penguins_0_feature_maps_permutation.json","type":"application/json","lastModified":1681979127993.653,"size":880917},{"id":"96e35c1b3213d472936d546cdee35a314fa8257251193ced13d722ee16023ad24b318a0500b46c330bf3164b97bcd7ecb61177f304722c657c72e75aae1533c2","href":"https://static.observableusercontent.com/files/96e35c1b3213d472936d546cdee35a314fa8257251193ced13d722ee16023ad24b318a0500b46c330bf3164b97bcd7ecb61177f304722c657c72e75aae1533c2","name":"penguins_0.csv","type":"text/csv","lastModified":1681979126352.239,"size":14779},{"id":"a16182995e87e639e7419193b566420b4c5cb9d2cf0e18fb6e10574e821e78b3290c39d315fe2c3830abcc78e2554943c03970cadfbd2cb6f8a58bf3af13754c","href":"https://static.observableusercontent.com/files/a16182995e87e639e7419193b566420b4c5cb9d2cf0e18fb6e10574e821e78b3290c39d315fe2c3830abcc78e2554943c03970cadfbd2cb6f8a58bf3af13754c","name":"iris_0_feature_maps_permutation.json","type":"application/json","lastModified":1681979119009.811,"size":325548}] 2f:{"title":"Visualizing Pairwise Feature Interactions in Neural Additive Models","cells":[{"id":0,"value":"md`# Visualizing Pairwise Feature Interactions in Neural Additive Models`","pinned":false,"hidden":false,"mode":"ojs"},{"id":9,"value":"## Introduction","pinned":false,"hidden":false,"mode":"md"},{"id":11,"value":"Neural Additive Models [(NAMs, Agarwal et al. 2020)](https://neural-additive-models.github.io/) are machine learning models that combine some of the expressivity of Deep Neural Networks (DNNs) with the inherent intelligibility of Generalized Additive Models [(GAMs, Hastie and Tibshirani 1986)](https://projecteuclid.org/euclid.ss/1177013604). NAMs learn a linear combination of neural networks that each attend to a single input feature. These networks are trained jointly and can learn arbitrarily complex relationships between their input feature and the output.\n\nOne of the benefits of NAMs is their ability to visualize feature maps. Since each neural network in a NAM attends to a single input feature, it is possible to plot the output of each network as a function of its input feature. This allows us to see how each feature contributes to the overall prediction and can help us understand how the model is making its decisions:","pinned":false,"hidden":false,"mode":"md"},{"id":14,"value":"html`

\n ${NAMImage}\n ${NAMImage2}\n

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":18,"value":"One significant limitation of NAMs is their inability to learn feature interactions. This can be a critical issue for datasets that require the modeling of interactions between features for accurate prediction. While the XOR problem is commonly used to illustrate the necessity for nonlinearity, it can also aid us in this case:","pinned":false,"hidden":false,"mode":"md"},{"id":20,"value":"html`\n

XOR - Feature 1

\n ${xorPlotFlat1}\n

XOR - Feature 2

\n ${xorPlotFlat2}\n

XOR - Both Features

\n ${xorPlot}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":22,"value":"From the plots above, it is clear that neither of the two features alone can effectively separate the classes. Even if we were to use a weighted sum of two classifiers, each trained on a single feature, it would not be sufficient to solve this problem (unlike with the OR and AND problems). To accurately separate the classes, it is necessary to consider both features simultaneously.","pinned":false,"hidden":false,"mode":"md"},{"id":25,"value":"$31","pinned":false,"hidden":false,"mode":"md"},{"id":27,"value":"md`## Loading the Data`","pinned":false,"hidden":false,"mode":"ojs"},{"id":29,"value":"viewof fileSelection = {\n const inputs = Inputs.select(\n [\n \"Iris, Importance: Output Range\",\n \"Iris, Importance: Permutation Feature Importance\",\n \"Penguins, Importance: Output Range\",\n \"Penguins, Importance: Permutation Feature Importance\",\n \"XOR, Importance: Output Range\",\n \"XOR, Importance: Permutation Feature Importance\"\n ],\n {\n label: \"Data Selection\"\n }\n );\n inputs.addEventListener(\"input\", (event) => {\n console.log(`Selected value: ${event.target.value}`);\n });\n return inputs;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":33,"value":"The data selection dropdown menu allows you to select a dataset and an importance score for the feature maps. Of course, additional scoring approaches could be considered for this methodology such as variance.\n* **Output range of the feature maps:** Feature maps with a high output range contribute strongly to the prediction and are ranked on top.\n* **Custom permutation feature importance:** We apply permutation to the data when it enters individual submodules. The resulting importance tells us the relative increase in prediction error when this feature map is \"removed\" by randomization. With large ensembles of numerous submodules it is also possible that removing a submodule does not increase the error, as the information provided by it is compensated for by other submodules.","pinned":false,"hidden":false,"mode":"md"},{"id":43,"value":"viewof summary_data = SummaryTable(data, { label: \"Data\" })","pinned":false,"hidden":false,"mode":"ojs"},{"id":45,"value":"### Simplified View of the Dataset\nWe display a basic view of the first two features in the data using a scatter plot.","pinned":false,"hidden":false,"mode":"md"},{"id":48,"value":"{\n return Plot.plot({\n marks: [\n Plot.dot(data, {\n x: featureNames[0],\n y: featureNames[1],\n fill: (datum) => getColor(datum.target), // use color map to set point color\n r: 5\n })\n ]\n });\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":50,"value":"# Visualization","pinned":false,"hidden":false,"mode":"md"},{"id":52,"value":"$32","pinned":false,"hidden":false,"mode":"md"},{"id":56,"value":"## Feature Maps Dashboard","pinned":false,"hidden":false,"mode":"md"},{"id":59,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":61,"value":"html`

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":63,"value":"## Function Definitions, Imports, Data Processing","pinned":false,"hidden":false,"mode":"md"},{"id":66,"value":"### Imports","pinned":false,"hidden":false,"mode":"md"},{"id":68,"value":"import { SummaryTable } from \"@observablehq/summary-table\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":70,"value":"import { Legend, Swatches } from \"@d3/color-legend\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":84,"value":"import { vl } from \"@vega/vega-lite-api-v5\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":88,"value":"### Data Processing, HTML, CSS","pinned":false,"hidden":false,"mode":"md"},{"id":90,"value":"renderPlotsInOrderOfImportance(\n data,\n featureMaps,\n minMaxPerClass,\n barChartObject.brushSelections,\n precomputedPlots\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":92,"value":"{\n const plotsContainer = document.getElementById(\"plots-container\");\n const barChartContainers = plotsContainer.querySelectorAll(\n \".bar-chart-container\"\n );\n\n barChartContainers.forEach((container) => {\n container.addEventListener(\"mouseup\", () => {\n console.log(barChartObject.brushSelections);\n renderPlotsInOrderOfImportance(\n data,\n featureMaps,\n minMaxPerClass,\n barChartObject.brushSelections,\n precomputedPlots\n );\n });\n });\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":94,"value":"minMaxPerClass = getMinMaxPerClass(featureMaps)","pinned":false,"hidden":false,"mode":"ojs"},{"id":96,"value":"classIndexMap = getClassIndexMap(featureMaps)","pinned":false,"hidden":false,"mode":"ojs"},{"id":98,"value":"indexClassMap = Object.keys(classIndexMap).reduce((acc, key) => {\n acc[classIndexMap[key]] = key;\n return acc;\n}, {})","pinned":false,"hidden":false,"mode":"ojs"},{"id":100,"value":"html`\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":102,"value":"NAMImage2 = FileAttachment(\"nam_feature_map_version.png\").image()","pinned":false,"hidden":false,"mode":"ojs"},{"id":104,"value":"(NAMImage2.height = 350)","pinned":false,"hidden":false,"mode":"ojs"},{"id":106,"value":"NAMImage = FileAttachment(\"nam_image.jpeg\").image()","pinned":false,"hidden":false,"mode":"ojs"},{"id":108,"value":"(NAMImage.height = 350)","pinned":false,"hidden":false,"mode":"ojs"},{"id":110,"value":"xorPlotFlat2 = {\n // Data for XOR problem with modified y values\n const xorData = [\n { x: 0, y: 0, class: 0 },\n { x: 0.1, y: 0, class: 1 },\n { x: 0, y: 1, class: 1 },\n { x: 0.1, y: 1, class: 0 }\n ];\n\n // Create a scatter plot for XOR problem\n return Plot.plot({\n width: 200,\n height: 200,\n padding: { top: 10, bottom: 30, left: 30, right: 10 },\n x: {\n domain: [-0.1, 1.1],\n ticks: [],\n label: null\n },\n y: {\n domain: [-0.1, 1.1]\n },\n marks: [\n Plot.dot(xorData, {\n x: \"x\",\n y: \"y\",\n fill: (d) => getColor(d.class),\n r: 10\n })\n ]\n });\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":112,"value":"xorPlot = {\n // Data for XOR problem\n const xorData = [\n { x: 0, y: 0, class: 0 },\n { x: 1, y: 0, class: 1 },\n { x: 0, y: 1, class: 1 },\n { x: 1, y: 1, class: 0 }\n ];\n\n // Create a scatter plot for XOR problem\n return Plot.plot({\n title: \"XOR-Problem\",\n width: 200,\n height: 200,\n padding: { top: 30, bottom: 30, left: 30, right: 10 },\n x: {\n domain: [-0.1, 1.1]\n },\n y: {\n domain: [-0.1, 1.1]\n },\n marks: [\n Plot.dot(xorData, {\n x: \"x\",\n y: \"y\",\n fill: (d) => getColor(d.class),\n r: 10\n })\n ]\n });\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":114,"value":"xorPlotFlat1 = {\n // Data for XOR problem with modified y values\n const xorData = [\n { x: 0, y: 0, class: 0 },\n { x: 1, y: 0, class: 1 },\n { x: 0, y: 0.1, class: 1 },\n { x: 1, y: 0.1, class: 0 }\n ];\n\n // Create a scatter plot for XOR problem\n return Plot.plot({\n width: 200,\n height: 200,\n padding: { top: 10, bottom: 30, left: 30, right: 10 },\n x: {\n domain: [-0.1, 1.1]\n },\n y: {\n domain: [-0.1, 1.1],\n ticks: [],\n label: null\n },\n marks: [\n Plot.dot(xorData, {\n x: \"x\",\n y: \"y\",\n fill: (d) => getColor(d.class),\n r: 10\n })\n ]\n });\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":116,"value":"featureNames = {\n if (data.length > 0) {\n const firstObject = data[0];\n const keys = Object.keys(firstObject);\n return keys.slice(0, 2);\n } else {\n return [];\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":118,"value":"prepareHtmlElements(data, featureMaps, minMaxPerClass)","pinned":false,"hidden":false,"mode":"ojs"},{"id":120,"value":"files = {\n return {\n \"XOR, Importance: Output Range\": await FileAttachment(\"xor_0.csv\").csv({\n typed: true\n }),\n \"XOR, Importance: Permutation Feature Importance\": await FileAttachment(\n \"xor_0.csv\"\n ).csv({\n typed: true\n }),\n \"Iris, Importance: Output Range\": await FileAttachment(\"iris_0.csv\").csv({\n typed: true\n }),\n \"Iris, Importance: Permutation Feature Importance\": await FileAttachment(\n \"iris_0.csv\"\n ).csv({ typed: true }),\n \"Penguins, Importance: Output Range\":\n await FileAttachment(\"penguins_0.csv\").csv({ typed: true }),\n \"Penguins, Importance: Permutation Feature Importance\":\n await FileAttachment(\"penguins_0.csv\").csv({ typed: true })\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":125,"value":"featureMapFiles = {\n return {\n \"XOR, Importance: Output Range\": processFeatureMaps(\n await FileAttachment(\"xor_0_feature_maps_ranges.json\").json()\n ),\n \"XOR, Importance: Permutation Feature Importance\": processFeatureMaps(\n await FileAttachment(\"xor_0_feature_maps_permutation.json\").json()\n ),\n \"Iris, Importance: Output Range\": processFeatureMaps(\n await FileAttachment(\"iris_0_feature_maps_ranges.json\").json()\n ),\n \"Iris, Importance: Permutation Feature Importance\": processFeatureMaps(\n await FileAttachment(\"iris_0_feature_maps_permutation.json\").json()\n ),\n \"Penguins, Importance: Output Range\": processFeatureMaps(\n await FileAttachment(\"penguins_0_feature_maps_ranges.json\").json()\n ),\n \"Penguins, Importance: Permutation Feature Importance\": processFeatureMaps(\n await FileAttachment(\"penguins_0_feature_maps_permutation.json\").json()\n )\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":129,"value":"data = files[fileSelection]","pinned":false,"hidden":false,"mode":"ojs"},{"id":131,"value":"featureMaps = featureMapFiles[fileSelection]","pinned":false,"hidden":false,"mode":"ojs"},{"id":133,"value":"barChartObject = generateBarChartObject(classIndexMap)","pinned":false,"hidden":false,"mode":"ojs"},{"id":135,"value":"### Preprocessing Functions","pinned":false,"hidden":false,"mode":"md"},{"id":137,"value":"$34","pinned":false,"hidden":false,"mode":"ojs"},{"id":139,"value":"function processFeatureMaps(data) {\n const newData = { ...data };\n\n for (const dim in newData) {\n for (const key in newData[dim]) {\n const dataObj = newData[dim][key];\n const dataArray = dataObj.data;\n\n if (dim === \"1D\") {\n dataObj.min = Math.min(...dataArray);\n dataObj.max = Math.max(...dataArray);\n } else if (dim === \"2D\") {\n let min = Infinity;\n let max = -Infinity;\n\n for (const arr of dataArray) {\n const arrMin = Math.min(...arr);\n const arrMax = Math.max(...arr);\n\n if (arrMin < min) {\n min = arrMin;\n }\n\n if (arrMax > max) {\n max = arrMax;\n }\n }\n\n dataObj.min = min;\n dataObj.max = max;\n }\n }\n }\n\n return newData;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":141,"value":"function getClassIndexMap(featureMaps) {\n const classIndexMap = {};\n\n for (const dimension in featureMaps) {\n for (const key in featureMaps[dimension]) {\n const featureMap = featureMaps[dimension][key];\n const className = featureMap.class;\n const classIndex = featureMap.classIndex;\n\n if (!classIndexMap.hasOwnProperty(className)) {\n classIndexMap[className] = classIndex;\n }\n }\n }\n\n return classIndexMap;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":143,"value":"function getColor(target, classIndexMap = null) {\n // Convert targets that are \"False\" to 0 and targets that are \"True\" to 1\n if (target === \"False\") {\n target = 0;\n } else if (target === \"True\") {\n target = 1;\n }\n\n // If a classIndexMap is provided and the target is a string, use the map to get the classIndex\n if (classIndexMap && typeof target === \"string\") {\n const className = target;\n if (classIndexMap.hasOwnProperty(className)) {\n target = classIndexMap[className];\n } else {\n throw new Error(\n `Class name '${className}' not found in the classIndexMap.`\n );\n }\n }\n\n // Calculate the index of the color in the schemeCategory10 array\n const index = target % d3.schemeCategory10.length;\n // Return the corresponding color\n return d3.schemeCategory10[index];\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":145,"value":"$35","pinned":false,"hidden":false,"mode":"ojs"},{"id":147,"value":"function minMax2D(data) {\n const flatData = data.flat();\n const values = flatData.map(d => d);\n return {\n min: Math.min(...values),\n max: Math.max(...values)\n };\n}\n","pinned":false,"hidden":false,"mode":"ojs"},{"id":149,"value":"function getScaledValues(data, feature, resolution) {\n const values = data.map(d => d[feature]);\n const min = Math.min(...values);\n const max = Math.max(...values);\n const step = (max - min) / (resolution - 1);\n return Array.from({ length: resolution }, (_, i) => min + i * step);\n};\n","pinned":false,"hidden":false,"mode":"ojs"},{"id":151,"value":"function generateHistogramData(data, feature) {\n return data.map((d) => ({ x: d[feature], class: `Class_${d.target}` }));\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":153,"value":"function preprocess1DData(featureMaps, featureMapKey, originalData) {\n const featureMap = featureMaps[\"1D\"][featureMapKey];\n const dataToPlot = featureMap.data;\n const feature = featureMap.feature;\n\n const scaledXValues = getScaledValues(\n originalData,\n feature,\n dataToPlot.length\n );\n\n const plotData = scaledXValues.map((x, index) => ({\n x: x,\n y: dataToPlot[index],\n class: featureMap.class\n }));\n\n const histogramData = generateHistogramData(originalData, feature);\n return { plotData, histogramData };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":155,"value":"function preprocess2DData(featureMaps, featureMapKey, data) {\n const featureMap = featureMaps[\"2D\"][featureMapKey];\n const feature_1 = featureMap.feature_1;\n const feature_2 = featureMap.feature_2;\n const data2D = featureMap.data;\n const resolution = data2D.length;\n const c = featureMap.class;\n\n const scaledRowValues = getScaledValues(data, feature_1, resolution);\n const scaledColumnValues = getScaledValues(data, feature_2, resolution);\n\n const formattedData = data2D.flatMap((row, rowIndex) =>\n row.map((value, columnIndex) => ({\n rowIndex: scaledRowValues[rowIndex],\n columnIndex: scaledColumnValues[columnIndex],\n value: value\n }))\n );\n\n return { data: formattedData, feature_1, feature_2 };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":157,"value":"$36","pinned":false,"hidden":false,"mode":"ojs"},{"id":159,"value":"function roundToNearestBase(value, base) {\n return Math.round(value / base) * base;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":161,"value":"$37","pinned":false,"hidden":false,"mode":"ojs"},{"id":163,"value":"$38","pinned":false,"hidden":false,"mode":"ojs"},{"id":165,"value":"precomputedPlots = precomputePlots(data, featureMaps, minMaxPerClass)","pinned":false,"hidden":false,"mode":"ojs"},{"id":167,"value":"function transformData(classIndexMap) {\n const transformedData = Object.entries(classIndexMap).map(\n ([className, classIndex]) => {\n // Get the 1D data for this class\n const classData1D = Object.values(featureMaps[\"1D\"]).filter(\n (d) => d.class === className\n );\n\n // Get the 2D data for this class\n const classData2D = Object.values(featureMaps[\"2D\"]).filter(\n (d) => d.class === className\n );\n\n // Combine the 1D and 2D data\n const classData = [\n ...classData1D.map((d) => ({\n feature: d.feature,\n importance: d.importance,\n color: getColor(classIndex)\n })),\n ...classData2D.map((d) => ({\n feature: `${d.feature_1} & ${d.feature_2}`,\n importance: d.importance,\n color: getColor(classIndex)\n }))\n ];\n\n return {\n className,\n data: classData\n };\n }\n );\n \n return transformedData;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":169,"value":"// insert and/or remove tuples using a Vega ChangeSet\n// re-run and return (a promise for) the View\nfunction update(view, name, insert, remove) {\n const cs = vl.vega.changeset();\n if (remove != null) cs.remove(remove);\n if (insert != null) cs.insert(insert);\n return view.change(name, cs).runAsync();\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":171,"value":"### Visualization Functions","pinned":false,"hidden":false,"mode":"md"},{"id":173,"value":"$39","pinned":false,"hidden":false,"mode":"ojs"},{"id":175,"value":"$3a","pinned":false,"hidden":false,"mode":"ojs"},{"id":177,"value":"$3b","pinned":false,"hidden":false,"mode":"ojs"},{"id":179,"value":"$3c","pinned":false,"hidden":false,"mode":"ojs"},{"id":181,"value":"$3d","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W3e","stdlib":"1","titling":"auto"}