22:null 23:["$","title",null,{"children":"NASA's Open-Source Code Projects: NLP Generated Tags Cluster | Justin Gosses | Observable"}] 28:Tacc,chart1 = { const root = pack(tag_holder); const svg = d3.select(DOM.svg(width, height)) .style("display", "block") .style("margin", "0 -14px") .style("width", "calc(100% + 28px)") .style("height", "auto") .attr("font-size", 10) .attr("font-family", "sans-serif") .attr("text-anchor", "middle"); const tooltip = d3.select("body") .append("div") .style("position", "absolute") .style("visibility", "hidden") .style("color", "white") .style("top", "0px") .style("padding", "8px") .style("background-color", "rgba(0,0,0,.8)") .style("border-radius", "6px") .style("text-align", "center") .style("font-family", "monospace") .style("max-width", "300px") .style("font-size", ".9rem"); const leaf = svg.selectAll("g") .style("font", "10px sans-serif") .attr("pointer-events", "none") .attr("text-anchor", "middle") .data(root.leaves()) .join("g") .attr("transform", d => `translate(${d.x + 1},${d.y + 1})`) .style("fill-opacity", d => d.parent === root ? 1 : 0) .style("display", d => d.parent === root ? "inline" : "none") .text(d => d.data.name); leaf.append("circle") .attr("id", d => (d.leafUid = DOM.uid("leaf")).id) .attr("r", d => d.r) .attr("fill-opacity", 0.7) .attr("fill", d => (d.data.value < 10) ? "#ffc6be" : (d.data.value < 25) ? "#ffa1c5" : (d.data.value < 50) ? "#a773c3" : (d.data.value < 75) ? "#854777" : "#aa000d"); leaf.append("clipPath") .attr("id", d => (d.clipUid = DOM.uid("clip")).id) .append("use") .attr("xlink:href", d => d.leafUid.href); leaf.append("text") .style("font", "10px sans-serif") .attr("clip-path", d => d.clipUid) .selectAll("tspan") .data(d => { return d.data.name.split(/(.*)/g); }) .style("fill-opacity", d => d.parent === root ? 1 : 0) .style("display", d => d.parent === root ? "inline" : "none") .text(d => d.data.name) .join("tspan") .attr("x", 0) .attr("y", (d, i, nodes) => `${i - nodes.length / 2 + .8}em`) .text(d => d) leaf.on("mouseover", function(){return tooltip.style("visibility", "visible");}) .on("mousemove", function(data){ tooltip.html("Tag Name: " + data.data.name + "
" + "Count: " + data.data.value + "
" + "Name of Projects with this Tag:

" + data.data.projects + ""); return tooltip.style("top", (d3.event.pageY+10)+"px").style("left",(d3.event.pageX+10)+"px") }) .on("mouseout", function(){return tooltip.style("visibility", "hidden");}); return svg.node(); }29:T744,// countNLPTagsWNames gets all the tag and group the tags based of project names that utilized this tag. function countNLPTagsWNames(catalog,selected_project_names){ var counts = {} var tag_holder = {} var project = {"name":"NASA Open Source","projects":[]} for (var i = 0; i < catalog.length; i++){ for (var j = 0; j < catalog[i]["Categories_NLP"].length; j++){ var tag = catalog[i]["Categories_NLP"][j] var name = catalog[i]["Software"] var children_1_list = project['projects'] var nlp_tag = catalog[i]["Categories_NLP"][j] //Compare selected_project_name against current project name var isProjNameInSelectedProjectsClusters = false for (var w = 0; w < selected_project_names.length; w++){ if (selected_project_names[w] == name){ isProjNameInSelectedProjectsClusters = true } } if (counts[nlp_tag] && tag_holder[nlp_tag] && isProjNameInSelectedProjectsClusters) { tag_holder[nlp_tag].push(name) counts[nlp_tag]+= 1 var count = counts[nlp_tag] var tags = tag_holder[nlp_tag] children_1_list.push({"names": tags, tag_name: nlp_tag, "value": count}) } else { if (!tag_holder[nlp_tag]){tag_holder[nlp_tag] = [name]} if (!counts[nlp_tag]){counts[nlp_tag] = 1} children_1_list.push({"names": [name], tag_name: nlp_tag, "value": 1}) } } } var results = {"name":"NASA Open Source","children":[]} var tag_array = Object.keys(counts) for (var k = 0; k < tag_array.length; k++){ var tag = tag_array[k] if (counts[tag] >= min_tag_counts && counts[tag] <= max_tag_counts){ results["children"].push({"name":tag,"value":counts[tag],"projects":tag_holder[tag]}) } } return results }2a:[] 26:{"id":"c3d18938993cb1d3","slug":"nasas-open-source-code-projects-nlp-generated-tags-cluster/2","title":"NASA's Open-Source Code Projects: NLP Generated Tags Cluster","thumbnail":"b7cb2d95c54f97027b6645205e79a818d3ca575b7f541898b238fb5c0c05b822","access_level":"public","create_time":"$D2019-10-21T21:53:33.681Z","update_time":"$D2019-12-03T18:27:23.344Z","view_time":null,"first_public_time":"$D2019-12-03T18:27:43.142Z","has_unpublished_edits":null,"fork":{"id":"617acadaa34842ad","slug":"untitled","title":"NASA's Open-Source Code Projects: NLP Generated Tags Cluster","owner":{"id":"28470c75453dbe99","login":"briantoliveira","name":"Elliot Briant"}},"stars":3,"starred":false,"owner":{"type":"team","id":"3168f49a68f07e98","login":"justingosses","name":"Justin Gosses","avatar_url":"https://avatars-next.observableusercontent.com/eba66f9b9de3134c0c22c453a1d562b450a8cf7f2274c4ed1ff16b7455312520","tier":"starter_2024"},"version":1493,"publish_version":1493,"role":null,"authored":false,"authors":[{"type":"individual","id":"44bc5e36abcab180","login":"justingosses","name":"Justin Gosses","avatar_url":"https://avatars-next.observableusercontent.com/eba66f9b9de3134c0c22c453a1d562b450a8cf7f2274c4ed1ff16b7455312520","tier":null}],"members":[{"type":"individual","id":"44bc5e36abcab180","login":"justingosses","name":"Justin Gosses","avatar_url":"https://avatars-next.observableusercontent.com/eba66f9b9de3134c0c22c453a1d562b450a8cf7f2274c4ed1ff16b7455312520","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":9,"collection":{"id":"b7c4119163229a19","slug":"nasa","title":"NASA (mostly forks so go to original)","owner":{"id":"3168f49a68f07e98","login":"justingosses","name":"Justin Gosses"}},"type":"notebook","body":{"title":"NASA's Open-Source Code Projects: NLP Generated Tags Cluster","cells":[{"id":270,"value":"md`# NASA's Open-Source Code Projects: NLP Generated Tags Cluster`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"$28","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":579,"value":"html `

This notebook was created as an alternative way to interact with the data on code.nasa.gov. Because the underlying data on code.nasa.gov is available as a JSON, we can load it in here and slice and dice it anyway we want without being constained by the search methodology available on code.nasa.gov.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":584,"value":"html `
`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":9,"value":"html `

Tag circles are scaled by number of times that term was tagged by natural langauge processing (NLP) model. Training dataset was from NASA's Science Technology and Information program database and input data is the short description of each code project.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":11,"value":"viewof min_tag_counts = slider({\n min: 0,\n max: 50,\n step: 1,\n value: 5,\n format: \",\",\n description:\n \"If more than this number of instances, the NLP tag is shown. Zero to 50, in steps of 1, formatted as a (US) number\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":154,"value":"viewof max_tag_counts = slider({\n min: 0,\n max: 50,\n step: 1,\n value: 50,\n format: \",\",\n description:\n \"If less than this number of instances, NLP tag is shown. Zero to 50, in steps of 1, formatted as a (US) number\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":447,"value":"html `

Each line represents the 5 most represented tags, once tag is selected it creates a cluster representation of projects with similar tags.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":166,"value":"viewof clusters_selected = select({\n description: \"Select a cluster to visualize\",\n options: cluster_descriptions_for_selection,\n multiple: true,\n value:[\"information system,distributed parameter system,distributed processing,control,technology utilization\"]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1188,"value":"nameDisplay = namesDisplay()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":13,"value":"html `

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":15,"value":"html `

Websites that contain NASA Open-Source Code

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":17,"value":"html`

There are a couple different places you can find lists of NASA open-source code.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":19,"value":"html`

The website that attempts to be the larger collection across the whole agency and only lists open-source is Code.NASA.Gov

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":21,"value":"html`

If you work in the federal government or inside NASA and would like to also see some closed or government source only code, you should check out software.nasa.gov as well.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":23,"value":"html `

Where the data we are using in this notebook come from?

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":591,"value":"html `

In the notebook, all the JSON's containing information on open-source NASA code is located here:

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":25,"value":"html`

Code.json is this JSON file which is harvested by code.gov. It contains the NASA open-source code projects metadata that you can find on their site. We also harvest this JSON into code.nasa.gov where it is transformed slightly into catalog.json and as raw project data to build code.nasa.gov.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":27,"value":"html`

Catalog.json is used to build code.nasa.gov. You can find the code repository to build code.nasa.gov here. Please submit an issue if you find any problems.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":29,"value":"html `catalog_clusters_demo_v1 is a json that consists of 15 clusters. Each cluster is a group of projects from catalog.json grouped by similar or shared nlp tags. This clustering work was done in late April 2019 and may or may not overlap perfectly with the latest code.json or catalog.json.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":31,"value":"html `

There is also an API on code.gov that you can used to interact with NASA's open source code projects list.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":33,"value":"html`

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":623,"value":"html `

Importing the data

` ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":59,"value":"html `

The two jsons mentioned above are loaded into this notebook below:

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":62,"value":"html`

First, we'll import d3.js to help up load some files

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":64,"value":"d3 = require(\"d3@5\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":66,"value":"codejson = d3.json('https://raw.githubusercontent.com/nasa/Open-Source-Catalog/master/code.json')","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":73,"value":"catalogjson = d3.json('https://raw.githubusercontent.com/nasa/Open-Source-Catalog//master/catalog.json')","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":75,"value":"catalog_clusters_demo_v0 = d3.json('https://gist.githubusercontent.com/JustinGOSSES/23af5c123a6df8d2c5d86055c56d72c2/raw/d1921435bc5c2e45401dbe53763fc30bfd68fb65/catalog_clusters_demo_v1.json')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":77,"value":"catalog_clusters_demo_v0[0]['keyword_descriptions'] = [\"simulation\",\"computerized simulation\",\"systems simulation\"\n,\"model\", \"autonomy\"]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":79,"value":"catalog_clusters_demo_v1 = catalog_clusters_demo_v0","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":81,"value":"html `

Each Code Project has certain metadata, including tags:

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":86,"value":"html`

NOTE:

Although the differences between code.json and catalog.json are usually just key names and data structure, there is an additional difference in early 2019 as there is some prototyping going on using natural language processing (NLP) to generate keyword tags for each code project. Eventually there will be the same nlp tags on code.json, but for now the NLP generated tags are only on catalog.json due to code.gov, which ingests code.json not being able to yet clear show users what are programmtic vs. human added keyword tags.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":88,"value":"html`

Code.nasa.gov has long had human-generated tags, but like human-generated tags in many places, they suffer from a small number of concept tags and variations in what humans think is important to tag. NLP generated tags can select tags from a standardized list and apply a larger number of tags to each text document in a uniform manner at very large scale.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":94,"value":"html`Catalog.json has a key name of \"Categories_NLP\", these are the tags generated through natural language processing. There is also \"Categories, which are the human-generated tags.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":97,"value":"html `

Exploring the data

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":103,"value":"// Function that gets lists of all the Categories_NLP with count for each tag\nfunction getAllTagClusterDescriptions(catalog_clusters_demo_v1){\n var cluster_descriptions = []\n var cluster_names = []\n const clusters_infos = Object.values(catalog_clusters_demo_v1)\n for (var i = 0; i < clusters_infos.length; i++){\n cluster_descriptions.push(clusters_infos[i][\"keyword_descriptions\"])\n cluster_names.push(clusters_infos[i][\"project_names\"])\n }\n var results = [cluster_descriptions,cluster_names]\n return results\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":107,"value":"cluster_desc_and_name_in_array = getAllTagClusterDescriptions(catalog_clusters_demo_v1)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":109,"value":"cluster_descriptions = cluster_desc_and_name_in_array[0]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":111,"value":"cluster_names = cluster_desc_and_name_in_array[1]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":115,"value":"function make_cluster_descriptions_for_selection(cluster_descriptions){\n // making a list of strings appear as single string so it can work in selection UI at top\n var cluster_descriptions_for_selection = []\n for (var i = 0; i < cluster_descriptions.length; i++){\n var cluster_desc_string = cluster_descriptions[i].join()\n cluster_descriptions_for_selection.push(cluster_desc_string)\n }\n return cluster_descriptions_for_selection\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":118,"value":"cluster_descriptions_for_selection = make_cluster_descriptions_for_selection(cluster_descriptions)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":121,"value":"$29","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":163,"value":"tag_holder = countNLPTagsWNames(catalogjson, selected_project_names)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":169,"value":"// selectTagCluster that select each tag cluster base of cluster tag selected \nfunction selectTagCluster(data,clusters_selected) {\n // split cluster tags into single tags\n var clusters_selected_list = clusters_selected[0].split(\",\")\n var length_ = Object.keys(data).length\n // loop though keys of data length\n for (var i = 0; i < length_; i++) {\n var tag = data[i][\"keyword_descriptions\"]\n var name = data[i][\"project_names\"]\n // loop through all the tags\n for (var k = 0; k < tag.length; k++){\n // compare current tag and clusters_selected_list and return project name\n if (tag[k] == clusters_selected_list[k]){\n return name\n }\n }\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":643,"value":"selected_project_names = selectTagCluster(catalog_clusters_demo_v1, clusters_selected)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":630,"value":"// selectTagClusterLink that get results from selectTagCluster and find project repo link\nfunction selectTagClusterLink(data, selected_project_names) {\n var project_list = []\n // Loop through catalog json and get index name and link\n for (var i = 0; i < data.length; i++) {\n var name = data[i][\"Software\"]\n var link = data[i][\"External Link\"]\n \n // Loop through the selected_project_names array \n for (var k = 0; k < selected_project_names.length; k++) {\n var project_links = project_list\n // Find selected_project_names element in catalog data names\n if (selected_project_names[k] == name) {\n // push project name and link found to an array\n project_links.push({\"name\": selected_project_names[k], \"link\": \"https://code.nasa.gov/?q=\" + link})\n }\n }\n }\n return project_list\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":636,"value":"project_names_with_links = selectTagClusterLink(catalogjson, selected_project_names)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1478,"value":"function namesDisplay() {\n return html`

\n ${ project_names_with_links.map(\n (item) => `

${item.name}

`\n )}\n

`\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":123,"value":"html `

Code for making visual at top:

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":125,"value":"pack = data => d3.pack()\n .size([width, height])\n .padding(3)\n (d3.hierarchy(data)\n .sum(d => d.value)\n .sort((a, b) => b.value - a.value))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":127,"value":"width = 932","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":129,"value":"height = width","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":133,"value":"format = d3.format(\",d\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":137,"value":"color = d3.scaleLinear()\n .domain([0, 5])\n .range([\"hsl(152,80%,80%)\", \"hsl(228,30%,40%)\"])\n .interpolate(d3.interpolateHcl)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":139,"value":"md `### IMPORTS FROM ELSEWHERE`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":141,"value":"html`

Importing the form function from another notebook

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":144,"value":"import {select} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":149,"value":"import {slider} from \"@jashkenas/inputs\"\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":152,"value":"vegalite = require(\"@observablehq/vega-lite@0.1\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W2a","stdlib":"1","titling":"auto"}} 27:"$26:body"