31:T933,chart = { const root = pack(catalogNLPcounts_obj); let focus = root; let view; const svg = d3.select(DOM.svg(width, height)) .attr("viewBox", `-${width / 2} -${height / 2} ${width} ${height}`) .style("display", "block") .style("margin", "0 -14px") .style("width", "calc(100% + 28px)") .style("height", "auto") .style("background", color(0)) .style("cursor", "pointer") .on("click", () => zoom(root)); const node = svg.append("g") .selectAll("circle") .data(root.descendants().slice(1)) .join("circle") .attr("fill", d => d.children ? color(d.depth) : "white") .attr("pointer-events", d => !d.children ? "none" : null) .on("mouseover", function() { d3.select(this).attr("stroke", "#000"); }) .on("mouseout", function() { d3.select(this).attr("stroke", null); }) .on("click", d => focus !== d && (zoom(d), d3.event.stopPropagation())); const label = svg.append("g") .style("font", "10px sans-serif") .attr("pointer-events", "none") .attr("text-anchor", "middle") .selectAll("text") .data(root.descendants()) .join("text") .style("fill-opacity", d => d.parent === root ? 1 : 0) .style("display", d => d.parent === root ? "inline" : "none") .text(d => d.data.name); zoomTo([root.x, root.y, root.r * 2]); function zoomTo(v) { const k = width / v[2]; view = v; label.attr("transform", d => `translate(${(d.x - v[0]) * k},${(d.y - v[1]) * k})`); node.attr("transform", d => `translate(${(d.x - v[0]) * k},${(d.y - v[1]) * k})`); node.attr("r", d => d.r * k); } function zoom(d) { const focus0 = focus; focus = d; const transition = svg.transition() .duration(d3.event.altKey ? 7500 : 750) .tween("zoom", d => { const i = d3.interpolateZoom(view, [focus.x, focus.y, focus.r * 2]); return t => zoomTo(i(t)); }); label .filter(function(d) { return d.parent === focus || this.style.display === "inline"; }) .transition(transition) .style("fill-opacity", d => d.parent === focus ? 1 : 0) .on("start", function(d) { if (d.parent === focus) this.style.display = "inline"; }) .on("end", function(d) { if (d.parent !== focus) this.style.display = "none"; }); } return svg.node(); }32:[] 2f:{"title":"NASA's Open-Source Code Projects","cells":[{"id":0,"value":"md`# NASA's Open-Source Code Projects`","pinned":false,"hidden":false,"mode":"ojs"},{"id":551,"value":"html `THIS IS TEMPORARLY BROKEN DUE TO SOURCE FILE KEYS CHANGING - WILL FIX SOON!`","pinned":false,"hidden":false,"mode":"ojs"},{"id":290,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":321,"value":"viewof min_tag_counts = slider({\n min: 0,\n max: 1500,\n step: 1,\n value: 7,\n format: \",\",\n description:\n \"If more than this number of instances, NLP tag not shown. Zero to 1500, in steps of 5, formatted as a (US) number\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":318,"value":"viewof max_tag_counts = slider({\n min: 0,\n max: 300,\n step: 1,\n value: 300,\n format: \",\",\n description:\n \"If less than this number of instances, NLP tag not shown. Zero to 800, in steps of 5, formatted as a (US) number\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":512,"value":"viewof date_proj_added_after = date({\n // title: \"filter to projects added after date\", \n min: \"2000-01-01\",\n max: \"2060-12-31\",\n value: \"2010-01-15\",\n description: \"This filters code projects to only those after this date\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":349,"value":"html `

Tag circles are scaled by number of times that term was tagged by natural langauge processing (NLP) model. Training dataset was from NASA's Science Technology and Information program database and input data is the short description of each code project.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":352,"value":"md `-----------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":232,"value":"html `

This notebook was created as an alternative way to interact with the data on code.nasa.gov. Because the underlying data on code.nasa.gov is available as a JSON, we can load it in here and slice and dice it anyway we want without being constained by the search methodology available on code.nasa.gov.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":197,"value":"html `

Websites that contain NASA Open-Source Code

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":10,"value":"html`

There are a couple different places you can find lists of NASA open-source code.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":13,"value":"html`

The website that attempts to be the larger collection across the whole agency and only lists open-source is Code.NASA.Gov

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":16,"value":"html`

If you work in the federal government or inside NASA and would like to also see some closed or government source only code, you should check out software.nasa.gov as well

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":194,"value":"html `

JSONs that contain information on open-source NASA code

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2,"value":"html`

This JSON: https://raw.githubusercontent.com/nasa/Open-Source-Catalog/master/code.json is harvested by code.gov. It contains the NASA open-source code projects metadata that you can find on their site. We also harvest this JSON into code.nasa.gov where it is transformed slightly into catalog.json and as raw project data to build code.nasa.gov.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":35,"value":"html`

Catalog.json is used to build code.nasa.gov. You can find the code repository to build code.nasa.gov here. Please submit an issue if you find any problems.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":238,"value":"html `

There is also an API on code.gov that you can used to interact with NASA's open source code projects list.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":201,"value":"md `---------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":186,"value":"html `

The two jsons mentioned above are loaded into this notebook below:

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":24,"value":"html`

First, we'll import d3.js to help up load some files

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":5,"value":"d3 = require(\"d3@5\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":4,"value":"codejson = d3.json('https://raw.githubusercontent.com/nasa/Open-Source-Catalog/master/code.json')","pinned":true,"hidden":false,"mode":"ojs"},{"id":40,"value":"catalogjson0 = d3.json('https://raw.githubusercontent.com/nasa/Open-Source-Catalog/master/catalog.json')","pinned":true,"hidden":false,"mode":"ojs"},{"id":522,"value":"function limitCatalogJsonByDate(catalog){\n var catalog_new = []\n for (var i=0; i< catalog.length; i++){\n // if(new Date(catalog[i][\"Update_Date\"]) <= new Date(date_proj_added_after)){\n if(catalog[i][\"Update_Date\"] >= date_proj_added_after){\n catalog_new.push(catalog[i])\n }\n }\n return catalog_new\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":525,"value":"catalogjson = limitCatalogJsonByDate(catalogjson0)","pinned":true,"hidden":false,"mode":"ojs"},{"id":205,"value":"html `

Each Code Project has certain metadata, including tags

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":44,"value":"html`

NOTE:

Although the differences between code.json and catalog.json are usually just key names and data structure, there is an additional difference in early 2019 as there is some prototyping going on using natural language processing (NLP) to generate keyword tags for each code project. Eventually there will be the same nlp tags on code.json, but for now the NLP generated tags are only on catalog.json due to code.gov, which ingests code.json not being able to yet clear show users what are programmtic vs. human added keyword tags.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":47,"value":"html`

Code.nasa.gov has long had human-generated tags, but like human-generated tags in many places, they suffer from a small number of concept tags and variations in what humans think is important to tag. NLP generated tags can select tags from a standardized list and apply a larger number of tags to each text document in a uniform manner at very large scale.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":51,"value":"html`Catalog.json has a key name of \"Categories_NLP\", these are the tags generated through natural language processing. There is also \"Categories, which are the human-generated tags.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":214,"value":"html `

Exploring the data

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":149,"value":"html`

Function that gets lists of all the Categories_NLP with count for each tag

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":131,"value":"function countNLPTags(catalog){\n var counts = []\n for (var i = 0; i < catalog.length; i++){\n for (var tag in catalog[i][\"Categories_NLP\"]){\n var nlp_tag = catalog[i][\"Categories_NLP\"][tag]\n if (counts[nlp_tag]){\n counts[nlp_tag]+= 1\n }\n else{\n counts[nlp_tag] = 1\n }\n }\n }\n return counts\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":222,"value":"html`

Click on the arrow in front of \"Array\" to expand the list of counts

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":141,"value":"catalogNLPcounts = countNLPTags(catalogjson)","pinned":true,"hidden":false,"mode":"ojs"},{"id":160,"value":"html`

Function that gets lists of all the Categories, those that were human-generated, with counts

`","pinned":true,"hidden":false,"mode":"ojs"},{"id":162,"value":"function countHumanTags(catalog){\n var counts = []\n for (var i = 0; i < catalog.length; i++){\n for (var tag in catalog[i][\"Categories\"]){\n var nlp_tag = catalog[i][\"Categories\"][tag]\n if (counts[nlp_tag]){\n counts[nlp_tag]+= 1\n }\n else{\n counts[nlp_tag] = 1\n }\n }\n }\n return counts\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":226,"value":"html`

Click on the arrow in front of \"Array\" to expand the list of counts

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":165,"value":"catalogHumancounts = countHumanTags(catalogjson)","pinned":true,"hidden":false,"mode":"ojs"},{"id":76,"value":"html`

Function that returns projects with a specific NLP generated tag

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":57,"value":"function getNLPtag(catalog,tag){\n var projs_that_match = []\n for (var i = 0; i < catalog.length; i++){\n var proj = catalog[i]\n if (proj[\"Categories_NLP\"].includes(tag)){\n projs_that_match.push(proj)\n }\n }\n return projs_that_match\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":112,"value":"html`

Examples of nlp tags: computer systems performance, defect,software development tool,java (programming language), computational grid, interface, etc. For more examples, expand count lists above.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":82,"value":"viewof tag = form(html``)","pinned":false,"hidden":false,"mode":"ojs"},{"id":107,"value":"tag","pinned":false,"hidden":false,"mode":"ojs"},{"id":115,"value":"html`

Click on the arrow in front of \"Array\" to expand resulting projects that match

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":68,"value":"projects_with_tag_in_form = getNLPtag(catalogjson,tag['message'])","pinned":false,"hidden":false,"mode":"ojs"},{"id":249,"value":"md `----------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":380,"value":"html `

Although not all NASA open-source code repositories are on github.com, a lot of them are.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":382,"value":"html `

We can use the github API to get statistics on things like forks, stars, and commits in order to gain insight into which repos are most loved by the public!

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":391,"value":"html `

Although we could use the github API here in this notebook, we'll instead pull in a data extract in CSV form that was created by a Jupyter Notebook running Python built by the U.S. General Services Administration or GSA. The GSA writes a lot of open-source code used by other government agencies & the public. They also run code.gov

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":396,"value":"nasa_code_on_github = d3.csv('https://raw.githubusercontent.com/open-nasa-robot/code-gov-stats-jupyter-notebook/master/nasa_data_from_agency-gh-org-stats_notebook/github-nasa-repos-data-2019-04-24%2010%3A48%3A36.984353.csv')","pinned":false,"hidden":false,"mode":"ojs"},{"id":401,"value":"html `

Let's see which repos have the 5 highest star counts

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":404,"value":"function getReposWithMoreThanXOfY(nasa_code_on_github,numericFieldName,minValue){\n var results = []\n for (var i = 0; i < nasa_code_on_github.length; i++){\n if (nasa_code_on_github[i][numericFieldName] >= minValue){\n results.push(nasa_code_on_github[i])\n }\n }\n return results\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":437,"value":"vegalite({\n height:200,\n width:800,\n data: {values: nasa_code_on_github},\n mark: \"bar\",\n encoding: {\n x: {\"bin\": {\n \"binned\": true,\"step\": 5}, field: fieldFromGithubPicked, type: \"quantitative\"},\n y: {aggregate: \"count\", field: \"*\", type: \"quantitative\"}\n }\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":477,"value":"html `How many NASA open-source repos have different number of stars, forks, etc. Binned`","pinned":false,"hidden":false,"mode":"ojs"},{"id":428,"value":"viewof fieldFromGithubPicked = select([\"forks\", \"stargazers\",\"watchers\",\"issues\"])","pinned":false,"hidden":false,"mode":"ojs"},{"id":415,"value":"viewof min_forks_to_be_included = slider({\n min: 0,\n max: 1500,\n step: 1,\n value: 90,\n format: \",\",\n description:\n \"If less than this number of blank, project not shown in results below. Blank is selected from drop-down above.\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":412,"value":"repoHasMinForks = getReposWithMoreThanXOfY(nasa_code_on_github,fieldFromGithubPicked,min_forks_to_be_included)","pinned":false,"hidden":false,"mode":"ojs"},{"id":472,"value":"html `

When are repos added that get a lot of stars? Interestingly they're older

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":446,"value":"vegalite({\n \"width\": 820,\n \"height\": 120,\n data: {values: nasa_code_on_github},\n mark: \"line\",\n encoding: {\n x: {timeUnit: \"year-month\", field: \"created_at\", type: \"temporal\"},\n y: {aggregate: \"sum\", field: \"stargazers\", type: \"quantitative\"}\n }\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":457,"value":"html `

When are stars added?

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":459,"value":"stars_data = d3.csv('https://raw.githubusercontent.com/open-nasa-robot/code-gov-stats-jupyter-notebook/master/nasa_data_from_agency-gh-org-stats_notebook/github-nasa-stars-data-2019-04-24%2010%3A48%3A36.984353.csv')","pinned":false,"hidden":false,"mode":"ojs"},{"id":467,"value":"html `

Distribution of when repos are stared across all repos

` ","pinned":false,"hidden":false,"mode":"ojs"},{"id":462,"value":"vegalite({\n \"width\": 820,\n \"height\": 120,\n data: {values: stars_data},\n mark: \"line\",\n encoding: {\n x: {timeUnit: \"year-month\", field: \"starredAt\", type: \"temporal\"},\n y: {aggregate: \"count\", field: \"repo\", type: \"quantitative\"}\n }\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":481,"value":"html `They don't coincide with when the heavily starred repos first appear, so what is driving it?`","pinned":false,"hidden":false,"mode":"ojs"},{"id":246,"value":"md `----------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":374,"value":"md `-----------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":271,"value":"typeof(catalogNLPcounts)","pinned":false,"hidden":false,"mode":"ojs"},{"id":251,"value":"html `

Code for making visual at top:

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":259,"value":"catalogNLPcounts","pinned":false,"hidden":false,"mode":"ojs"},{"id":277,"value":"Object.keys(catalogNLPcounts)","pinned":false,"hidden":false,"mode":"ojs"},{"id":261,"value":"function turnCountsIntoObj(orig_data){\n var keys = Object.keys(orig_data)\n var count_obj = {\"name\":\"NASA Open Source\",\"children\":[]}\n var test_list = []\n for (var i = 0; i < keys.length; i++){\n var key = keys[i]\n var children_1_list = count_obj['children']\n if (orig_data[key] >= min_tag_counts && orig_data[key] <= max_tag_counts){\n children_1_list.push({\"name\":key,\"value\":orig_data[key]})\n }\n }\n return count_obj\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":264,"value":"catalogNLPcounts_obj = turnCountsIntoObj(catalogNLPcounts)","pinned":true,"hidden":false,"mode":"ojs"},{"id":504,"value":"catalogNLPcounts_obj[\"name\"]","pinned":true,"hidden":false,"mode":"ojs"},{"id":292,"value":"pack = data => d3.pack()\n .size([width, height])\n .padding(3)\n (d3.hierarchy(data)\n .sum(d => d.value)\n .sort((a, b) => b.value - a.value))","pinned":false,"hidden":false,"mode":"ojs"},{"id":294,"value":"width = 932","pinned":true,"hidden":false,"mode":"ojs"},{"id":296,"value":"height = width","pinned":true,"hidden":false,"mode":"ojs"},{"id":298,"value":"format = d3.format(\",d\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":300,"value":"color = d3.scaleLinear()\n .domain([0, 5])\n .range([\"hsl(152,80%,80%)\", \"hsl(228,30%,40%)\"])\n .interpolate(d3.interpolateHcl)","pinned":true,"hidden":false,"mode":"ojs"},{"id":147,"value":"md `### IMPORTS FROM ELSEWHERE`","pinned":false,"hidden":false,"mode":"ojs"},{"id":95,"value":"html`

Importing the form function from another notebook

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":426,"value":"import {select} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":316,"value":"import {slider} from \"@jashkenas/inputs\"\n","pinned":false,"hidden":false,"mode":"ojs"},{"id":90,"value":"import {form} from \"@mbostock/form-input\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":435,"value":"vegalite = require(\"@observablehq/vega-lite@0.1\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":509,"value":"import {date} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":170,"value":"html`

To do:

Get only code projects added or updated within a time range

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":174,"value":"html`

Get back results based on license or hosting location

`","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W32","stdlib":"1","titling":"auto"}