32:Td3e,function flattenJSON(code_json){ // var top_keys = list(Object.keys(code_json[0])) var all_proj = [] for (var i = 0; i < code_json.length; i++){ var proj = code_json[i] var flatp = {} flatp["version"] = proj["version"] flatp["measurementType"] = proj["measurementType"] flatp["agency-name"] = proj["agency"]["name"] flatp["score"] = proj["score"] flatp["name"] = proj["name"] flatp["description"] = proj["description"] flatp["downloadURL"] = proj["downloadURL"] flatp["homepageURL"] = proj["homepageURL"] flatp["languages"] = proj["languages"] flatp["date"] = proj["date"] flatp["searchScore"] = proj["searchScore"] flatp["contact"] = proj["contact"] flatp["disclaimerURL"] = proj["disclaimerURL"] flatp["disclaimerText"] = proj["disclaimerText"] flatp["laborHours"] = proj["laborHours"] flatp["organization"] = proj["organization"] flatp["status"] = proj["status"] flatp["repositoryURL"] = proj["repositoryURL"] flatp["tags"] = proj["tags"] flatp["vcs"] = proj["vcs"] flatp["agency"] = proj["agency"] flatp["agency-requirements-agencyWidePolicy"] = proj["agency"]["requirements"]["agencyWidePolicy"] flatp["agency-requirements-openSourceRequirement"] = proj["agency"]["requirements"]["openSourceRequirement"] flatp["agency-requirements-inventoryRequirement"] = proj["agency"]["requirements"]["inventoryRequirement"] flatp["agency-requirements-schemaFormat"] = proj["agency"]["requirements"]["schemaFormat"] flatp["agency-requirements-overallCompliance"] = proj["agency"]["requirements"]["overallCompliance"] if (typeof(proj["permissions"]) == "undefined"){ flatp["permissions-licenses-name"] = "not_here" flatp["permissions-licenses-url"] = "not_here" } else if (typeof(proj["permissions"]["licenses"]) == "undefined"){ flatp["permissions-licenses-name"] = "not_here" flatp["permissions-licenses-url"] = "not_here" } else { flatp["permissions-licenses-name"] = proj["permissions"]["licenses"][0]["name"] flatp["permissions-licenses-url"] = proj["permissions"]["licenses"][0]["URL"] } all_proj.push(flatp) var keys = Object.keys(flatp) for (var k = 0; k < keys.length; k++){ if (flatp[keys[k]] === undefined){ flatp[keys[k]] = "not_here" } if (Array.isArray(flatp[keys[k]])){ var possible_array = flatp[keys[k]] var possible_array_len = possible_array.length flatp[keys[k]+"-length"] = possible_array_len flatp[keys[k]] = flatp[keys[k]].join() } if (typeof(flatp[keys[k]])=="object" && keys[0] != "0"){ var dict_keys = Object.keys(flatp[keys[k]]) for (var key = 0; key < dict_keys.length; key++){ flatp[keys[k]+"-"+[dict_keys[key]].toString()] = flatp[keys[k]][dict_keys[key]] flatp[keys[k]+"-"+"allparts".toString()] = JSON.stringify(flatp[keys[k]]) } } if (typeof(flatp[keys[k]]) != "string"){ flatp[keys[k]] =flatp[keys[k]].toString() } } flatp["agency-requirements"] = "SeeOtherRequirementValues" if(flatp["agency-complianceDashboard"]==true){ flatp["agency-complianceDashboard"]="true" } else{ flatp["agency-complianceDashboard"]="false" } } return all_proj }33:[] 30:{"title":"Public Engagement with NASA's Open-Source Code Projects on Github","cells":[{"id":0,"value":"md`# Public Engagement with NASA's Open-Source Code Projects on Github`","pinned":false,"hidden":false,"mode":"ojs"},{"id":443,"value":"html `

This table shows which NASA open-source code projects on Github have the most public engagement (exported Jun 2019)

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":115,"value":"md `Select a measure of engagememt to sort & filter on:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":23,"value":"viewof fieldFromGithubPicked = select([\"forks\", \"stargazers\",\"watchers\",\"issues\"])","pinned":false,"hidden":false,"mode":"ojs"},{"id":760,"value":"html `Click on the following words to definitions for forks, watchers,stargazers, and issues.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":36,"value":"viewof min_forks_to_be_included = slider({\n min: 0,\n max: 1500,\n step: 1,\n value: 100,\n format: \",\",\n description:\n \"This slider selects the minimum number of \"+fieldFromGithubPicked+\", a project has to have to be shown in the table or first histogram results below!\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":320,"value":"printTable(selected_nasa_code_on_github)","pinned":false,"hidden":false,"mode":"ojs"},{"id":45,"value":"vegalite({\n height:200,\n width:800,\n data: {values: selected_nasa_code_on_github},\n mark: \"bar\",\n encoding: {\n x: {\"bin\": {\n \"binned\": true,\"step\": 50}, field: fieldFromGithubPicked, type: \"quantitative\"},\n y: {aggregate: \"count\", field: \"*\", type: \"quantitative\"},\n color: {\"value\": \"#ff9900\"}\n }\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":117,"value":"md `

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":147,"value":"html `

It is also useful to look at the unfiltered charts

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":167,"value":"html `

These charts are not limited to repos selected for by the slider above but include everything.

One observation is that there are more people that star a repo, than submit an issue, than start a fork, than select to watch the repo. Another is that the most repositories have a small amount of engagement and a few have very large amounts.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":527,"value":"vegalite({\n height:200,\n width:800,\n data: {values: nasa_code_on_github},\n mark: \"bar\",\n encoding: {\n x: {\"bin\": {\n \"binned\": true,\"step\": 50}, field: \"stargazers\", type: \"quantitative\"},\n y: {aggregate: \"count\", field: \"*\", type: \"quantitative\"},\n color: {\"value\": \"red\"}\n }\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":526,"value":"vegalite({\n height:200,\n width:800,\n data: {values: nasa_code_on_github},\n mark: \"bar\",\n encoding: {\n x: {\"bin\": {\n \"binned\": true,\"step\": 50}, field: \"issues\", type: \"quantitative\"},\n y: {aggregate: \"count\", field: \"*\", type: \"quantitative\"},\n color: {\"value\": \"red\"}\n }\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":538,"value":"vegalite({\n height:200,\n width:800,\n data: {values: nasa_code_on_github},\n mark: \"bar\",\n encoding: {\n x: {\"bin\": {\n \"binned\": true,\"step\": 50}, field: \"forks\", type: \"quantitative\"},\n y: {aggregate: \"count\", field: \"*\", type: \"quantitative\"},\n color: {\"value\": \"red\"}\n }\n })","pinned":false,"hidden":false,"mode":"ojs"},{"id":16,"value":"vegalite({\n height:200,\n width:800,\n data: {values: nasa_code_on_github},\n mark: \"bar\",\n encoding: {\n x: {\"bin\": {\n \"binned\": true,\"step\": 10}, field: \"watchers\", type: \"quantitative\"},\n y: {aggregate: \"count\", field: \"*\", type: \"quantitative\"},\n color: {\"value\": \"red\"}\n }\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":128,"value":"html `

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":322,"value":"md `### Distribution of Public Engagement on Github.com with NASA open-source code`","pinned":false,"hidden":false,"mode":"ojs"},{"id":352,"value":"vl.markPoint().data(nasa_code_on_github).encode(\n vl.size().fieldQ('stargazers').scale( {\"domain\": [1,300]}),\n vl.x().fieldQ('forks'),\n vl.y().fieldQ('issues'),\n vl.color().fieldN('owner'),\n vl.tooltip(['name','owner','stargazers','forks','issues']) // show these fields in a tooltip\n \n).width(800)\n .height(500).render()","pinned":false,"hidden":false,"mode":"ojs"},{"id":613,"value":"html `Some repositories have high forks but low issues. Others have large number of issues but few forks. These differences tell us important things about what type of repository they are and how people interact with them.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":699,"value":"md `----`","pinned":false,"hidden":false,"mode":"ojs"},{"id":722,"value":"viewof min_stars_for_gen_popularity = slider({\n min: 0,\n max: 400,\n step: 10,\n value: 20,\n format: \",\",\n description:\n \"min_stars_for_gen_popularity threshold for both of the tables below, This cutoff lets you ignore some of the repos with small numbers of stars, forks, and issues where the ratios don't mean as much.\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":687,"value":"html `

High Issues vs Forks table

These are projects with many issues relative to their forks. Generally, there are projects under fairly active development or at least have an active user base with problems to solve.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":634,"value":"viewof issues_vs_forks_ratio_threshold_forHighForks = slider({\n min: 0.1,\n max: 20,\n step: 0.05,\n value: 5,\n format: \",\",\n description:\n \"Select minimum issues / forks ratio to be included in table below, default is 5:1 or 5\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":678,"value":"printTable(testtest[\"HighForksVsIssue\"])","pinned":false,"hidden":false,"mode":"ojs"},{"id":685,"value":"md `### High Forks vs Issues table`","pinned":false,"hidden":false,"mode":"ojs"},{"id":640,"value":"viewof forks_vs_issues_ratio = slider({\n min: 0.01,\n max: 20,\n step: 1,\n value: 4,\n format: \",\",\n description:\n \"forks_vs_issues_ratio threshold, selects for high forks vs. issue count, default is 200:100 or 2\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":726,"value":"html `Many of these repositories hold data instead of code or are for working with metadata. Others get included because people fork them to \"save\" them for later. NASA-3D-Resources is an example of this. Others like visionworkbench are more likely to be forked for development reasons`","pinned":false,"hidden":false,"mode":"ojs"},{"id":675,"value":"printTable(testtest[\"HighIssueVsForks\"])","pinned":false,"hidden":false,"mode":"ojs"},{"id":606,"value":"md `-------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":428,"value":"html `for another way to visualize this data using Microsoft's recently open-sourced sanddance tool, check out this notebook`","pinned":false,"hidden":false,"mode":"ojs"},{"id":612,"value":"md `-------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":454,"value":"html `

Now let's go through some of the code we used to make the charts above

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":139,"value":"html `

Get the Data:

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":10,"value":"html `

First, we'll get data on stars, forks, issues, etc. from nasa repositories on Github. This data was extracted using a starter script from data.gov, which is run by GSA. Thanks GSA! They have many code resources here

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":459,"value":"html `NOTE: This data might be slightly out of date as its being pulled from a CSV generated using a notebook that has to be re-run to be accurate!`","pinned":false,"hidden":false,"mode":"ojs"},{"id":21,"value":"stars_data = d3.csv('https://raw.githubusercontent.com/open-nasa-robot/code-gov-stats-jupyter-notebook/master/nasa_data_from_agency-gh-org-stats_notebook/github-nasa-stars-data-2019-04-24%2010%3A48%3A36.984353.csv')","pinned":false,"hidden":false,"mode":"ojs"},{"id":4,"value":"nasa_code_on_github = d3.csv('https://raw.githubusercontent.com/open-nasa-robot/code-gov-stats-jupyter-notebook/master/nasa_data_from_agency-gh-org-stats_notebook/github-nasa-repos-data-2019-04-24%2010%3A48%3A36.984353.csv')","pinned":false,"hidden":false,"mode":"ojs"},{"id":202,"value":"html `

Next, we'll pull in some information on licenses from the code.gov API. It pulls this information from the code.json that NASA and all federal agencies supply code.gov.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":170,"value":"md `We'll call the API twice here as the results are limited to 1000 at a time and the total results are more than that`","pinned":false,"hidden":false,"mode":"ojs"},{"id":172,"value":"url1 = \"https://code-api.app.cloud.gov/api/repos?q=NASA&agency.name=National%20Aeronautics%20and%20Space%20Administration&from=0&api_key=90LWafgJ8bcxpCNHxkhA3jcxbG8TQKlQates3tJ9&size=10000\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":174,"value":"url2 = \"https://code-api.app.cloud.gov/api/repos?q=NASA&agency.name=National%20Aeronautics%20and%20Space%20Administration&from=999&api_key=90LWafgJ8bcxpCNHxkhA3jcxbG8TQKlQates3tJ9&size=10000\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":179,"value":"md `To get all the results we have to include both \"\"&size=10000\" & \"from=999\" in the API call. \"from=999\" means start the results actually given to me at result 999. \"size=10000\" means search for 10000 results no matter how many their actually are and knowing you'll only get 1000 at a time back.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":187,"value":"repos_1 = fetch(url1).then(response => response.json()).then(data => data.repos)","pinned":true,"hidden":false,"mode":"ojs"},{"id":188,"value":"repos_2 = fetch(url2).then(response => response.json()).then(data => data.repos)","pinned":false,"hidden":false,"mode":"ojs"},{"id":193,"value":"md `Combining the results from the two calls with concat. We'll end up getting more code projects than the earlier data import as the earlier data import only gets open-source code projects on github.com/nasa`","pinned":false,"hidden":false,"mode":"ojs"},{"id":194,"value":"repos =repos_1.concat(repos_2)","pinned":false,"hidden":false,"mode":"ojs"},{"id":130,"value":"html `

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":142,"value":"html `

Now that we have the data from a couple different sources, we'll Filter, Sort, and Transform the data:

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":471,"value":"html `You can click to the left of each cell to see the functions that we'll use`","pinned":false,"hidden":false,"mode":"ojs"},{"id":208,"value":"html `

Working with the github.com/nasa data

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":62,"value":"function limitBy(data,attribute,min_threshold){\n var new_data = []\n for(var item in data){\n if (data[item][attribute] >= min_threshold){\n new_data.push(data[item])\n }\n }\n return new_data\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":13,"value":"function getReposWithMoreThanXOfY(nasa_code_on_github,numericFieldName,minValue){\n var results = []\n for (var i = 0; i < nasa_code_on_github.length; i++){\n if (nasa_code_on_github[i][numericFieldName] >= minValue){\n results.push(nasa_code_on_github[i])\n }\n }\n return results\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":20,"value":"repoHasMinForks = getReposWithMoreThanXOfY(nasa_code_on_github,fieldFromGithubPicked,min_forks_to_be_included)","pinned":false,"hidden":false,"mode":"ojs"},{"id":68,"value":"selected_nasa_code_on_github_0 = limitBy(nasa_code_on_github,fieldFromGithubPicked,min_forks_to_be_included)","pinned":false,"hidden":false,"mode":"ojs"},{"id":81,"value":"selected_nasa_code_on_github = selected_nasa_code_on_github_0.sort(function (a, b) {\n return b[fieldFromGithubPicked]- a[fieldFromGithubPicked];\n});","pinned":false,"hidden":false,"mode":"ojs"},{"id":210,"value":"html `

Working with the code.gov data that is harvested from code.nasa.gov

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":215,"value":"mappings = {\n return {\n \"AGPL-3.0\": \"AGPL\",\n \"Apache License 2.0\": \"Apache\",\n \"ALv2\":\"Apache\",\n \"Apache\":\"Apache\",\n \"Apache 2.0\": \"Apache\",\n \"Apache-2.0\": \"Apache\", \n \"BSD-2-Clause\": \"BSD\",\n \"BSD-3-Clause\": \"BSD\",\n \"Creative Commons Zero (CC0)\": \"CC0\",\n \"CC0-1.0\": \"CC0\",\n \"GNU General Public License v2.0\": \"GPL\",\n \"GPLv2\": \"GPL\",\n \"GPL-2.0\": \"GPL\",\n \"GPL-2.1\": \"GPL\",\n \"GPL.v3\": \"GPL\",\n \"GPL-3.0\": \"GPL\",\n \"GPLV2\": \"GPL\",\n \"GPLv3\": \"GPL\",\n \"GPLv6\": \"GPL\",\n \"LGPL-2.1\": \"LGPL\",\n \"LGPL-3.0\": \"LGPL\",\n \"Mozilla Public License\": \"MPL\",\n \"MPL-2.0\": \"MPL\",\n \"NASA Open Source\": \"NASA\",\n \"NASA Open Source Agreement Version 1.3\": \"NASA\",\n \"NASA Open Source v1.3\": \"NASA\",\n \"NASA v3\": \"NASA\"\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":269,"value":"function appendLicenseToProject(selected_nasa_code_on_github,licensesByProject){\n var new_selected_nasa_code_on_github = []\n // licensesByProject = repos \n for (var proj in selected_nasa_code_on_github){\n for (var proj_lic in licensesByProject){\n var repourl_list = licensesByProject[proj_lic][\"repositoryURL\"].split(\"/\")\n var repo_name_from_licenses_obj = repourl_list[repourl_list.length-1]\n if (selected_nasa_code_on_github[proj].name == repo_name_from_licenses_obj){\n new_selected_nasa_code_on_github.push(Object.assign({}, selected_nasa_code_on_github[proj], licensesByProject[proj_lic]))\n // new_selected_nasa_code_on_github.push(repo_name_from_licenses_obj)\n }\n }\n }\n return new_selected_nasa_code_on_github\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":331,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":338,"value":"test = flattenJSON(repos)","pinned":false,"hidden":false,"mode":"ojs"},{"id":654,"value":"html `Let's return code projects that meet certain issues/forks ratios cutoffs and such`","pinned":false,"hidden":false,"mode":"ojs"},{"id":659,"value":"nasa_code_on_github[0].issues","pinned":false,"hidden":false,"mode":"ojs"},{"id":657,"value":"function returnCodeProjWithEngagementThresholds(nasa_code_on_github,issues_vs_forks_ratio_threshold_forHighForks,forks_vs_issues_ratio){\n let proj_threshold_forHighForksVsIssue = []\n let proj_threshold_forHighIssueVsForks = []\n for (var i = 0; i < nasa_code_on_github.length; i++){\n let proj = nasa_code_on_github[i]\n if((parseInt(proj.issues)/parseInt(proj.forks)) > issues_vs_forks_ratio_threshold_forHighForks){\n if(proj.forks > 0 && proj.stargazers > min_stars_for_gen_popularity){proj_threshold_forHighForksVsIssue.push(proj)}\n \n }\n else if((parseInt(proj.forks)/parseInt(proj.issues)) > forks_vs_issues_ratio){\n if(proj.forks > 0 && proj.stargazers > min_stars_for_gen_popularity){proj_threshold_forHighIssueVsForks.push(proj)}\n \n }\n }\n return {\"HighForksVsIssue\":proj_threshold_forHighForksVsIssue,\"HighIssueVsForks\":proj_threshold_forHighIssueVsForks}\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":672,"value":"testtest = returnCodeProjWithEngagementThresholds(nasa_code_on_github,issues_vs_forks_ratio_threshold_forHighForks,forks_vs_issues_ratio,min_stars_for_gen_popularity)","pinned":false,"hidden":false,"mode":"ojs"},{"id":135,"value":"html `

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":131,"value":"html `

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":6,"value":"html `

Import Dependencies:

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":340,"value":"import {vl} from '@vega/vega-lite-api'","pinned":false,"hidden":false,"mode":"ojs"},{"id":26,"value":"import {select} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":27,"value":"import {slider} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":30,"value":"import {form} from \"@mbostock/form-input\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":48,"value":"import {printTable} from '@uwdata/data-utilities'","pinned":false,"hidden":false,"mode":"ojs"},{"id":7,"value":"d3 = require(\"d3@5\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":17,"value":"vegalite = require(\"@observablehq/vega-lite@0.1\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":222,"value":"_ = require('lodash')","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W33","stdlib":"1","titling":"auto"}