32:T64e,chartHumanTags = { let data = human_generated_tags_most_common_obj let height = data.length * 50 + margin.top + margin.bottom const svg = d3.select(DOM.svg(width, height)); let x = d3.scaleLinear() .domain([0, d3.max(data, d => d.value)]) .range([margin.left, width - margin.right]) let y = d3.scaleBand() .domain(data.map(d => d.name)) .range([margin.top, height - margin.bottom]) .padding(0.1) svg.append("g") .attr("fill", "orange") .selectAll("rect") .data(data) .join("rect") .attr("x", x(0)) .attr("y", d => y(d.name)) .attr("width", d => x(d.value) - x(0)) .attr("height", y.bandwidth()); svg.append("g") .attr("fill", "black") .attr("text-anchor", "end") .style("font", "14px sans-serif") .selectAll("text") .data(data) .join("text") .attr("x", d => x(d.value) + 250) .attr("y", d => y(d.name) + y.bandwidth() / 2) .attr("dy", "0.35em") .text(function(d) { var formatted_num = String(d3.format(",.3s")(d.value).replace("G","b")); return (formatted_num)}); svg.append("g") .attr("fill", "black") .attr("text-anchor", "start") .style("font", "14px sans-serif") .selectAll("text") .data(data) .join("text") // .attr("x", d => x(d.value) - 4) .attr("x", 12) .attr("y", d => y(d.name) + y.bandwidth() / 2) .attr("dy", "0.35em") .text(function(d) { return d.name}); svg.append("g") .call(xAxis); svg.append("g") .call(yAxis); return svg.node(); }33:T64d,chartNLPTags = { let data = nlp_generated_tags_most_common_obj let height = data.length * 50 + margin.top + margin.bottom let x = d3.scaleLinear() .domain([0, d3.max(data, d => d.value)]) .range([margin.left, width - margin.right]) let y = d3.scaleBand() .domain(data.map(d => d.name)) .range([margin.top, height - margin.bottom]) .padding(0.1) const svg = d3.select(DOM.svg(width, height)); svg.append("g") .attr("fill", "steelblue") .selectAll("rect") .data(data) .join("rect") .attr("x", x(0)) .attr("y", d => y(d.name)) .attr("width", d => x(d.value) - x(0)) .attr("height", y.bandwidth()); svg.append("g") .attr("fill", "black") .attr("text-anchor", "end") .style("font", "14px sans-serif") .selectAll("text") .data(data) .join("text") .attr("x", d => x(d.value) + 50) .attr("y", d => y(d.name) + y.bandwidth() / 2) .attr("dy", "0.35em") .text(function(d) { // var formatted_num = String(d3.format(",.3s")(d.value).replace("G","b")); return (d.value)}); svg.append("g") .attr("fill", "white") .attr("text-anchor", "start") .style("font", "14px sans-serif") .selectAll("text") .data(data) .join("text") // .attr("x", d => x(d.value) - 4) .attr("x", 12) .attr("y", d => y(d.name) + y.bandwidth() / 2) .attr("dy", "0.35em") .text(function(d) { return d.name}); svg.append("g") .call(xAxis); svg.append("g") .call(yAxis); return svg.node(); }34:[] 30:{"title":"Comparing human-generated vs. A.I.-generated keyword tags on Code.nasa.gov","cells":[{"id":0,"value":"md`# Comparing human-generated vs. A.I.-generated keyword tags on Code.nasa.gov`","pinned":false,"hidden":false,"mode":"ojs"},{"id":340,"value":"html `

Total Number Of Unique Tags: Human vs. A.I Generated

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":347,"value":"text_total_tags = html `

There are ${nlp_generated_tags_all.length} A.I. generated keyword tags vs. only ${human_generated_tags_all.length} human-generated keyword tags for these code projects

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":364,"value":"html `Machine-learning greatly increases the number of tags available for users to filter on`","pinned":false,"hidden":false,"mode":"ojs"},{"id":367,"value":"html `

What about unique tags?

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":376,"value":"text_unique_tags = html `

There are ${nlp_generated_tags_all_counts_sorted.length} unique A.I.-generated keywords vs. ${human_generated_tags_all_counts_sorted.length} unique human-generated keywords

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":374,"value":"text_explain_1 = html `

The ratio between A.I generated tags and human generated tags is a bit smaller when only unique keyword names are considered. This is in line with qualitative observations that the A.I-generated tags tend to use different words to represent the same thing less often compared to human tags. The machine-learning model used is constrained to a fixed list of tags of approximately 7000 keywords.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":371,"value":"human_generated_tags_all_counts_sorted","pinned":true,"hidden":false,"mode":"ojs"},{"id":369,"value":"nlp_generated_tags_all_counts_sorted","pinned":true,"hidden":false,"mode":"ojs"},{"id":343,"value":"header_common_tags_usefullness = html `

How useful are the most common tags?

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":414,"value":"html `

The most common human-generated tags reflect the perspective of the dataset supplier and aren't that useful for end-user

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":550,"value":"html `

By contrast, the NLP tags are less about who wrote the code and why they wrote it and more about what the code is \"about\". We would suggest the most common A.I. generated tags are from a perpsective that is more useful for end-users trying to figure out if the code is potentially useful to them.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":395,"value":"viewof this_many_tags = slider({\n min: 2,\n max: 100,\n step: 1,\n value: 10,\n format: \",\",\n description:\n \"Return the most commong tags in each tag type,only this many\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":500,"value":"header_bar_chart_1 = html `${this_many_tags} Most Common Human-generated tags`","pinned":false,"hidden":false,"mode":"ojs"},{"id":447,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":504,"value":"html `${this_many_tags} Most Common NLP-generated tags`","pinned":false,"hidden":false,"mode":"ojs"},{"id":471,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":586,"value":"header_lists_of_tags = html `

Lists of Tags: Both Counts of Unique tags & Non-unique Lists

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":403,"value":"nlp_generated_tags_most_common = getXmostCommonTags(nlp_generated_tags_all_counts_sorted,this_many_tags)","pinned":false,"hidden":false,"mode":"ojs"},{"id":409,"value":"human_generated_tags_most_common = getXmostCommonTags(human_generated_tags_all_counts_sorted,this_many_tags)","pinned":false,"hidden":false,"mode":"ojs"},{"id":275,"value":"human_generated_tags_all = FindMostCommonHumanTags(codejson_filter_by_date_and_source)","pinned":false,"hidden":false,"mode":"ojs"},{"id":322,"value":"nlp_generated_tags_all = FindMostCommonNLPTags(codejson_filter_by_date_and_source)","pinned":false,"hidden":false,"mode":"ojs"},{"id":489,"value":"md `-----------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":7,"value":"html `

Websites that contain NASA Open-Source Code

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":9,"value":"html`

There are a couple different places you can find lists of NASA open-source code.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":12,"value":"html`

The website that attempts to be the larger collection across the whole agency and only lists open-source is Code.NASA.Gov

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":13,"value":"html`

If you work in the federal government or inside NASA and would like to also see some closed or government source only code, you should check out software.nasa.gov as well

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":15,"value":"html `

JSONs that contain information on open-source NASA code

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":25,"value":"html`

This JSON: https://raw.githubusercontent.com/nasa/Open-Source-Catalog/master/code.json is harvested by code.gov. It contains the NASA open-source (and government-source) code projects metadata that you can find on their site. We also harvest this JSON into code.nasa.gov where it is transformed slightly into catalog.json and as raw project data to build code.nasa.gov.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":28,"value":"html`

Catalog.json is used to build code.nasa.gov. You can find the code repository to build code.nasa.gov here. Please submit an issue if you find any problems.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":216,"value":"html `

Why have both code.json and catalog.json?

Catalog.json is basically code.json with some formatting changed, A.I. tags added, and only contains open-source code with government-only-source code excluded - which is what we need to build code.nasa.gov

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":169,"value":"md `-----------------------------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":39,"value":"md `## Getting the JSONs`","pinned":false,"hidden":false,"mode":"ojs"},{"id":31,"value":"codejson = d3.json('https://raw.githubusercontent.com/nasa/Open-Source-Catalog/master/code.json')","pinned":true,"hidden":false,"mode":"ojs"},{"id":166,"value":"md `-----------------------------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":119,"value":"md `## Filtering them to only certain projects by Licensing AND/OR Update Date`","pinned":false,"hidden":false,"mode":"ojs"},{"id":41,"value":"md `### Select Source Type`","pinned":false,"hidden":false,"mode":"ojs"},{"id":85,"value":"viewof source = radio({\n title: 'License Type',\n description: 'Code on code.json is both source type, code on catalog.json is open-source only!',\n options: [\n { label: 'open-source', value: 'openSource' },\n { label: 'government-source', value: 'governmentWideReuse' },\n { label: 'both', value: 'both'},\n ],\n value: 'both'\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":44,"value":"md `### Select Metadata Changed Date`","pinned":false,"hidden":false,"mode":"ojs"},{"id":62,"value":"viewof date_proj_added_after = date({\n // title: \"filter to projects added after date\", \n min: \"2000-01-01\",\n max: \"2060-12-31\",\n value: \"2010-01-15\",\n description: \"This filters code projects to only those after this date\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":162,"value":"md `-----------------------------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":538,"value":"html `Please ignore the mess below....`","pinned":false,"hidden":false,"mode":"ojs"},{"id":541,"value":"md `-----------------------------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":542,"value":"md `-----------------------------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":544,"value":"md `-----------------------------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":250,"value":"html `

Generating a JSON with counts of each tag type

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":254,"value":"function addKeyForHumanGenTagsCount(codejson_filter_by_date_and_source){\n \n for (var i=0; i< codejson_filter_by_date_and_source.length; i++){\n codejson_filter_by_date_and_source[i][\"humanGenTagCount\"] = codejson_filter_by_date_and_source[i][\"tags\"].length\n codejson_filter_by_date_and_source[i][\"nlpGenTagCount\"] = codejson_filter_by_date_and_source[i][\"sti_keywords_passed_thresholds\"].length\n }\n return codejson_filter_by_date_and_source\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":264,"value":"codejson_filter_by_date_and_source_wTagTotals = addKeyForHumanGenTagsCount(codejson_filter_by_date_and_source)","pinned":false,"hidden":false,"mode":"ojs"},{"id":282,"value":"function FindMostCommonHumanTags(codejson_filter_by_date_and_source){\n let most_common_human_tags = []\n let most_common_nlp_tags = []\n let tag_types = [\"humanGenTagCount\",\"nlpGenTagCount\"]\n let human_tags = []\n let nlp_tags = []\n let keys = []\n let holder = []\n for (var j=0; j< codejson_filter_by_date_and_source.length; j++){\n holder = holder.concat(codejson_filter_by_date_and_source[j][\"tags\"])\n }\n return holder //return [human_tags,nlp_tags]\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":293,"value":"function countStringsInList(arr){\n var counts = {};\n for (var i = 0; i < arr.length; i++) {\n counts[arr[i]] = 1 + (counts[arr[i]] || 0);\n }\n return counts\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":297,"value":"human_generated_tags_all_counts = countStringsInList(human_generated_tags_all)","pinned":false,"hidden":false,"mode":"ojs"},{"id":301,"value":"function sortProperties(obj)\n{\n // convert object into array\n\tvar sortable=[];\n\tfor(var key in obj)\n\t\tif(obj.hasOwnProperty(key))\n\t\t\tsortable.push([key, obj[key]]); // each item is an array in format [key, value]\n\t\n\t// sort items by value\n\tsortable.sort(function(a, b)\n\t{\n\t return a[1]-b[1]; // compare numbers\n\t});\n\treturn sortable; // array in format [ [ key1, val1 ], [ key2, val2 ], ... ]\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":305,"value":"human_generated_tags_all_counts_sorted = sortProperties(human_generated_tags_all_counts)","pinned":false,"hidden":false,"mode":"ojs"},{"id":308,"value":"human_generated_tags_all_counts_sorted.slice(human_generated_tags_all_counts_sorted.length-40,human_generated_tags_all_counts_sorted.length)","pinned":false,"hidden":false,"mode":"ojs"},{"id":319,"value":"function FindMostCommonNLPTags(codejson_filter_by_date_and_source){\n let most_common_human_tags = []\n let most_common_nlp_tags = []\n let tag_types = [\"humanGenTagCount\",\"nlpGenTagCount\"]\n let human_tags = []\n let nlp_tags = []\n let keys = []\n let holder = []\n for (var j=0; j< codejson_filter_by_date_and_source.length; j++){\n holder = holder.concat(codejson_filter_by_date_and_source[j][\"sti_keywords_passed_thresholds\"])\n }\n return holder //return [human_tags,nlp_tags]\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":324,"value":"nlp_generated_tags_all_counts = countStringsInList(nlp_generated_tags_all)","pinned":false,"hidden":false,"mode":"ojs"},{"id":327,"value":"nlp_generated_tags_all_counts_sorted = sortProperties(nlp_generated_tags_all_counts)","pinned":false,"hidden":false,"mode":"ojs"},{"id":330,"value":"nlp_generated_tags_all_counts_sorted.slice(nlp_generated_tags_all_counts_sorted.length-40,nlp_generated_tags_all_counts_sorted.length)","pinned":false,"hidden":false,"mode":"ojs"},{"id":389,"value":"function getXmostCommonTags(tags_sorted,howManyTags){\n return tags_sorted.slice(tags_sorted.length-howManyTags,tags_sorted.length)\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":391,"value":"nlp_generated_tags_all_counts_sorted_12 = getXmostCommonTags(nlp_generated_tags_all_counts_sorted,12)","pinned":false,"hidden":false,"mode":"ojs"},{"id":421,"value":"function convertArrayOfArrayToObjects(arr){\n let new_array_of_obj = []\n for (var i=0; i< arr.length; i++){\n let new_obj = {}\n new_obj[\"name\"] = arr[i][0]\n new_obj[\"value\"] = arr[i][1]\n new_array_of_obj.push(new_obj)\n }\nreturn new_array_of_obj\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":426,"value":"human_generated_tags_most_common_obj = convertArrayOfArrayToObjects(human_generated_tags_most_common)","pinned":false,"hidden":false,"mode":"ojs"},{"id":469,"value":"nlp_generated_tags_most_common_obj = convertArrayOfArrayToObjects(nlp_generated_tags_most_common)","pinned":false,"hidden":false,"mode":"ojs"},{"id":249,"value":"md `-----------------------------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":46,"value":"md `## Results From Filter Selections`","pinned":false,"hidden":false,"mode":"ojs"},{"id":203,"value":"md `### What you must likely want to see is the code.json filtered by the source & date selections above:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":205,"value":"codejson_filter_by_date_and_source","pinned":true,"hidden":false,"mode":"ojs"},{"id":231,"value":"html `

Download the Code Projects Since Date Selected With Selected License Type

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":233,"value":"createJsonDownloadButton({name: 'codejson_filter_by_date_and_source', results: codejson_filter_by_date_and_source}, 'codejson_filter_by_date_and_source.json', 'Download codejson_filter_by_date_and_source as JSON')","pinned":false,"hidden":false,"mode":"ojs"},{"id":243,"value":"html `Other intermediate products`","pinned":false,"hidden":false,"mode":"ojs"},{"id":109,"value":"md `#### code.json filtered by date and/or source`","pinned":false,"hidden":false,"mode":"ojs"},{"id":71,"value":"codejson_array = getReleasesFromCodeJSON(codejson)","pinned":false,"hidden":false,"mode":"ojs"},{"id":98,"value":"codejson_filter_by_source_only = filterCodeJSONBySource(codejson_array,source)","pinned":false,"hidden":false,"mode":"ojs"},{"id":80,"value":"codejson_filter_by_date = filterCodeJSONByDate(codejson,date_proj_added_after)","pinned":false,"hidden":false,"mode":"ojs"},{"id":103,"value":"codejson_filter_by_date_and_source = filterCodeJSONBySource(codejson_filter_by_date,source)","pinned":false,"hidden":false,"mode":"ojs"},{"id":237,"value":"function createJsonDownloadButton(data, filename = 'data.json', buttonTitle = 'Download JSON') {\n const blob = new Blob([JSON.stringify(data, null, 2)], {type: \"text/json\"});\n return DOM.download(blob, filename, buttonTitle);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":189,"value":"md `-----------------------------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":186,"value":"md `-----------------------------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":54,"value":"md `## Functions Used`","pinned":false,"hidden":false,"mode":"ojs"},{"id":141,"value":"function filterCatalogByDate(catalog_array,date){\n return catalog_array.filter(catalog_array => catalog_array[\"Update_Date\"] > date);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":134,"value":"function filterCatalogBySource(catalog_array){\n if (source == \"both\" || source == \"openSource\"){return catalog_array}\n else{\n return []\n }\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":55,"value":"function limitCatalogJsonByDate(catalog){\n var catalog_new = []\n for (var i=0; i< catalog.length; i++){\n // if(new Date(catalog[i][\"Update_Date\"]) <= new Date(date_proj_added_after)){\n if(catalog[i][\"Update_Date\"] >= date_proj_added_after){\n catalog_new.push(catalog[i])\n }\n }\n return catalog_new\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":68,"value":"function getReleasesFromCodeJSON(code_json){\n return code_json[\"releases\"]\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":75,"value":"function filterCodeJSONByDate(codejson,date){\n return codejson_array.filter(codejson_array => codejson_array[\"date\"][\"metadataLastUpdated\"] > date);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":93,"value":"function filterCodeJSONBySource(code_json_array,source){\n if (source == \"both\"){return code_json_array}\n else{\n return code_json_array.filter(code_json_array => code_json_array[\"permissions\"][\"usageType\"] == source)\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":182,"value":"md `-----------------------------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":445,"value":"data = human_generated_tags_most_common_obj","pinned":true,"hidden":false,"mode":"ojs"},{"id":430,"value":"format = d3.format(\".3f\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":431,"value":"x = d3.scaleLinear()\n .domain([0, d3.max(data, d => d.value)])\n .range([margin.left, width - margin.right])\n","pinned":true,"hidden":false,"mode":"ojs"},{"id":433,"value":"y = d3.scaleBand()\n .domain(data.map(d => d.name))\n .range([margin.top, height - margin.bottom])\n .padding(0.1)","pinned":true,"hidden":false,"mode":"ojs"},{"id":436,"value":"xAxis = g => g\n .attr(\"transform\", `translate(0,${margin.top})`)\n .call(d3.axisTop(x).ticks(0))\n .call(g => g.select(\".domain\").remove())","pinned":true,"hidden":false,"mode":"ojs"},{"id":437,"value":"yAxis = g => g\n .attr(\"transform\", `translate(${margin.left},0)`)\n // .attr(\"style\",\n // .call(d3.axisLeft(y).tickSizeOuter(0))\n .call(d3.axisLeft(y).tickFormat(\"\").tickSizeOuter(0))","pinned":true,"hidden":false,"mode":"ojs"},{"id":439,"value":"height = data.length * 50 + margin.top + margin.bottom","pinned":true,"hidden":false,"mode":"ojs"},{"id":441,"value":"margin = ({top: 30, right: 50, bottom: 10, left: 10})","pinned":true,"hidden":false,"mode":"ojs"},{"id":35,"value":"md `## IMPORTS`","pinned":false,"hidden":false,"mode":"ojs"},{"id":4,"value":"d3 = require(\"d3@5\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":83,"value":"import {radio} from \"@jashkenas/inputs\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":64,"value":"import {date} from \"@jashkenas/inputs\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":397,"value":"import {slider} from \"@jashkenas/inputs\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":418,"value":"import {table} from \"@tmcw/tables@513\"","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W34","stdlib":"1","titling":"auto"}