31:[{"id":"6cb23074808f8e51af83c9019caa74933608ed6c0451597a1aa800fffec83253f323a29ebd11539a6b28978846586ca59aab8912b979eca407c650697a21b8bf","href":"https://static.observableusercontent.com/files/6cb23074808f8e51af83c9019caa74933608ed6c0451597a1aa800fffec83253f323a29ebd11539a6b28978846586ca59aab8912b979eca407c650697a21b8bf","name":"data_20211002_shortened.json","type":"application/json","lastModified":1633228330546.573,"size":49382389},{"id":"dbf9b97f8ae2e86523fadff56d96cb278be20b5d8be420e0813572ba9aca8d1ca9e8a981a96190f4bd1a12fe5ce4b75f908ee49e4f0ebfea54b8b6ea441ea3d9","href":"https://static.observableusercontent.com/files/dbf9b97f8ae2e86523fadff56d96cb278be20b5d8be420e0813572ba9aca8d1ca9e8a981a96190f4bd1a12fe5ce4b75f908ee49e4f0ebfea54b8b6ea441ea3d9","name":"data.json","type":"application/json","lastModified":1633225907431.366,"size":62797},{"id":"92ad4c9dbe0ba4e1a2a1f5af8469de08684e4903f3689e7be2524ebf42137bdb841600ae98fc9067ab6e485569de06a9d5a2622801d7b6e9830d65f3e62c9eab","href":"https://static.observableusercontent.com/files/92ad4c9dbe0ba4e1a2a1f5af8469de08684e4903f3689e7be2524ebf42137bdb841600ae98fc9067ab6e485569de06a9d5a2622801d7b6e9830d65f3e62c9eab","name":"data_20211002_shortenedB.json","type":"application/json","lastModified":1633288111803.384,"size":161780},{"id":"bb4c5957f252ed06f49904887b63188a7ed482f8b8fec8694cf2232603fd54c17b65eb9c96e927beaccbcb20e9b9e84d4e3d56d7d50a9f524f6bbcbc0be95127","href":"https://static.observableusercontent.com/files/bb4c5957f252ed06f49904887b63188a7ed482f8b8fec8694cf2232603fd54c17b65eb9c96e927beaccbcb20e9b9e84d4e3d56d7d50a9f524f6bbcbc0be95127","name":"data_20211002_shortenedC.json","type":"application/json","lastModified":1633289212888.438,"size":17770190}] 2f:{"title":"Quick Exploration of a selected subset of the DATA.NASA.GOV's data.json file. Not all the metadata but a selection so people can understand the fields that exist in the data schema.","cells":[{"id":0,"value":"md`# Quick Exploration of a selected subset of the DATA.NASA.GOV's data.json file. Not all the metadata but a selection so people can understand the fields that exist in the data schema.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":416,"value":"html `

NOTE: This JSON follows the project open data standard that technically speaking all federal agencies should follow for datasets that get harvested into https://data.gov , meaning this notebook should work for other agency's metadata as well.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":163,"value":"html `

LOADING THE INITIAL DATASET WILL TAKE 3-10 MINUTES, depending on your internet connection speed!

\n
\n

However, due to the data.json that holds all the metadata for data.nasa.gov often being just above 50MB now, it likely will not load at all.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":20,"value":"url2 = \"https://data.nasa.gov/data.json\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":27,"value":"html `URL2 takes 4-5 minutes to fetch when used in a browser!`","pinned":false,"hidden":false,"mode":"ojs"},{"id":22,"value":"data2 = soFetch(url2).then(response => response.json())","pinned":true,"hidden":false,"mode":"ojs"},{"id":441,"value":"html `

DUE TO ISSUES MENTIONED ABOUT, WE'LL NOW USE A FIXED SUBSET OF THE FULL FILE INSTEAD

\n
\n

This only has the first 5726 datsets' metadata as of 2021-10-02. The full file at data.nasa.gov/data.json covers approximately approximately 28,000 datasets (this number varies over time)! We're attaching that subset file to the notebook here.

\n
\n

Despite note being the full file, it should still be useful for understanding the metadata structure,content, and variance.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":428,"value":"data_shortened = FileAttachment(\"data_20211002_shortenedC.json\").json()","pinned":true,"hidden":false,"mode":"ojs"},{"id":92,"value":"createJsonDownloadButton({name: 'data.json from data.nasa.gov/data.json', results: data_shortened}, 'data.json', 'Download the subset of metadata being used as a JSON file')","pinned":false,"hidden":false,"mode":"ojs"},{"id":103,"value":"md `_____`","pinned":false,"hidden":false,"mode":"ojs"},{"id":188,"value":"html `

Here's the raw data. You can click on it to expand it.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":40,"value":"datasets = data_shortened.dataset","pinned":true,"hidden":false,"mode":"ojs"},{"id":289,"value":"md `Total number of datasets is:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":280,"value":"datasets.length","pinned":false,"hidden":false,"mode":"ojs"},{"id":100,"value":"md `-------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":191,"value":"html `

Or download the data.json as a file

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":242,"value":"md `# What domains contain these NASA datasets?`","pinned":false,"hidden":false,"mode":"ojs"},{"id":66,"value":"html `

This function takes all the URLs in the landingPage key and find the strings between the first '//' and next '/' which is generally the domain for data archives, catalogs, datasets, etc.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":61,"value":"domains = GetDomains(datasets,\"landingPage\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":69,"value":"html `Let's test how many of those have 'nasa.gov' in them:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":73,"value":"obj_domains_by_if_nasa = countWithHaveNasaDotGov(domains,\"nasa.gov\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":199,"value":"html `

This many DO have nasa.gov in the domain URL

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":77,"value":"obj_domains_by_if_nasa[\"withNasaDotGov\"]","pinned":true,"hidden":false,"mode":"ojs"},{"id":201,"value":"html `

This many DO NOT have nasa.gov in the domain URL

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":81,"value":"obj_domains_by_if_nasa[\"withoutNasaDotGov\"]","pinned":true,"hidden":false,"mode":"ojs"},{"id":232,"value":"md `# What \"themes\" are used to describe / organize these datasets?`","pinned":false,"hidden":false,"mode":"ojs"},{"id":222,"value":"themes = GetValuesForMetadataKey(datasets,\"theme\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":228,"value":"themes_unique_counts = getUniqueValuesForMetadataKey(datasets,\"theme\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":253,"value":"md `# What \"keywords\" are used to describe / organize these datasets?`","pinned":false,"hidden":false,"mode":"ojs"},{"id":258,"value":"keywords_unique_counts = getUniqueValuesForMetadataKey(datasets,\"keyword\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":356,"value":"md `# What \"publisher\" are used to describe / organize these datasets?`","pinned":false,"hidden":false,"mode":"ojs"},{"id":303,"value":"publisher_list = GetValuesForNestedKeys(datasets,\"publisher\",\"name\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":335,"value":"publisher_unique_counts = getUniqueValuesForNestedKey(datasets,\"publisher\",\"name\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":363,"value":"md `# Exploring dates in the metadata`","pinned":false,"hidden":false,"mode":"ojs"},{"id":368,"value":"issued_dates = GetValuesForMetadataKey(datasets,\"issued\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":365,"value":"issued_unique_counts = getUniqueValuesForMetadataKey(datasets,\"issued\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":374,"value":"year_counts_issues = countYears(datasets,\"issued\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":378,"value":"year_counts_modified = countYears(datasets,\"modified\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":397,"value":"check_datasets_with_date_1896 = returnMetadataMatchingKeyValue(datasets,\"modified\",1896)","pinned":true,"hidden":false,"mode":"ojs"},{"id":402,"value":"check_datasets_with_date_1958 = returnMetadataMatchingKeyValue(datasets,\"modified\",1958)","pinned":true,"hidden":false,"mode":"ojs"},{"id":407,"value":"future_datasets_mistake_2100 = returnMetadataMatchingKeyValue(datasets,\"modified\",2100)","pinned":true,"hidden":false,"mode":"ojs"},{"id":206,"value":"html `

Functions used above

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":71,"value":"function countWithHaveNasaDotGov(domains,stringInDomain){\n let withNasaDotGov = []\n let withoutNasaDotGov = []\n for (let i=0; i !arrayB.includes(e))\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":87,"value":"function createJsonDownloadButton(data, filename = 'data.json', buttonTitle = 'Download JSON') {\n const blob = new Blob([JSON.stringify(data, null, 2)], {type: \"text/json\"});\n return DOM.download(blob, filename, buttonTitle);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":161,"value":"function GetDomains(datasets,keyName){\n let set_of_domains = []\n var domains = datasets.map( (value) => value[keyName].split('//').pop().split('/')[0]).filter( (value, index, _arr) => _arr.indexOf(value) == index);\n return domains\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":218,"value":"function GetValuesForMetadataKey(datasets,keyName){\n let set_of_values = []\n var array_of_values = datasets.map( (value) => value[keyName]).filter( (value, index, _arr) => _arr.indexOf(value) == index);\n return array_of_values\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":224,"value":"function getUniqueValuesForMetadataKey(datasets,keyName){\n var key_values = GetValuesForMetadataKey(datasets,keyName).flat()\n const countUnique = key_values => {\n const counts = {};\n for (var i = 0; i < key_values.length; i++) {\n counts[key_values[i]] = 1 + (counts[key_values[i]] || 0);\n };\n return counts;\n };\n var counted_key_values = countUnique(key_values)\n var sortable = [];\n for (var vehicle in counted_key_values) {\n sortable.push([vehicle, counted_key_values[vehicle]]);\n }\n\n sortable.sort(function(a, b) {\n return b[1] - a[1];\n });\n return sortable\n\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":300,"value":"function GetValuesForNestedKeys(datasets,primaryKey,secondaryKey){\n let set_of_values = []\n var array_of_objects = datasets.map( (value) => value[primaryKey]).filter( (value, index, _arr) => _arr.indexOf(value) == index);\n var array_just_names = []\n for (var i = 0; i < array_of_objects.length; i++) {\n array_just_names.push(array_of_objects[i][secondaryKey])\n }\n return array_just_names\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":332,"value":"function getUniqueValuesForNestedKey(datasets,primaryKey,secondaryKey){\n var key_values = GetValuesForNestedKeys(datasets,primaryKey,secondaryKey).flat()\n const countUnique = key_values => {\n const counts = {};\n for (var i = 0; i < key_values.length; i++) {\n counts[key_values[i]] = 1 + (counts[key_values[i]] || 0);\n };\n return counts;\n };\n var counted_key_values = countUnique(key_values)\n var sortable = [];\n for (var vehicle in counted_key_values) {\n sortable.push([vehicle, counted_key_values[vehicle]]);\n }\n sortable.sort(function(a, b) {\n return b[1] - a[1];\n });\n return sortable\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":370,"value":"function countYears(datasets,dateKey){\n var issued_dates = GetValuesForMetadataKey(datasets,dateKey)\n var year_counts = {}\n for (var i = 0; i < issued_dates.length; i++) {\n var year = 0\n try {\n year = issued_dates[i].slice(0,4)\n }\n catch(err) {\n console.log(\"failed to use slice in issued_dates[i].slice(0,4)\",err.message)\n }\n if(year_counts[year]){\n year_counts[year] = 1 + year_counts[year]\n }\n else{\n year_counts[year] = 1\n }\n }\n return year_counts\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":390,"value":"function returnMetadataMatchingKeyValue(datasets,primaryKey,matchCondition){\n var matching_dataset_items = []\n for (var i = 0; i < datasets.length; i++) {\n if (datasets[i][primaryKey].slice(0,4) == matchCondition){\n matching_dataset_items.push(datasets[i])\n }\n }\n return matching_dataset_items\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":52,"value":"html `

IMPORTS

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":10,"value":" import { soFetch } from '@alecglassford/so-fetch'","pinned":false,"hidden":false,"mode":"ojs"},{"id":47,"value":"html `Thanks to soFetch makers! https://observablehq.com/@alecglassford/so-fetch`","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W31","stdlib":"1","titling":"auto"}