31:T820,aggregationWLMContest = (data) => { return dates.map((dateISO) => { const date = new Date(dateISO); // spot database errors const errors = data.filter((d) => { const type1 = d.first_image_date && !d.wlm_n; const type2 = !d.first_revision; d.error = { type1, type2 }; return type1 || type2; }); const data_noErrors = data.filter((d) => errors.indexOf(d) < 0); // get photographed monuments const photographed = data_noErrors.filter((d) => { const tempDate = new Date(d.first_image_date); const checkDate = tempDate <= date; return d.first_image_date && checkDate; // immagine WLM!!! }); // get inContest monuments const data_noPhotographed = data_noErrors.filter( (d) => photographed.indexOf(d) < 0 ); const inContest = data_noPhotographed.filter((d) => { const tempDate = new Date(d.start); const checkDate = tempDate <= date; return d.start && checkDate; // la data di autorizzazione }); // get onWIki monuments const data_noInContest = data_noPhotographed.filter( (d) => inContest.indexOf(d) < 0 ); const onWIkidata = data_noInContest.filter((d) => { const tempDate = new Date(d.first_revision); const checkDate = tempDate <= date; return d.first_revision && checkDate; // prima revisione pagina wikidata }); // get notYetOnWIkidata const notYetOnWIkidata = data_noInContest.filter( (d) => onWIkidata.indexOf(d) < 0 ); return { date: dateISO, groups_exclusive: { photographed, inContest, onWIkidata }, groups: { photographed, inContest: [photographed, inContest].flat(), onWIkidata: [photographed, inContest, onWIkidata].flat() }, groups_sum: photographed.length + inContest.length + onWIkidata.length, notYetOnWIkidata: notYetOnWIkidata.length, check_sum: photographed.length + inContest.length + onWIkidata.length + notYetOnWIkidata.length + errors.length, errors }; }); }32:T771,aggregationAllPhotosCoverage = (data) => { return dates.map((dateISO) => { const date = new Date(dateISO); // spot database errors const errors = data.filter((d) => { const type1 = d.first_image_date && !d.wlm_n; const type2 = !d.first_revision; d.error = { type1, type2 }; return type1 || type2; }); // get monuments with pictures (photos that can be also unrelated to WLM) const data_noErrors = data.filter((d) => errors.indexOf(d) < 0); const withPicture = data_noErrors.filter((d) => { const check1 = d.first_image_date || d.first_image_date_commons; const first_wlm_image_date = new Date(d.first_image_date); const relevant_image_date = new Date(d.first_image_date_commons); const checkDate = Math.min(first_wlm_image_date, relevant_image_date) <= date; return check1 && checkDate; }); // get all other monuments (which should have no pictures at all) const data_noWithPicture = data_noErrors.filter( (d) => withPicture.indexOf(d) < 0 ); const onWIkidataOnly = data_noWithPicture.filter((d) => { const tempDate = new Date(d.first_revision); const checkDate = tempDate <= date; return d.first_revision && checkDate; // prima revisione pagina wikidata }); // get notYetOnWIkidata const notYetOnWIkidata = data_noWithPicture.filter( (d) => onWIkidataOnly.indexOf(d) < 0 ); return { date: dateISO, groups_exclusive: { withPicture, onWIkidataOnly }, groups: { withPicture, onWIkidataOnly: [withPicture, onWIkidataOnly].flat() }, groups_sum: withPicture.length + onWIkidataOnly.length, notYetOnWIkidata: notYetOnWIkidata.length, check_sum: withPicture.length + onWIkidataOnly.length + notYetOnWIkidata.length + errors.length, errors }; }); }33:Tfde,getCategoriesWLM_failedAttempt01 = (data) => { return dates.map((date) => { const _dateObj = new Date(date); // get photographed const photographed = data.filter((d) => { if (d.first_image_date) { const tempDate = new Date(d.first_image_date); const dateCheck = tempDate <= _dateObj; return d.first_image_date && d.wlm_n && dateCheck; } else { return false; } }); const photographed_Qnumbers = photographed.map((d) => d.q_number); // get inContest const inContest = data.filter((d) => { if (d.start) { const tempDate = new Date(d.start); const dateCheck = tempDate <= _dateObj; return !d.first_image_date && d.wlm_n && dateCheck; } else { return false; } }); const inContest_Qnumbers = inContest.map((d) => d.q_number); // get onWIki const onWIki = data.filter((d) => { // monuments only onWIki const tempDate = new Date(d.first_revision); const dateCheck = tempDate <= _dateObj; return !d.first_image_date && !d.wlm_n && d.wlm_n != "" && dateCheck; }); const onWIki_Qnumbers = onWIki.map((d) => d.q_number); // get errors const errors = data.filter((d) => { const tempDate = new Date(d.first_revision); const dateCheck = tempDate <= _dateObj; return !d.wlm_n && d.first_image_date && dateCheck; }); const errors_Qnumbers = errors.map((d) => d.q_number); // get leftovers const leftovers = data.filter((d) => { const notPhotographed = photographed_Qnumbers.indexOf(d.q_number) < 0; const notInContest = inContest_Qnumbers.indexOf(d.q_number) < 0; const notOnWIki = onWIki_Qnumbers.indexOf(d.q_number) < 0; const notErrors = errors_Qnumbers.indexOf(d.q_number) < 0; const tempDate = new Date(d.first_revision); const dateCheck = tempDate <= _dateObj; return ( notPhotographed && notInContest && notOnWIki && notErrors && dateCheck ); }); const existingMonuments = [ photographed, inContest, onWIki, errors, leftovers ].flat(); const notYetOnWIkidata = data.filter((d) => { const tempDate = new Date(d.first_revision); const dateCheck = tempDate <= _dateObj; return !dateCheck; }); const overlaps = [ photographed.filter(d => notYetOnWIkidata.includes(d)), inContest.filter(d => notYetOnWIkidata.includes(d)), onWIki.filter(d => notYetOnWIkidata.includes(d)), errors.filter(d => notYetOnWIkidata.includes(d)), leftovers.filter(d => notYetOnWIkidata.includes(d)), ] return { date: date, existingMonuments: existingMonuments.length, notYetOnWIkidata: notYetOnWIkidata, generalSum: existingMonuments.length + notYetOnWIkidata.length, overlaps: overlaps, groups: [ { label: "photographed", value: photographed.length, value_exclusive: photographed.length, monuments_exclusive: photographed }, { label: "inContest", value: photographed.length + inContest.length, value_exclusive: inContest.length, monuments_exclusive: inContest }, { label: "onWIki", value: photographed.length + inContest.length + onWIki.length, value_exclusive: onWIki.length, monuments_exclusive: onWIki }, { label: "errors", value: photographed.length + inContest.length + onWIki.length + errors.length, value_exclusive: errors.length, monuments_exclusive: errors }, { label: "leftovers", value: photographed.length + inContest.length + onWIki.length + errors.length + leftovers.length, value_exclusive: leftovers.length, monuments_exclusive: leftovers } ] }; }); }34:[{"id":"04c8bf830f7886309fa1b9ecec191c969fa570771b51ee54754e07a4f0c066fc4c3b11fb05f9c0e26e4c180600593fbe30bb88242dac1d013df0ce773aebc286","href":"https://static.observableusercontent.com/files/04c8bf830f7886309fa1b9ecec191c969fa570771b51ee54754e07a4f0c066fc4c3b11fb05f9c0e26e4c180600593fbe30bb88242dac1d013df0ce773aebc286","name":"monumentiProvinciaMilano.json","type":"application/json","lastModified":1663703811354.375,"size":19756615},{"id":"67876fbc2c0915b35501689c3922316ff84f33c9cbf128a538102fdcdfb568907abbd062bc65abbe5bef2a90311413eb29437036dd7752c23b8cfe456e2487f5","href":"https://static.observableusercontent.com/files/67876fbc2c0915b35501689c3922316ff84f33c9cbf128a538102fdcdfb568907abbd062bc65abbe5bef2a90311413eb29437036dd7752c23b8cfe456e2487f5","name":"data_italy_smaller.json","type":"application/json","lastModified":1663768254305.29,"size":31328648}] 2f:{"title":"Data aggregations for data.wikilovesmonuments.it","cells":[{"id":7,"value":"# Data aggregations for data.wikilovesmonuments.it","pinned":false,"hidden":false,"mode":"md"},{"id":131,"value":"Define two data aggregations related to the project. The first allows to monitor the WLM contest, the latter the general photographic coverage of cultural heritage.\n\nWe group items according to specific conditions and we subtract created groups from the rest of data before creating another group; this solution helps in dealing with data discrepancies and messiness.","pinned":false,"hidden":false,"mode":"md"},{"id":19,"value":"dataLombardia = FileAttachment(\"monumentiProvinciaMilano.json\").json();","pinned":false,"hidden":false,"mode":"ojs"},{"id":277,"value":"dataItalia = FileAttachment(\"data_italy_smaller.json\").json();","pinned":false,"hidden":false,"mode":"ojs"},{"id":448,"value":"We deal with two types of database errors (see code). Corresponding monuments must be removed from the database and harvested from scratches in a new data snapshot.","pinned":false,"hidden":false,"mode":"md"},{"id":169,"value":"### Monitor WLM contest","pinned":false,"hidden":false,"mode":"md"},{"id":208,"value":"$31","pinned":true,"hidden":false,"mode":"ojs"},{"id":351,"value":"#### Lombardia","pinned":false,"hidden":false,"mode":"md"},{"id":49,"value":"history_wlm_lombardia = aggregationWLMContest(dataLombardia)","pinned":false,"hidden":false,"mode":"ojs"},{"id":355,"value":"#### Italia","pinned":false,"hidden":false,"mode":"md"},{"id":284,"value":"history_wlm_italy = aggregationWLMContest(dataItalia)","pinned":false,"hidden":false,"mode":"ojs"},{"id":317,"value":"error_type1 = history_wlm_italy[0].errors.filter(d=>d.error && d.error.type1)","pinned":false,"hidden":false,"mode":"ojs"},{"id":325,"value":"error_type2 = history_wlm_italy[0].errors.filter(d=>d.error && d.error.type2)","pinned":false,"hidden":false,"mode":"ojs"},{"id":174,"value":"### Monitor general photo coverage","pinned":false,"hidden":false,"mode":"md"},{"id":178,"value":"$32","pinned":true,"hidden":false,"mode":"ojs"},{"id":431,"value":"#### Lombardia","pinned":false,"hidden":false,"mode":"md"},{"id":297,"value":"coverageLombardia = aggregationAllPhotosCoverage(dataLombardia)","pinned":false,"hidden":false,"mode":"ojs"},{"id":436,"value":"#### Italia","pinned":false,"hidden":false,"mode":"md"},{"id":299,"value":"coverageItalia = aggregationAllPhotosCoverage(dataItalia)","pinned":false,"hidden":false,"mode":"ojs"},{"id":17,"value":"### Appendix","pinned":false,"hidden":false,"mode":"md"},{"id":46,"value":"dates = [\"2012-01-01\",\"2013-01-01\",\"2014-01-01\",\"2015-01-01\",\"2016-01-01\",\"2017-01-01\",\"2018-01-01\",\"2019-01-01\",\"2020-01-01\",\"2021-01-01\",\"2022-01-01\",\"2023-01-01\"]","pinned":false,"hidden":false,"mode":"ojs"},{"id":9,"value":"urlLombardia = \"https://wlm.inmagik.com/api/monument/?region=3&format=json\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":148,"value":"urlItalia = \"https://wlm.inmagik.com/api/monument/?format=json\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":30,"value":"mutable count = 0","pinned":false,"hidden":false,"mode":"ojs"},{"id":0,"value":"function paginated_fetch(\n url, // Improvised required argument in JS\n page = 1,\n previousResponse = []\n) {\n // console.log(page)\n mutable count = page\n return fetch(`${url}&page=${page}`) // Append the page number to the base URL\n .then((response) => response.json())\n .then((newResponse) => {\n // console.log(previousResponse);\n // console.log(newResponse);\n\n let response = [...previousResponse, ...newResponse.results];\n\n if (newResponse.next != null) {\n page++;\n\n return paginated_fetch(url, page, response);\n }\n\n return response;\n });\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":243,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W34","stdlib":"1","titling":"auto"}