30:T46d,table = function(...tests) { tests = tests.reduce((acc, cur) => acc.concat(cur)); let table = html`

Status	Test	Link

`; let body = table.appendChild(html``); const levels_order = ['error', 'warn', 'ok', 'info']; let ts = tests .map(d => { d.sort_level = d.level; if (d.status) d.sort_level = 'ok'; return d; }) //.sort((x, y) => d3.ascending(x.sort_level, y.sort_level)); .sort((x, y) => { if ( levels_order.indexOf(x.sort_level) > levels_order.indexOf(y.sort_level) ) { return 1; } else { return -1; } }); let items = ''; for (let test of ts) { const link = Object.assign(html`show`, { onclick: e => ( e.preventDefault(), document.getElementById(test.id).scrollIntoView() ) }); const item = html` ${tests_status_char(test.status, test.level)} ${test.msg} ${link} `; body.appendChild(item); } table.appendChild(styles()); return table; }31:T6f5,date_format = { let f = function(data, variable, config = {}) { const { level = 'error', id = 'test_dateformat' + string_hash(variable), format = '%Y-%m-%d', expected = 0 } = config; check_column_existence(data, variable); let date_parse = d3.utcParse(format); let bad_dates = data .map(d => d[variable]) // Remove already parsed dates .filter(d => !(d instanceof Date)) .map(date_parse) .filter(d => d === null); // Status and msg const n_min = Array.isArray(expected) ? d3.min(expected) : expected; const n_max = Array.isArray(expected) ? d3.max(expected) : expected; const status = bad_dates.length >= n_min && bad_dates.length <= n_max; const msg = `Number of dates not recognised with format '${format}' : ${bad_dates.length}.`; return new Test({ msg: msg, status: status, level: level, id: id, show_details: false }); }; // Documentation f.doc = { title: "Check date values format", category: "Date tests", description: '\`tests.date_format\` tries to parse a string column to a date using `d3.utcParse` and the given `config.format` format. The function returns the number of `null` (unrecognised) values after parsing.', args: [ { name: 'data', desc: 'the parsed data object' }, { name: 'variable', desc: 'column name' }, { name: 'config.format', desc: 'date format, passed to `d3.utcParse`', default: '`%Y-%m-%d`' }, { name: 'config.expected', desc: 'expected number of unrecognised values, either as a single number or a `[min, max]` array', default: 0 } ] }; return f; }32:T5bc,date_before = { let f = function(data, variable, config = {}) { config.id = config.id || 'test_datebefore' + string_hash(variable); config.msg_fail = `${variable} values before ${config.date} : %s.`; config.msg_ok = config.msg_fail; if (typeof config.date == 'string') config.date = d3.utcParse('%Y-%m-%d')(config.date); config.test_func = d => d < config.date && d !== null; return check_date_values(data, variable, config); }; // Documentation f.doc = { title: "Check if values are before a given date", category: "Date tests", description: '\`tests.date_before\` is applied to a date column. The test fails if any value is anterior to `config.date`. If the column is a string column, it is parsed using `config.format`', args: [ { name: 'data', desc: 'the parsed data object' }, { name: 'variable', desc: 'date column name' }, { name: 'config.date', desc: 'date to test against, as a Date object or string in `%Y-%m-%d` format', default: '' }, { name: 'config.format', desc: 'format with which to parse string values, if any', default: '`%Y-%m-%d`' }, { name: 'config.expected', desc: 'expected number of date values before the given date, either as a single number or a `[min, max]` array', default: 0 } ] }; return f; }33:T5bd,date_after = { let f = function(data, variable, config = {}) { config.id = config.id || 'test_dateafter' + string_hash(variable); config.msg_fail = `${variable} values after ${config.date} : %s.`; config.msg_ok = config.msg_fail; if (typeof config.date == 'string') config.date = d3.utcParse('%Y-%m-%d')(config.date); config.test_func = d => d > config.date && d !== null; return check_date_values(data, variable, config); }; // Documentation f.doc = { title: "Check if dates values are after a given date", category: "Date tests", description: '\`tests.date_after\` is applied to a date column. The test fails if any value is posterior to `config.date`. If the column is a string column, it is parsed using `config.format`', args: [ { name: 'data', desc: 'the parsed data object' }, { name: 'variable', desc: 'date column name' }, { name: 'config.date', desc: 'date to test against, as a Date object or string in `%Y-%m-%d` format', default: '' }, { name: 'config.format', desc: 'format with which to parse string values, if any', default: '`%Y-%m-%d`' }, { name: 'config.expected', desc: 'expected number of date values after the given date, either as a single number or a `[min, max]` array', default: 0 } ] }; return f; }34:Tafc,duplicates = { let f = function(data, variable, config = {}) { const { level = 'warn', id = 'test_duplicates' + string_hash(variable), max_dist = 1, expected = 0 } = config; check_column_existence(data, variable); // Detect duplicates let dupes = []; let values = data.map(d => d[variable]); values = [...new Set(values)]; for (let i in values) { for (let j = 0; j < i; j++) { let dist = levenshtein_dist( values[i].trim().toLowerCase(), values[j].trim().toLowerCase() ); if (dist <= max_dist) dupes.push({ Values: [values[i], values[j]], Type: dist == 0 ? "Spaces / case" : "Typo ?" }); } } dupes = dupes.map(v => { let values = v.Values.map( d => `

${d.replace(
            /(^\s+|\s+$)/g,
            "$1"
          )}

` ); v.Values = values.join("
"); return v; }); // Status and msg const n_min = Array.isArray(expected) ? d3.min(expected) : expected; const n_max = Array.isArray(expected) ? d3.max(expected) : expected; const status = dupes.length >= n_min && dupes.length <= n_max; const msg = dupes.length == 0 ? `No possible duplicates found for ${variable}.` : `Possible duplicates found for ${variable} : ${dupes.length}.`; return new Test({ msg: msg, status: status, level: level, id: id, show_details: dupes.length > 0, details: build_table(dupes), details_summary: 'Show values' }); }; // Documentation f.doc = { title: "Check for duplicated values inside string column", category: "Data values tests", description: `\`tests.duplicates\` compares every values of a string column to check for possible duplicates. Two values are considered duplicates if they are at most at a \`config.max_dist\` Levenshtein distance after being trimmed and converted to lower case. *Warning :* this can be computationally intensive for columns with many different values.`, args: [ { name: 'data', desc: 'the parsed data object' }, { name: 'variable', desc: 'column name' }, { name: 'config.max_dist', desc: 'the maximum distance between two values to be considered duplicates', default: 1 }, { name: 'config.expected', desc: 'expected number of duplicates, either as a single number or a `[min, max]` array', default: 0 }, { name: 'config.n_max', desc: 'maximum number of duplicates for the test to pass', default: 0 } ] }; return f; }35:T45e,numeric_range = { let f = function(data, variable, config = {}) { config.id = config.id || 'test_numericrange' + string_hash(variable); config.msg_fail = `${variable} values not in range : %s.`; config.msg_ok = config.msg_fail; config.values = data.map(d => d[variable]); config.test_func = d => d < config.range[0] || d > config.range[1]; return check_values(config); }; // Documentation f.doc = { title: "Check if numeric values are in a given range", category: "Data values tests", description: '\`tests.numeric_range\` is applied to a numeric column. The test returns the number of values outside of a given range', args: [ { name: 'data', desc: 'the parsed data object' }, { name: 'variable', desc: 'numeric column name' }, { name: 'config.range', desc: 'a length 2 array giving the numeric range', default: '' }, { name: 'config.expected', desc: 'expected number of values outside of the given range', default: 0 } ] }; return f; }36:T8ba,look_for_values = { let f = function(data, variable, config = {}) { let { level = 'warn', id = 'test_valuespresence' + string_hash(variable) + string_hash(config.value.toString()), value = null, expected = true } = config; check_column_existence(data, variable); if (!Array.isArray(value)) value = [value]; const variable_values = [...new Set(data.map(d => d[variable]))]; const found = value.reduce((acc, v) => { if (v instanceof RegExp) { let presence = false; for (let val of variable_values) { if (v.test(val)) { presence = true; break; } } return acc && presence; } return acc && variable_values.includes(v); }, true); // Status and msg const status = found == expected; let msg = found ? `Values

${value.join(
          ', '
        )}

found in ${variable}` : `Values

${value.join(
          ', '
        )}

not found in ${variable}`; if (expected === true) msg += ` (expected : found)`; if (expected === false) msg += ` (expected : not found)`; return new Test({ msg: msg, status: status, level: level, id: id, show_details: false }); }; // Documentation f.doc = { title: "Check presence of value(s) in a column", category: "Data values tests", description: "`tests.look_for_values` is applied to a column of parsed data and checks if it includes some value(s). If several values are provided as an array, the test returns true if all the values are found. A value can also be a RegExp which will then be tested against the variable", args: [ { name: 'data', desc: 'parsed data object' }, { name: 'variable', desc: 'column name' }, { name: 'config.value', desc: 'a single value or an array of values to be looked for : number, string or RegExp', default: '' }, { name: 'config.expected', desc: 'boolean which indicates if values are expected to be found or not', default: true } ] }; return f; }37:T83a,allowed_values = { let f = function(data, variable, config = {}) { let { level = 'warn', id = 'test_allowedvalues' + string_hash(variable) + string_hash(config.values.toString()), values = null, expected = 0 } = config; check_column_existence(data, variable); if (!Array.isArray(values)) values = [values]; const variable_values = [...new Set(data.map(d => d[variable]))]; let not_allowed_values = new Set(); variable_values.forEach(v => { if (!values.includes(v)) not_allowed_values.add(v); }); not_allowed_values = [...not_allowed_values]; const nb_not_allowed = not_allowed_values.length; // Status and msg const status = nb_not_allowed == expected; let msg = `Values not allowed found in ${variable} : ${nb_not_allowed} `; if (!status) msg += ` (expected : ${expected})`; const details = not_allowed_values .map(v => { if (typeof v == "string") return `"${v}"`; if (v === undefined) return "undefined"; if (v === null) return "null"; return v; }) .map(v => `${v}`) .join(", "); return new Test({ msg: msg, status: status, level: level, id: id, show_details: nb_not_allowed > 0, details: html`

${details}

`, details_summary: "Show values" }); }; // Documentation f.doc = { title: "Check if all value(s) in a column are included in a set of allowed values", category: "Data values tests", description: "`tests.allowed_values` is applied to a column of parsed data and checks if all its values are in a list of allowed value(s)", args: [ { name: 'data', desc: 'parsed data object' }, { name: 'variable', desc: 'column name' }, { name: 'config.values', desc: 'an array of allowed values', default: '' }, { name: 'config.expected', desc: 'expected number of not allowed values', default: 0 } ] }; return f; }38:T49e,NaN_values = { let f = function(data, variable, config = {}) { let { level = 'warn', id = 'test_nanvalues' + string_hash(variable), expected = 0 } = config; check_column_existence(data, variable); const nb_nan = data.map(d => d[variable]).filter(v => isNaN(v)).length; // Status and msg const status = nb_nan == expected; let msg = `NaN values found in ${variable} : ${nb_nan}`; if (!status) msg += ` (expected : ${expected})`; return new Test({ msg: msg, status: status, level: level, id: id, show_details: false }); }; // Documentation f.doc = { title: "Check presence of NaN values in a column", category: "Data values tests", description: "`tests.NaN_values is applied to a column of parsed data and checks if it includes NaN value(s)", args: [ { name: 'data', desc: 'parsed data object' }, { name: 'variable', desc: 'column name' }, { name: 'config.expected', desc: 'expected number of NaN values', default: 0 } ] }; return f; }39:T698,n_distinct = { let f = function(data, variable, config = {}) { const { remove_empty = true, level = 'info', id = 'test_ndistinct' + string_hash(variable), expected = null } = config; check_column_existence(data, variable); let values = data.map(d => d[variable]); if (remove_empty) { values = values.filter(d => !(d == '' || d === null)); } const n_values = [...new Set(values)].length; // Status and msg const n_min = Array.isArray(expected) ? d3.min(expected) : expected; const n_max = Array.isArray(expected) ? d3.max(expected) : expected; const status = n_values >= n_min && n_values <= n_max; const msg = `Number of distinct values of ${variable} : ${n_values}` + (expected !== null ? ` (expected : ${expected}).` : '.'); return new Test({ msg: msg, status: status, level: level, id: id, show_details: false }); }; // Documentation f.doc = { title: "Check number of distinct values", category: "Data values tests", description: "`tests.n_distinct` is applied to a column of parsed data and checks the number of distinct values.", args: [ { name: 'data', desc: 'parsed data object' }, { name: 'variable', desc: 'column name' }, { name: 'config.expected', desc: 'expected number of distinct values, either as a single number or a `[min, max]` array', default: '' }, { name: 'config.remove_empty', desc: 'if `true`, do not count empty or `null` values', default: '`true`' } ] }; return f; }3a:T848,discrepancy = { let f = function(data, group, variable, config = {}) { const { level = 'warn', id = 'test_discrepancy' + string_hash(group + variable), expected = 0 } = config; check_column_existence(data, group); check_column_existence(data, variable); // Detect discrepancies let discr = d3 .rollups( data, v => { let values = v.map(d => d[variable]); return [...new Set(values)]; }, d => d[group] ) .filter(d => d[1].length > 1) .map(d => { let res = {}; res[group] = `${d[0]}`; res[`${variable} values`] = d[1].join("
"); return res; }); // status and msg const n_min = Array.isArray(expected) ? d3.min(expected) : expected; const n_max = Array.isArray(expected) ? d3.max(expected) : expected; const status = discr.length >= n_min && discr.length <= n_max; const msg = discr.length == 0 ? `No discrepancy found for ${variable} inside ${group}.` : `Discrepancies found for ${variable} inside ${group} : ${discr.length}.`; return new Test({ msg: msg, status: status, level: level, id: id, show_details: discr.length > 0, details: build_table(discr), details_summary: 'Show discrepancies' }); }; // Documentation f.doc = { title: "Check for values discrepancies inside groups", category: "Data values tests", description: "`tests.discrepancy` checks if the values inside a column is always the same for groups defined by values inside another column.", args: [ { name: 'data', desc: 'parsed data object' }, { name: 'group', desc: 'groups column name' }, { name: 'variable', desc: 'values column name' }, { name: 'config.expected', desc: 'expected number of discrepancies for the test to pass, either as a single number or a `[min, max]` array', default: 0 } ] }; return f; }3b:T891,not_trimmed = { let f = function(data, config = {}) { const { level = 'warn', id = 'test_nottrimmed', expected = 0 } = config; // Detect not trimmed values let variables = colnames(data) .map(col => { let values = [...new Set(data.map(d => d[col]))] .filter(v => typeof v === "string" && v != v.trim()) .map(v => v.replace(/(^\s+|\s+$)/g, "$1") ); values = values.length > 10 ? values.length + ' different values' : values.join("
"); values = values.length > 0 ? { Variable: `${col}`, Values: `${values}` } : null; return values; }) .filter(v => v !== null); // status and msg const n_min = Array.isArray(expected) ? d3.min(expected) : expected; const n_max = Array.isArray(expected) ? d3.max(expected) : expected; const status = variables.length >= n_min && variables.length <= n_max; const msg = variables.length == 0 ? `No variables with not trimmed values.` : `Variables with not trimmed values : ${variables.length}.`; return new Test({ msg: msg, status: status, level: level, id: id, show_details: variables.length > 0, details: build_table(variables), details_summary: 'Show variables' }); }; // Documentation f.doc = { title: "Detect values with spaces at start or end", category: "Data values tests", description: "`tests.not_trimmed` is applied to parsed data and checks all the string values to detect if there are spaces at start or end.", args: [ { name: 'data', desc: 'parsed data object' }, { name: 'config.expected', desc: 'expected number of values with spaces, either as a single number or a `[min, max]` array', default: 0 }, { name: 'config.n_max', desc: 'maximum number of values with spaces for the test to pass', default: 0 } ] }; return f; }3c:T5e8,unnamed_columns = { let f = function(data, config = {}) { const { level = 'warn', id = 'test_unnamedcols', expected = 0 } = config; // Detect unnamed cols let cols = []; let colnames = data.columns !== undefined ? data.columns : Object.keys(data[0]); colnames.forEach((v, i) => { if (v.trim() == '') cols.push({ Position: i, Name: v }); }); // status and msg const n_min = Array.isArray(expected) ? d3.min(expected) : expected; const n_max = Array.isArray(expected) ? d3.max(expected) : expected; const status = cols.length >= n_min && cols.length <= n_max; const msg = cols.length == 0 ? `No unnamed columns at end of file.` : `Unnamed columns at end of file : ${cols.length}.`; return new Test({ msg: msg, status: status, level: level, id: id, show_details: cols.length > 0, details: build_table(cols), details_summary: 'Show columns' }); }; // Documentation f.doc = { title: "Unnamed columns detection", category: "Data structure tests", description: "`tests.unnamed_columns` is applied to parsed data and checks if some columns have empty or spaces-only names.", args: [ { name: 'data', desc: 'parsed data object' }, { name: 'config.expected', desc: 'expected number of unnamed columns, either as a single number or a `[min, max]` array', default: 0 } ] }; return f; }3d:T679,empty_rows = { let f = function(data, config = {}) { const { level = 'warn', id = 'testemptyrows', expected = 0 } = config; // Detect empty rows const reducer = (acc, d, i) => { let empty = true; let rows = []; Object.keys(d).forEach(v => { if (String(d[v]).trim() != '') empty = false; }); if (empty) acc.push({ row: i }); return acc; }; const rows = data.reduce(reducer, []); // status and msg const n_min = Array.isArray(expected) ? d3.min(expected) : expected; const n_max = Array.isArray(expected) ? d3.max(expected) : expected; const status = rows.length >= n_min && rows.length <= n_max; const msg = rows.length == 0 ? `No empty rows at end of file.` : `Empty rows at end of file : ${rows.length}.`; const rows_list = rows.map(row => `

${row.row + 1}

`); return new Test({ msg: msg, status: status, level: level, id: id, show_details: rows.length > 0, details: html`

`;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1170,"value":"$30","pinned":false,"hidden":false,"mode":"ojs"},{"id":1169,"value":"results = function(...tests) {\n tests = tests.reduce((acc, cur) => acc.concat(cur));\n if (!Array.isArray(tests)) tests = [tests];\n const container = html`

`;\n container.appendChild(colorize_msg(this));\n\n // Show test details if any\n if (this.show_details) {\n let details_node = container.appendChild(\n html`

${this.details_summary}

`\n );\n details_node.appendChild(this.details);\n }\n return container;\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":52,"value":"md`## Utility functions`","pinned":false,"hidden":false,"mode":"ojs"},{"id":54,"value":"build_table = function(tdata) {\n if (tdata.length == 0) return null;\n\n let reducer = (acc, d) => acc + d;\n let header = colnames(tdata)\n .map(d => `${d}`)\n .reduce(reducer);\n let body = tdata\n .map(d =>\n Object.values(d)\n .map(d => `${d}`)\n .reduce(reducer)\n )\n .map(d => `${d}`)\n .reduce(reducer);\n return html`\n\n${header}\n\n${body}\n\n