30:Taeb,// Function to wrangle data Wrangler = (data = [], data2 = []) => { // Keep track of all the inputs let inputs = []; // Set up the three panels for the UI let wrapper = html`

`; let wrangler_container = html`

`; let right_panel = html`

Transformations Applied

`; let left_panel = html`

`; let bottom_panel = html`

`; wrangler_container.appendChild(left_panel); wrangler_container.appendChild(right_panel); wrapper.appendChild(wrangler_container); wrapper.appendChild(bottom_panel); // Function to add an operation (callback for the render_operations_panel) const addOperation = (operation) => { // Get current state of data and columns for building next input let data, cols; if (operation.includes("join")) { data = [wrapper.value.data.objects(), data2]; cols = data.map(get_cols); } else { data = wrapper.value.data.objects(); cols = get_cols(data); } const id = id_generator(); // Get default value const value = operations[operation].get_default_value ? operations[operation].get_default_value(cols, data) : [cols[0].label]; const options = operation === "filter" ? { data } : {}; inputs.push({ id, cols, value, options, type: operation }); update_inputs(); }; const update_wrapper_value = (value) => { wrapper.value = value; wrapper.dispatchEvent(new Event("input", { bubbles: true })); }; // Update inputs panel const update_inputs = () => { right_panel.replaceChild( render_inputs_panel(inputs, update_code_and_results), right_panel.querySelector(".inputs_wrapper") ); }; // Update coee and results const update_code_and_results = (updated_inputs) => { inputs = updated_inputs; const expression = get_code_expression(inputs); right_panel.replaceChild( render_code_panel(expression), right_panel.querySelector(".code_wrapper") ); bottom_panel.replaceChild( render_results_panel(expression, data, data2, update_wrapper_value), bottom_panel.querySelector(".results_wrapper") ); }; // Append the initial state left_panel.appendChild( html`${render_operations_panel(operations, addOperation)}` ); right_panel.appendChild( html`${render_inputs_panel(inputs, update_code_and_results)}` ); right_panel.appendChild(html`${render_code_panel(get_code_expression(inputs))}`); bottom_panel.appendChild(html`${render_results_panel("data", data, data2)}`); wrapper.value = { data: aq.from(data), code: "" }; // initial value // Append styles wrapper.appendChild(make_styles()); return wrapper; }31:T5ed,// Render the panel displaying available operations, firing a callback function on click render_operations_panel = (operations = [], callback = () => null) => { // Visual elements const wrapper = html`

`; const search = html``; // Search through operations search.oninput = () => { const value = search.op_search.value.toLowerCase(); const filtered_operations = {}; Object.keys(operations) .filter( (d) => d.includes(value) || operations[d].description?.includes(value) || operations[d].type?.includes(value) ) .forEach((d) => (filtered_operations[d] = operations[d])); left_panel.replaceChild( OperationsMenu(filtered_operations, callback), left_panel.querySelector(".menu_wrapper") ); }; const left_panel = html`

Data Wrangler ${make_docs_icon()}Docs

${search}

`; search.oninput(); wrapper.appendChild(left_panel); return wrapper; }32:T768,// Accepts an array of inputs ({id, type, cols, options}) render_inputs_panel = (inputs = [], callback = () => null) => { // Panel to contain the transformations and code const wrapper = html`

`; const inputs_wrapper = html`

`; wrapper.append(inputs_wrapper); // Prompt to add operations const prompt = html`

Add transformations using the list aboveto the left.

`; const clear = html`

Clear all

`; clear.onclick = () => { inputs = []; callback(inputs); inputs_wrapper.innerHTML = ""; inputs_wrapper.appendChild(prompt); }; // Event for the form (when events are deleted or changed through the UI itself) inputs_wrapper.oninput = (trigger) => { // If there aren't any inputs, prompt the user to create inputs if (inputs?.length === 0) { inputs_wrapper.appendChild(prompt); inputs_wrapper.removeChild(clear); } // Update the inputs array if the event comes from dragging (reorders them) if (trigger?.type === "drag") inputs = trigger.inputs; // Otherwise, get id and value of form, update the inputs array else if (trigger.target?.form?.id) { const id = trigger.target.form.id; const value = trigger.target.form.value; const index = inputs.findIndex((d) => d.id === id); inputs[index].value = value; } // Run callback function (to pass information to parent) callback(inputs); }; // Draw the initial state (prompt or inputs) if (inputs?.length) { render_inputs(inputs, inputs_wrapper); inputs_wrapper.appendChild(clear); } else inputs_wrapper.appendChild(prompt); return wrapper; }33:Tb4a,// Function to make draggable inputs inside of a wrapper render_inputs = (input_arr, wrapper) => { function redraw(data) { d3.select(wrapper) .selectAll(".item") .data(data, (d) => d.id) .join("div") .selectAll("span") .style("transform", `translate(0px, 0px)`); // Fire an event on the parent wrapper.oninput({ type: "drag", inputs: data }); } function dragged(event, d) { const parent = this.parentNode; // wrapper span around the hamburger menu const nextSiblingY = parent.parentNode.nextElementSibling?.offsetTop; const y = parent.offsetTop; const prevSiblingY = parent.parentNode.previousElementSibling?.offsetTop; // Move all the child nodes d3.select(parent) // content .selectAll(function () { return parent.childNodes; }) .style("transform", `translate(0px, ${event.y - this.offsetTop - 10}px)`); // Shift up or down if (y + event.y > nextSiblingY || y + event.y < prevSiblingY) { const index = input_arr.findIndex((ele) => ele.id === d.id); const new_pos = y + event.y > nextSiblingY ? index + 1 : index - 1; input_arr.splice(new_pos, 0, input_arr.splice(index, 1)[0]); redraw(input_arr); } } function dragended(event, d) { redraw(input_arr); } const drag = d3.drag().on("drag", dragged).on("end", dragended); const divs = d3 .select(wrapper) .selectAll("div") .data(input_arr, (d) => d.id) .join( (enter) => { const divs = enter.append("div").attr("class", "item"); // Content const content = divs .append("span") .style("display", "inline-block") .style("transform", `translate(0px, 0px)`) .style("width", "100%") .attr("class", "content"); // Hamburger icons content .append("span") .style("vertical-align", "top") .style("cursor", "pointer") .style("display", "inline-block") .call(drag) .each(function (d) { this.innerHTML = make_hamburger_icon(); }); // Remove icons content .append("span") .attr("class", "remove") .text("x") .on("click", (event, value) => { const index = input_arr.findIndex((ele) => ele.id === value.id); input_arr.splice(index, 1); redraw(input_arr); }); // Craete the visual elements (forms) content.each(function (d) { this.appendChild( html`${operations[ d.type ].make_input(d)}` ); }); return divs; }, (update) => update, (exit) => exit.remove() ); wrapper.dispatchEvent(new Event("input", { bubbles: true })); }34:T6df,// Build the list of operations OperationsMenu = ( operations = ["semijoin"], update = (d) => d, hoverUpdate = (d) => d ) => { const icon_width = 85; const padding = 15; // const categories = Object.keys(operations) const categories = ["core", "join", "reshape", "clean", "set"]; const wrapper = html`

`; categories.forEach((category) => { const ops = Object.keys(operations).filter( (d) => operations[d].type === category ); let warning = ""; if (category === "join") { warning = html`
To join, pass in a second dataset:
Wrangler(data, data2)`; } else if (category === "reshape") { warning = html`
These functions can dramatically increase data size.`; } else if (category === "set") { warning = html`
For sets, pass in a second data set: Wrangler(data, data2)`; } const category_wrapper = html`
${capitalize( category )} operations ${warning}
`; ops.forEach((d) => { const value = operations[d]; const text = value.description; const control = html`
${icons[d]}
${d}\n ${text}
`; control.onclick = () => update(d); category_wrapper.appendChild(control); }); wrapper.appendChild(category_wrapper); }); return wrapper; }35:T274b,// List of all operations, including how to build their input, get the syntax, icon, description operations = ({ select: { make_input: (config) => SelectInput(config), description: "choose columns", type: "core", get_syntax: (d) => { if (!d.value) return ""; // Wrap in quotes const quoted_values = d.value.map((d) => `'${d}'`).join(","); if (!quoted_values) return ""; return `.select(${quoted_values})`; } }, filter: { make_input: (config) => FilterInput(config), description: "choose rows", type: "core", get_default_value: (cols, data) => { const comparison_type = cols[0].type === "categorical" ? "match" : "gt"; const starter_value = cols[0].type === "categorical" ? "" : d3.mean(data, (d) => +d[cols[0].label]); return [cols[0].label, comparison_type, starter_value]; }, get_syntax: (d) => { // Implement this logic better const [col, comparison, value] = d.value; if (value === "") return ""; if (comparison.includes("equal")) { const comparison_symbol = comparison === "equal" ? "===" : "!=="; return `.filter(d => d["${col}"] ${comparison_symbol} "${value}")`; } else if (comparison === "match") { return `.filter(d => op.match(d["${col}"], "${value}"))`; } else if (comparison.endsWith("with")) { return `.filter(d => op.${comparison}(d["${col}"], "${value}"))`; } else if (comparison === "lt" || comparison === "gt") { const comparison_symbol = comparison === "lt" ? "<" : ">"; return `.filter(d => d["${col}"] ${comparison_symbol} ${+value})`; } return ""; } }, orderby: { make_input: (config) => OrderbyInput(config), description: "sort rows", type: "core", get_syntax: (d) => { const [order_col, order] = d.value; if (order === "Descending") { return `.orderby(aq.desc("${order_col}"))`; } else { return `.orderby("${order_col}")`; } } }, derive: { make_input: (config) => DeriveInput(config), description: "add a new column", type: "core", get_default_value: (cols) => [], get_syntax: (d) => { const [col_name, col_value] = d.value; if (!col_name || !col_value) return ""; return `.derive({${col_name}: d => ${prefixVarsInFormula(col_value)}})`; } }, rename: { make_input: (config) => RenameInput(config), description: "change a column's name", type: "core", get_syntax: (d) => { let [old_name, new_name] = d.value; if (!new_name) return ""; if (old_name.includes(" ")) old_name = `"${old_name}"`; return `.rename({${old_name}: "${new_name}"})`; } }, groupby: { make_input: (config) => GroupbyInput(config), description: "associate rows", type: "core", get_syntax: (d) => { const quoted_groups = d.value.map((d) => `'${d}'`).join(","); return `.groupby(${quoted_groups})`; } }, rollup: { make_input: (config) => RollupInput(config), description: "compute column statistics", type: "core", get_default_value: (cols) => [ cols.filter((d) => d.type === "continuous")[0].label, ["mean", "count"] ], get_syntax: (d) => { const [summary_col, metrics] = d.value; if (!metrics.length) return ""; const str = metrics.map((d, i) => { const end = i === metrics.length - 1 ? "" : ",\n\t\t"; const arg = d === "count" ? `` : `d["${summary_col}"]`; return `${d}:d => op.${d}(${arg})${end}`; }); return `.rollup({${str.join("\t")}})`; } }, count: { make_input: (config) => CountInput(config), description: "count rows", type: "core", get_syntax: () => `.count()` }, sample: { make_input: (config) => SampleInput(config), description: "retrieve random rows", type: "core", get_default_value: () => [10, ""], get_syntax: (d) => { const [n_sample, replacement] = d.value; const end = replacement === "replacement" ? ", {replace:true})" : ")"; return `.sample(${n_sample}${end}`; } }, slice: { make_input: (config) => SliceInput(config), description: "retrieve rows by index", type: "core", get_default_value: () => [0, 10], get_syntax: (d) => { const [lower, upper] = d.value; return `.slice(${lower}, ${upper})`; } }, relocate: { make_input: (config) => RelocateInput(config), description: "move a column", type: "core", get_default_value: (cols) => [cols[0].label, "after", cols[1].label], get_syntax: (d) => { const [selected, position, reference] = d.value; return `.relocate("${selected}", {${position}:"${reference}"})`; } }, reify: { make_input: (config) => ReifyInput(config), description: "materialize the table", type: "core", get_syntax: () => `.reify()` }, ungroup: { make_input: (config) => UngroupInput(config), description: "remove groupings from data", type: "core", get_syntax: () => `.ungroup()` }, unorder: { make_input: (config) => UnorderInput(config), description: "remove orderings from data", type: "core", get_syntax: () => `.unorder()` }, join: { make_input: (config) => JoinInput(config), description: "join two tables", type: "join", get_default_value: (cols) => [cols[0][0]?.label, cols[1][0]?.label], get_syntax: (d) => { const [left_col, right_col] = d.value; return `.${d.type}(aq.from(data2), ["${left_col}", "${right_col}"])`; } }, join_left: { make_input: (config) => JoinInput(config), description: "left join two tables", type: "join", get_default_value: (cols) => [cols[0][0]?.label, cols[1][0]?.label], get_syntax: (d) => { const [left_col, right_col] = d.value; return `.${d.type}(aq.from(data2), ["${left_col}", "${right_col}"])`; } }, join_right: { make_input: (config) => JoinInput(config), description: "right join two tables", type: "join", get_default_value: (cols) => [cols[0][0]?.label, cols[1][0]?.label], get_syntax: (d) => { const [left_col, right_col] = d.value; return `.${d.type}(aq.from(data2), ["${left_col}", "${right_col}"])`; } }, join_full: { make_input: (config) => JoinInput(config), description: "full join two tables", type: "join", get_default_value: (cols) => [cols[0][0]?.label, cols[1][0]?.label], get_syntax: (d) => { const [left_col, right_col] = d.value; return `.${d.type}(aq.from(data2), ["${left_col}", "${right_col}"])`; } }, semijoin: { make_input: (config) => JoinInput(config), description: "return matching left rows", type: "join", get_default_value: (cols) => [cols[0][0]?.label, cols[1][0]?.label], get_syntax: (d) => { const [left_col, right_col] = d.value; return `.${d.type}(aq.from(data2), ["${left_col}", "${right_col}"])`; } }, antijoin: { make_input: (config) => JoinInput(config), description: "return rows not in right table", type: "join", get_default_value: (cols) => [cols[0][0]?.label, cols[1][0]?.label], get_syntax: (d) => { const [left_col, right_col] = d.value; return `.${d.type}(aq.from(data2), ["${left_col}", "${right_col}"])`; } }, fold: { make_input: (config) => FoldInput(config), description: "transform into key-value pairs", type: "reshape", get_syntax: (d) => { const fold_cols = d.value; // Wrap in quotes const fold_quoted = d.value.map((d) => `'${d}'`).join(","); if (!fold_quoted) return; return d.value.length > 1 ? `.fold([${fold_quoted}])` : `.fold(${fold_quoted})`; } }, pivot: { make_input: (config) => PivotInput(config), description: "create new columns for keys", type: "reshape", get_syntax: (d) => { const [key_col, value_col] = d.value; return `.pivot("${key_col}", "${value_col}")`; } }, spread: { make_input: (config) => SpreadInput(config), description: "separate column of arrays", type: "reshape", get_syntax: (d) => `.spread("${d.value}")` }, unroll: { make_input: (config) => UnrollInput(config), description: "separate array column (rows)", type: "reshape", get_syntax: (d) => `.unroll("${d.value}")` }, dedupe: { make_input: (config) => DedupeInput(config), description: "remove duplicate rows", type: "clean", get_syntax: (d) => { if (!d.value) return ""; // Wrap in quotes const deduplicate_values = d.value.map((d) => `'${d}'`).join(","); return `.dedupe(${deduplicate_values})`; } }, impute: { make_input: (config) => ImputeInput(config), description: "fill in missing values", type: "clean", get_default_value: (cols) => [ cols.filter((d) => d.type === "continuous")[0].label, "mean" ], get_syntax: (d) => { const [impute_col, operation] = d.value; const [quoted_name, col_ref] = impute_col.includes(" ") ? [`"${impute_col}"`, `["${impute_col}"]`] : [impute_col, `.${impute_col}`]; return `.impute({${quoted_name}: d => op.${operation}(d${col_ref})})`; } }, concat: { make_input: (config) => SetInput(config), description: "append a second table", type: "set", get_syntax: (d) => `.${d.type}(aq.from(data2))` }, union: { make_input: (config) => SetInput(config), description: "concat and remove duplicates", type: "set", get_syntax: (d) => `.${d.type}(aq.from(data2))` }, intersect: { make_input: (config) => SetInput(config), description: "keep only matching rows", type: "set", get_syntax: (d) => `.${d.type}(aq.from(data2))` }, except: { make_input: (config) => SetInput(config), description: "Compute table set difference", type: "set", get_syntax: (d) => `.${d.type}(aq.from(data2))` } })36:Tbeb,// Input to create a `filter` operation -- certainly the most complex input FilterInput = ({ id = "", cols = [], value = [], options = {} } = {}) => { // Get column types let col_data = cols; if (col_data.length === 0) return html`
filter not possible: there are no categorical or continuous columns present
`; const type_map = new Map(col_data.map((d) => [d.label, d.type])); const select = make_select( "filter_col", col_data.map((d) => d.label), value[0] ); // Comparison operation (e.g., equal, not equal, etc.) const comparisonOptions = { categorical: [ { label: "matches", value: "match" }, { label: "starts with", value: "startswith" }, { label: "ends with", value: "endswith" }, { label: "is equal to", value: "equal" }, { label: "is *not* equal to", value: "not_equal" } ], continuous: [ { label: "is less than", value: "lt" }, { label: "is greater than", value: "gt" } ], date: [ { label: "is less than", value: "lt" }, { label: "is greater than", value: "gt" } ] }; // Element for selecting comparison type const buildComparison = (type) => { return make_select("comparison", comparisonOptions[type], value[1]); }; const comparison = buildComparison(type_map.get(value[0])); const buildComparisonValue = (col) => { const type = type_map.get(col); let ele; switch (type) { case "categorical": const inputVal = typeof value[2] === "string" ? value[2] : "" ele = html``; break; case "date": case "continuous": const min = d3.min(options.data, (d) => +d[col]); const max = d3.max(options.data, (d) => +d[col]); ele = html``; break; } return ele; }; const comparisonValue = buildComparisonValue(value[0]); const filterForm = html`
Filter rows: ${select} ${comparison} ${comparisonValue}
`; filterForm.oninput = (event) => { if (event.target && event.target.name === "filter_col") { // Rebuild the other options const new_col = filterForm.filter_col.value; const type = type_map.get(new_col); const new_comparison = buildComparison(type); const new_comparison_value = buildComparisonValue(new_col); filterForm.comparison.parentNode.replaceChild( new_comparison, filterForm.comparison ); filterForm.comparisonValue.parentNode.replaceChild( new_comparison_value, filterForm.comparisonValue ); } // Update the value filterForm.value = [ filterForm.filter_col.value, filterForm.comparison.value, filterForm.comparisonValue.value ]; }; filterForm.oninput({}); return filterForm; }37:T427,// Input to create a `Rollup` operation RollupInput = ({ id = "", cols = [], value = [], options = {} } = {}) => { const summarizers = cols .filter((d) => d.type === "continuous") .map((d) => d.label); const defaults = value[1] || ["count", "mean"]; const select = cols.length === 0 ? "" : make_select("col", summarizers, [value[0]]); const summarize_options = summarizers.length === 0 ? ["count"] : ["min", "max", "mean", "median", "sum", "stdev", "count"]; const metrics = make_checkboxes("metric", summarize_options, defaults); const form = html`
Rollup: ${select} ${metrics}
`; form.oninput = () => { const checked = !Object.keys(form.metric).length ? form.metric.checked ? [form.metric.value] : "" : Object.keys(form.metric) .filter((d) => form.metric[d].checked === true) .map((d) => form.metric[d].value); const value = cols.length === 0 ? "" : form.col.value; form.value = [value, checked]; }; form.oninput(); return form; }38:T42d,// Error to display if operations are dragged into an unexecutable state display_error = (err) => { let message = ""; switch (true) { case err.message.includes("column"): message = "Your code statement is no longer valid, as it references a column that doesn't exist. This is often the result of reordering your operations, or changing a select statement. Undoing your previous operation should resolve the issue."; break; case err.message.includes("Unexpected token"): message = "Your code statement is not valid (yet). You may simply need to fill in the options above."; break; case err.message.includes("results"): message = "Make sure to pass an array of objects to the Wrangler() function."; break; default: message = "You encountered an error: "; break; } const styles = ` background-color:#caeaf985; padding: 10px; border-radius:5px; `; return html`
${message}
${err.message}

`; }39:T109d,make_styles = () => html``3a:[{"id":"3c2fd71a8faeeebda50f3647621d658fd2943e8e9c4661ae86f05309b75b5569b1a6731a3058b51726f88675fa46e9b3b57be9ec5e411fd76bb415c3d7730823","href":"https://static.observableusercontent.com/files/3c2fd71a8faeeebda50f3647621d658fd2943e8e9c4661ae86f05309b75b5569b1a6731a3058b51726f88675fa46e9b3b57be9ec5e411fd76bb415c3d7730823","name":"icons@4.json","type":"application/json","lastModified":1629187929425.963,"size":34750},{"id":"100ba6c9ab25d7a4687eb6bc46809219508a9faf2b7ade6c7034b0490cc7709fcfca7a50f4b76fda051310f6a4898815c53a702418c3d8644f767e3a6e44ac7e","href":"https://static.observableusercontent.com/files/100ba6c9ab25d7a4687eb6bc46809219508a9faf2b7ade6c7034b0490cc7709fcfca7a50f4b76fda051310f6a4898815c53a702418c3d8644f767e3a6e44ac7e","name":"protests.csv","type":"text/csv","lastModified":1629187929425.963,"size":8455883}] 2e:{"title":"Data Wrangler","cells":[{"id":0,"value":"# Data Wrangler","pinned":false,"hidden":false,"mode":"md"},{"id":2425,"value":"// Load the data (e.g., a .csv file)\ndata= await FileAttachment(\"protests.csv\").csv({ typed: true })","pinned":false,"hidden":false,"mode":"ojs"},{"id":1752,"value":"---","pinned":false,"hidden":false,"mode":"md"},{"id":4019,"value":"Wrangler(data)","pinned":false,"hidden":false,"mode":"ojs"},{"id":2804,"value":"
","pinned":false,"hidden":false,"mode":"md"},{"id":32,"value":"## Implementation","pinned":false,"hidden":false,"mode":"md"},{"id":31,"value":"$30","pinned":false,"hidden":false,"mode":"ojs"},{"id":4016,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":4012,"value":"// Render the formatted code, including a button to copy it\nrender_code_panel = (expression = \"\") => {\n // Panel to contain the transformations and code\n const comment = `// To use copied code replace \"data\" with your own variable\\n`;\n\n const end = `\\n\\t// Call \".objects()\" to return an array of objects`;\n \n const wrapper = html`\n
\n
Code \n
\n Copy into a cell to save work\n ${copy_code_button(comment + expression + end)}\n
\n
\n ${format_code(comment + expression)}\n
`;\n\n return wrapper;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":4011,"value":"render_results_panel = (expression, data, data2, callback = () => null) => {\n const wrapper = html`
Results
`;\n // Update the displayed data\n let wrangled_data;\n let err_message;\n try {\n wrangled_data = evaluate(\"return \" + expression, { data, aq, op, data2 });\n } catch (err) {\n err_message = display_error(err);\n }\n\n // Passing an error if there are no results\n if (!data.length) err_message = display_error({ message: \"No results\" });\n\n // Display the error or results\n let results;\n if(err_message) {\n results = err_message\n } else {\n results = Inputs.table(wrangled_data, { height: 150});\n // Set a minimum width to the table for mobile\n results.appendChild(html``)\n }\n \n wrapper.appendChild(results);\n\n // Use a callback to set the value of the parent\n callback({ data: wrangled_data, code: expression });\n return wrapper;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":4015,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":2557,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":2929,"value":"$34","pinned":false,"hidden":false,"mode":"ojs"},{"id":2930,"value":"$35","pinned":false,"hidden":false,"mode":"ojs"},{"id":3336,"value":"### Inputs","pinned":false,"hidden":false,"mode":"md"},{"id":1529,"value":"// Input to create a `Slice` operation\nSliceInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const lower = html``;\n const upper = html``;\n const form = html`
Slice: Retrieve rows between\n ${lower}\n and\n ${upper}\n
`;\n form.oninput = (event) => {\n // Update the value\n form.value = [form.lower.value, form.upper.value];\n };\n form.oninput({});\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":194,"value":"$36","pinned":false,"hidden":false,"mode":"ojs"},{"id":1502,"value":"// Input to create a `Sample` operation\nSampleInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const ele = html``;\n const order_value = value[1] ? \"With Replacement\" : \"\";\n const order = make_checkboxes(\n \"replacement\",\n [\"With Replacement\"],\n [order_value]\n );\n const form = html`
Sample Rows:\n ${ele}\n ${order}\n
`;\n form.oninput = (event) => {\n const replacement = form.replacement.checked === true ? \"replacement\" : \"\";\n // Update the value \n form.value = [form.n_sample.value, replacement]; \n };\n form.oninput({});\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":904,"value":"// Input to create a `derive` operation\nDeriveInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const ele = html`is defined as`;\n\n const form = html`
Derive:\n ${ele}\n
`;\n form.oninput = (event) => {\n // Update the value\n form.value = [form.col_name.value, form.col_value.value];\n };\n form.oninput({});\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":2119,"value":"// Input to create a `fold` operation\nFoldInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const labels = cols.map((d) => d.label);\n const checkboxes = make_checkboxes(\"select\", labels, value);\n const form = html`
Fold columns:\n${checkboxes}\n
`;\n form.oninput = () =>\n (form.value = Object.keys(form.select)\n .filter((d) => form.select[d].checked === true)\n .map((d) => form.select[d].value));\n\n form.oninput();\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1832,"value":"// Input to create a `join` operation\nJoinInput = ({\n id = \"\",\n type = \"\",\n cols = [],\n value = [],\n options = {}\n} = {}) => {\n const left_labels = cols[0].map((d) => d.label);\n const left_select = make_select(\"left_col\", left_labels, value[0]);\n const right_labels = cols[1].map((d) => d.label);\n const right_select = make_select(\"right_col\", right_labels, value[1]);\n\n // Join data input\n const label =\n type.includes(\"semi\") || type.includes(\"anti\")\n ? `${capitalize(type.replace(\"join\", \"\"))} Join`\n : type.includes(\"join_\")\n ? `Join ${type.replace(\"join_\", \"\")}`\n : \"Join\";\n const form = html`
${label}\n data2\n on:\n left column ${left_select} matches right column ${right_select}\n
`;\n\n // Event handling\n form.oninput = () => {\n form.value = [form.left_col.value, form.right_col.value];\n };\n form.oninput();\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":2175,"value":"SpreadInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const categories = cols\n .filter((d) => d.type === \"categorical\")\n .map((d) => d.label);\n const col_input = make_select(\"col\", categories, value);\n const form = html`
Spread: ${col_input}
`;\n\n form.oninput = () => {\n form.value = form.col.value;\n };\n\n form.oninput();\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":2201,"value":"UnrollInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const categories = cols\n .filter((d) => d.type === \"categorical\")\n .map((d) => d.label);\n const col_input = make_select(\"col\", categories);\n\n const form = html`
Unroll: ${col_input}
`;\n\n form.oninput = () => {\n form.value = form.col.value;\n };\n\n form.oninput();\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":2143,"value":"PivotInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const categories = cols\n .filter((d) => d.type === \"categorical\" || d.type === \"date\")\n .map((d) => d.label);\n const key_input = make_select(\"key_col\", categories, value[0]);\n const value_input = make_select(\n \"value_col\",\n cols.map((d) => d.label),\n value[1]\n );\n const form = html`
Pivot: ${key_input} ${value_input}
`;\n\n form.oninput = () => {\n form.value = [form.key_col.value, form.value_col.value];\n };\n form.oninput();\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":2239,"value":"ImputeInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const col_input = make_select(\n \"col\",\n cols.filter((d) => d.type === \"continuous\").map((d) => d.label),\n value[0]\n );\n const operations = [\"max\", \"mean\", \"median\", \"min\", \"mode\"];\n const ele = make_select(\"operation\", operations, value[1]);\n const form = html`
Impute:${col_input} using the column ${ele}\n
`;\n form.oninput = () => {\n form.value = [form.col.value, form.operation.value];\n };\n form.oninput();\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1544,"value":"// Input to create an `rename` operation\nRenameInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const labels = cols.map((d) => d.label);\n const select = make_select(\"col\", labels, [value[0]]);\n const new_label = html``;\n const form = html`
Rename:\n${select}\nto \n${new_label}\n
`;\n form.oninput = () => {\n form.value = [form.col.value, form.new_label.value];\n };\n form.oninput();\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1657,"value":"// Input to create an `relocate` operation\nRelocateInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const labels = cols.map((d) => d.label);\n const selected = make_select(\"selected\", labels, value[0]);\n const position = make_select(\"position\", [\"after\", \"before\"], value[1]);\n const reference = make_select(\"reference\", labels, value[2]);\n const new_label = html``;\n const form = html`
Relocate:\n${selected}\n${position}\n${reference}\n
`;\n form.oninput = () => {\n form.value = [\n form.selected.value,\n form.position.value,\n form.reference.value\n ];\n };\n form.oninput();\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":634,"value":"// Input to create an `orderby` operation\nOrderbyInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const labels = cols.map((d) => d.label);\n const select = make_select(\"col\", labels, value[0]);\n const order = make_checkboxes(\"order\", [\"Descending\"], [value[1]]);\n const form = html`
Order by (sort):\n${select}\n${order}\n
`;\n form.oninput = () => {\n const order = form.order.checked === true ? \"Descending\" : \"ascending\";\n form.value = [form.col.value, order];\n };\n form.oninput();\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":861,"value":"$37","pinned":false,"hidden":false,"mode":"ojs"},{"id":843,"value":"// Input to create an `groupby` operation\nGroupbyInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const groups = cols\n .filter((d) => d.type === \"categorical\" || d.type === \"date\")\n .map((d) => d.label);\n if (groups.length === 0)\n return html`
groupby not possible: there are no categorical or date columns present
`;\n\n const checkboxes = make_checkboxes(\"groups\", groups, value);\n const form = html`
Group by:\n${checkboxes}\n
`;\n form.oninput = () => {\n if (!form.groups) return;\n const checked = !Object.keys(form.groups).length\n ? form.groups.checked\n ? [form.groups.value]\n : \"\"\n : Object.keys(form.groups)\n .filter((d) => form.groups[d].checked === true)\n .map((d) => form.groups[d].value);\n form.value = checked;\n };\n form.oninput();\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1600,"value":"// Input to create an `ungroup` operation\nUngroupInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const form = html`
Ungroup data ✓
`;\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1680,"value":"// Input to create an `reify` operation\nReifyInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const form = html`
Reify Table ✓
`;\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":2335,"value":"// Input to create any `set` operation\nSetInput = ({\n id = \"\",\n type = \"\",\n cols = [],\n value = [],\n options = {}\n} = {}) => {\n const form = html`
${capitalize(\n type\n )} Table data2 ✓
`;\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1650,"value":"// Input to create an `count` operation\nCountInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const form = html`
Count Rows ✓
`;\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1611,"value":"// Input to create an `ungroup` operation\nUnorderInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const form = html`
Unorder data ✓
`;\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":43,"value":"// Input to create a `select` operation\nSelectInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const labels = cols.map((d) => d.label);\n const default_cols = value.length === 0 ? [labels[0]] : value;\n const checkboxes = make_checkboxes(\"select\", labels, default_cols);\n const form = html`
Select columns:\n ${checkboxes}\n
`;\n form.oninput = () =>\n (form.value = Object.keys(form.select)\n .filter((d) => form.select[d].checked === true)\n .map((d) => form.select[d].value));\n\n form.oninput();\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1448,"value":"// Input to create a `deduplicate` operation\nDedupeInput = ({ id = \"\", cols = [], value = [], options = {} } = {}) => {\n const labels = cols.map((d) => d.label);\n const checkboxes = make_checkboxes(\"select\", labels, value);\n const form = html`
Remove Duplicates:\n${checkboxes}\n
`;\n form.oninput = () =>\n (form.value = Object.keys(form.select)\n .filter((d) => form.select[d].checked === true)\n .map((d) => form.select[d].value));\n\n form.oninput();\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":3324,"value":"### Helper Functions","pinned":false,"hidden":false,"mode":"md"},{"id":2713,"value":"$38","pinned":false,"hidden":false,"mode":"ojs"},{"id":4005,"value":"// Accepts an array of inputs ({id, type, element}) where the element is an HTML input\n// Returns the code expression captured by the input\nget_code_expression = (inputs) => {\n let expression = `aq.from(data)`;\n if (!inputs?.length) return expression;\n inputs.map((d, i) => {\n const next_line = operations[d.type].get_syntax(d);\n if (!next_line) return;\n expression += \"\\n\\t\" + operations[d.type].get_syntax(d);\n });\n return expression;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":3506,"value":"make_select = (name = \"select\", options = [\"a\", \"b\"], selected = \"\") => {\n return html`\n \n `;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":3489,"value":"make_checkboxes = (name = \"boxes\", options = [\"a\", \"b\"], selected = []) => {\n return html`${options.map((d) => {\n return `${d}`;\n })}\n`;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":3470,"value":"get_cols = (data) =>\n Object.keys(data[0] || {}).map((d) => {\n return {\n label: d,\n type: getType(data, d)\n };\n })","pinned":false,"hidden":false,"mode":"ojs"},{"id":2980,"value":"copy_code_button = (code_str) => {\n return html`
${Copier(\n html`${make_copy_icon()}Copy`,\n { value: code_str }\n )}
`;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":41,"value":"// Function to get column types\ngetType = (data, column) => {\n for (const d of data) {\n const value = d[column];\n if (value == null) continue;\n if (typeof value === \"number\") return \"continuous\";\n if (value instanceof Date) return \"date\";\n return \"categorical\";\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":3714,"value":"// see unit tests in https://observablehq.com/@fil/prefixvarsinformula\nfunction prefixVarsInFormula(str, arg = \"d\", ignore = [\"aq\", \"op\"]) {\n let shift = 0;\n const insert = `${arg}.`;\n for (const { type, start, end, name } of P.parseCell(str).references) {\n if (type === \"Identifier\" && name !== arg && !ignore.includes(name)) {\n str =\n str.slice(0, start + shift) + insert + name + str.slice(end + shift);\n shift += insert.length;\n }\n }\n return str;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":703,"value":"// A function to evaluate a string, given an object\nevaluate = (s, obj) =>\n new Function(...Object.keys(obj), s)(...Object.values(obj))","pinned":false,"hidden":false,"mode":"ojs"},{"id":1033,"value":"// To generate a unique ID for each input element (probably a better way to do this)\nid_generator = () => {\n var S4 = function () {\n return (((1 + Math.random()) * 0x10000) | 0).toString(16).substring(1);\n };\n return \"a\" + S4() + S4();\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":2925,"value":"format_code = (code_str) => {\n return html`
${md`~~~js\n${code_str}\n~~~`}`;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":2073,"value":"capitalize = (string) => {\n return string.charAt(0).toUpperCase() + string.slice(1);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":3562,"value":"### Styles","pinned":false,"hidden":false,"mode":"md"},{"id":1712,"value":"$39","pinned":false,"hidden":false,"mode":"ojs"},{"id":3316,"value":"### Icons","pinned":false,"hidden":false,"mode":"md"},{"id":2556,"value":"make_hamburger_icon = () => {\n return ``;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":2912,"value":"make_docs_icon = () => svg``","pinned":false,"hidden":false,"mode":"ojs"},{"id":2911,"value":"make_copy_icon = () => svg`\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2909,"value":"icons = FileAttachment(\"icons@4.json\").json()","pinned":false,"hidden":false,"mode":"ojs"},{"id":3208,"value":"---","pinned":false,"hidden":false,"mode":"md"},{"id":21,"value":"## Appendix","pinned":false,"hidden":false,"mode":"md"},{"id":1369,"value":"import { Copier } from \"@mbostock/copier\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":1122,"value":"// Load the arquero function into the notebook\nimport { aq, op } from \"@uwdata/arquero\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":2427,"value":"import {dataInput} from \"@john-guerra/file-input-with-default-value\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":3715,"value":"// https://github.com/observablehq/parser\nP = require(\"@observablehq/parser@4.4.4\")","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W3a","stdlib":"1","titling":"auto"}