2f:T5d9,30:T78a,RecursiveStep = { let RecursiveStep = ({ step }) => { if (step.merged) { return htl.html`

{ console.log("MERGED:", step); }} >

MERGED

`; } if (step.simple.actions.length === 0) { if (step.simple.default_reduce != null) { return htl.html`

${StepCard({ simple_stack: step.simple, action: { reduce: step.simple.default_reduce } })} ${step.children.map((child) => RecursiveStep({ step: child }))}

`; } else { return htl.html`

{ console.log("ENDED:", step); }} class="${step.simple.finished ? "happy-end" : "dead-end"} card" >

state ${step.simple.state} ${" - "} score ${step.simple.score}

${step.result.toString()}

`; } } if (step.simple.actions.length === 1) { return htl.html`

${StepCard({ simple_stack: step.simple, action: step.simple.actions[0] })} ${step.children.map((child) => RecursiveStep({ step: child }))}

`; } return htl.html`

${_.zip(step.simple.actions, step.children).map( ([action, child]) => htl.html`

${StepCard({ simple_stack: step.simple, action: action })} ${RecursiveStep({ step: child })}

` )}

`; }; return RecursiveStep; }31:T6e0,steps = { let parse = parser.startParse(CODE); let root = { children: [] }; root.children = parse.stacks.map((stack) => ({ children: [], stack: stack, simple: stack_to_object(stack.split()), parent: root })); let current = root.children; let i = 0; let once = false; while (true) { current = advance_and_collect_stacks(current, parse.parsedPos); let possibly_tree = parse.advance(); // I couldn't get my own de-duping stuff to work... so we got // THIS MONSTROSITY let stack_i_want_to_mimick = [...parse.stacks]; let deduping_stack = []; current = current.filter((possibly, i) => { let still_in_the_race = stack_i_want_to_mimick.some((definitely, i) => { if (definitely.sameState(possibly.stack)) { // This is to de-dupe, but it wasn't working correctly // (taking away beautifully correct seperate states... at least I _think_ they were) if (DEDUPING) { deduping_stack.push({ branch: possibly, reference_stack: definitely.split() }); stack_i_want_to_mimick.splice(i, 1); } return true; } return false; }); // Check if the current branch will be removed because of a dedupe, then note that if (!still_in_the_race) { for (let { reference_stack, branch } of deduping_stack) { if (reference_stack.sameState(possibly.stack)) { possibly.merged = { reference_stack, with_branch: branch }; } } } return still_in_the_race; }); if (possibly_tree != null) { current = advance_and_collect_stacks(current, parse.parsedPos); break; } } return root; }32:T4bc,advance_and_collect_stacks = (children, pos = parse.parsedPos) => { let branches_for_next_cycle = []; let branches_to_parse_now = [...children]; for (let branch of branches_to_parse_now) { let stack = branch.stack.split(); for (;;) { stack.p.tokens.mainToken = null; let stacks_for_now = []; if (stack.pos > pos) { branches_for_next_cycle.push(branch); } else if (stack.p.advanceStack(stack, stacks_for_now, stacks_for_now)) { for (let stack of stacks_for_now) { let child_branch = { children: [], stack: stack, simple: stack_to_object(stack.split()), parent: branch }; branch.children.push(child_branch); branches_to_parse_now.push(child_branch); } let new_branch = { children: [], stack: stack, simple: stack_to_object(stack.split()), parent: branch }; branch.children.push(new_branch); branch = new_branch; // Cool continue; } else { branch.result = parse.stackToTree(branch.stack); } break; } } return branches_for_next_cycle; }33:T420,stack_to_object = (stack) => { let parser = stack.parser; let parse = stack.p; parse.tokens.getActions(stack); let default_reduce = parser.stateSlot(stack.state, ParseState.DefaultReduce); let state = states_as_object[stack.state]; let possible_actions = state_to_actions(state.actions); let tokens = parser.tokenizers.map((x, i) => parse.tokens.tokens[i]); return { score: stack.score, state: stack.state, state_object: state, possible_actions: possible_actions, pos: stack.pos, tokens: tokens.map((x) => ({ ...x, token: parser.termNames[x.value] })), default_reduce: default_reduce > 0 ? reduce_to_object(default_reduce) : null, actions: _.chunk(parse.tokens.getActions(stack), 3).map( action_triplet_to_object ), current_buffer: _.chunk(stack.buffer, 4).map( ([node, from, to, size_in_bytes]) => { return { node: parser.termNames[node], from, to, children: size_in_bytes / 4 - 1 }; } ) }; }34:T562,Action = ({ // Distinguishes between shift (off) and reduce (on) actions. ReduceFlag: 1 << 16, // The first 16 bits hold the target state's id for shift actions, // and the reduced term id for reduce actions. ValueMask: 2 ** 16 - 1, // In reduce actions, all bits beyond 18 hold the reduction's depth // (the amount of stack frames it reduces). ReduceDepthShift: 19, // This is set for reduce actions that reduce two instances of a // repeat term to the term (but _not_ for the reductions that match // the repeated content). RepeatFlag: 1 << 17, // Goto actions are a special kind of shift that don't actually // shift the current token, just add a stack frame. This is used for // non-simple skipped expressions, to enter the skip rule when the // appropriate token is seen (because the arbitrary state from which // such a rule may start doesn't have the correct goto entries). GotoFlag: 1 << 17, // Both shifts and reduces can have a stay flag set. For shift, it // means that the current token must be shifted but the state should // stay the same (used for single-token skip expression). For // reduce, it means that, instead of consulting the goto table to // determine which state to go to, the state already on the stack // must be returned to (used at the end of non-simple skip // expressions). StayFlag: 1 << 18 })35:[] 2d:{"title":"Lezer Debugger","cells":[{"id":0,"value":"# Lezer Debugger\n\nI needed a place to \"see inside\" a Lezer parser, to know why it is not behaving as I want!\nTurns out, this is only the first step: I now need to figure out a way to turn lezer-generator inside out.\n\nStill, this can still be useful if your lezer grammar seems to be misbehaving, while you are sure it should be doing what you want. Now you can see what choices it makes.","pinned":false,"hidden":false,"mode":"md"},{"id":400,"value":"viewof CODE = htl.html``","pinned":false,"hidden":false,"mode":"ojs"},{"id":426,"value":"htl.html`

${parser.parse(CODE).toString()}

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":553,"value":"htl.html`

\n ${steps.children.map(\n (step) => htl.html`\n

\n ${RecursiveStep({ step: step })}\n

\n `\n )}\n

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1341,"value":"viewof DEDUPING = Inputs.toggle({ label: \"Enable (questionable) deduping\" })","pinned":false,"hidden":false,"mode":"ojs"},{"id":1349,"value":"I run a parser kind of parallel to a normal parser, to make sure my parser is not making too many odd choices or mistakes.\nOne thing I found a bit tricky is the stack deduping, removing stacks that are similar (equal?) to each other.\nAlso I have no way to show nicely in the UI what stacks are being merged, so I decided to disable it by default.\nI still found it sometimes reveals some behaviour, which I'm unsure if it is actually how the parser works, so I have a toggle here to still enable it.","pinned":false,"hidden":false,"mode":"md"},{"id":682,"value":"$2f","pinned":false,"hidden":false,"mode":"html"},{"id":1382,"value":"Tokenizers = ({ tokens }) => {\n return htl.html`

tokenizers

\n ${tokens\n .filter((x) => x.token != null)\n .map((token) => htl.html`

${token.token ?? \"null\"}

`)}\n

`\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1386,"value":"PossibleActions = ({ possible_actions }) => {\n return possible_actions.length > 0\n ? htl.html`

Possible Actions

\n ${possible_actions.map(\n (action) => htl.html`

${action.token}

`\n )}\n

`\n : null;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":758,"value":"StepCard = ({ simple_stack: s, action }) => {\n // console.log(\"action.reduce:\", action);\n\n return htl.html`

\n\n

\n state ${s.state}\n ${\" - \"}\n score ${s.score}\n

\n\n ${\n action.reduce\n ? htl.html`

\n Reduce (${action.reduce.reducedepth})\n

${action.reduce.name}

`\n : htl.html`

\n Shift ${\n action.action.stay ? \"(skip)\" : null\n }\n

${action.token}

`\n }\n\n \n\n \n\n

`;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":774,"value":"$30","pinned":false,"hidden":false,"mode":"ojs"},{"id":543,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":593,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":1153,"value":"md`## Grammar`","pinned":false,"hidden":false,"mode":"ojs"},{"id":115,"value":"viewof GRAMMAR = htl.html``","pinned":false,"hidden":false,"mode":"ojs"},{"id":1264,"value":"// Place to edit the (BuildOptions.externalTokenizer)[https://lezer.codemirror.net/docs/ref/#generator.BuildOptions.externalTokenizer] option if you need to\nEXTERNAL_TOKENIZER = (name, terms) => {\n const CHAR_SPACE = \" \".codePointAt(0);\n const CHAR_DOT = \".\".codePointAt(0);\n const CHAR_STAR = \"*\".codePointAt(0);\n const CHAR_LBRACKET = \"{\".codePointAt(0);\n\n return new ExternalTokenizer(\n (input, stack) => {\n // nowhitespace\n if (\n (input.peek(-1) === CHAR_SPACE || input.peek(0) === CHAR_SPACE) &&\n stack.canShift(terms.whitespace)\n ) {\n input.acceptToken(terms.whitespace, 0);\n return;\n }\n },\n { extend: true }\n );\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":1222,"value":"md`\n# Advanced debugger\n\nUsed by me to get a more intimate experience with how lezer parses.\nWould only recommend if you want to get really close with the internals\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":130,"value":"htl.html`\n {\n mutable parse.advanceStack(first_stack);\n mutable parse = parse;\n }}\n>NEXT one frame\n\n {\n mutable parse.advance();\n mutable parse = parse;\n }}\n>NEXT token\n\n {\n mutable parse = parser.startParse(CODE);\n }}\n>RESTART\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":18,"value":"mutable parse = parser.startParse(CODE)","pinned":true,"hidden":false,"mode":"ojs"},{"id":21,"value":"first_stack = parse.stacks[0]","pinned":true,"hidden":false,"mode":"ojs"},{"id":265,"value":"parser.termNames","pinned":true,"hidden":false,"mode":"ojs"},{"id":36,"value":"s = stack_to_object(first_stack)","pinned":true,"hidden":false,"mode":"ojs"},{"id":1189,"value":"md`# Appendix`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1136,"value":"","pinned":false,"hidden":false,"mode":"html"},{"id":509,"value":"INITIAL = ({\n GRAMMAR: `@precedence {\n times @left\n plus @left\n} \n\n@top SourceFile {\n expression\n}\n\nexpression { \n Identifier |\n Plus |\n Times | \n MacroExpression\n}\n\nTimes {\n expression\n !times\n // I could put 'whitespace' before and after this and the\n // plus operator, but this whitespace is the only one necessary\n // for the effect\n whitespace\n \"*\"\n expression\n}\n\nPlus {\n expression\n !plus\n \"+\"\n expression\n}\n\nMacroExpression {\n // Necessary parts here are the opening \"(\"\n // and the 'expression \"*\"' without 'whitespace' in between them.\n \"(\" expression \"*\" \")\"\n}\n\n@skip { \" \" }\n@external tokens layout from \"./index.tokens.js\" {\n whitespace\n}\n\n@tokens {\n Identifier { $[A-Z]+ }\n \"::\" \"()\"\n}\n`,\n CODE: `A + B * X`\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1215,"value":"md`## Parser`","pinned":false,"hidden":false,"mode":"ojs"},{"id":9,"value":"parser = generator.buildParser(GRAMMAR, {\n externalTokenizer: EXTERNAL_TOKENIZER\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":243,"value":"goto_table = {\n let table = parser.goto;\n\n let terms = table[0];\n let term_to_offset = table.slice(1, terms + 1);\n\n let target_states = [];\n let offset_to_place_in_final_array = {};\n\n let p = terms + 1;\n while (p < table.length) {\n let groupTag = table[p];\n let last = groupTag & 1;\n let start_state_count = groupTag >> 1;\n let target_state = table[p + 1];\n let start_states = table.slice(p + 2, p + 2 + start_state_count);\n\n offset_to_place_in_final_array[p] = target_states.length;\n target_states.push({\n target_state,\n target_state,\n start_states,\n last: last === 1\n });\n p = p + start_state_count + 2;\n }\n\n return {\n terms: Object.fromEntries(\n Array.from(term_to_offset)\n .map((x, i) => {\n if (x === 1) return null;\n return [parser.termNames[i], offset_to_place_in_final_array[x]];\n })\n .filter((x) => x != null)\n ),\n target_states\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":38,"value":"states_as_object = _.chunk(parser.states, 6).map((state) => {\n let actions = state[ParseState.Actions];\n let flags = state[ParseState.Flags];\n let skip = state[ParseState.Skip];\n let tokenisers_mask = state[ParseState.TokenizerMask];\n let forced_reduce = state[ParseState.ForcedReduce];\n let default_reduce = state[ParseState.DefaultReduce];\n\n let tokenisers = parser.tokenizers.filter((x, i) => {\n return ((1 << i) & tokenisers_mask) != 0;\n });\n\n return {\n actions: actions,\n flags: flags,\n skip: skip,\n tokenisers,\n forced_reduce: forced_reduce !== 0 ? reduce_to_object(forced_reduce) : null,\n default_reduce:\n default_reduce !== 0 ? reduce_to_object(default_reduce) : null\n };\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1171,"value":"md`## Lezer Internals Helpers`","pinned":false,"hidden":false,"mode":"ojs"},{"id":34,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":451,"value":"state_to_actions = (action) => {\n let data = parser.data;\n\n let possible_actions = [];\n for (let i = action; i < 100; i += 3) {\n if (data[i] == Seq.End) {\n if (data[i + 1] == Seq.Next) {\n i = pair(data, i + 2);\n } else {\n if (possible_actions.length == 0 && data[i + 1] == Seq.Other) {\n possible_actions.push({\n target: action_number_to_object(pair(data, i + 2)),\n token: null\n });\n // index = this.putAction(pair(data, i + 2), token, end, index);\n }\n break;\n }\n }\n\n let action_object = action_number_to_object(pair(data, i + 1));\n possible_actions.push({\n target: action_object,\n token: parser.termNames[data[i]]\n });\n // index = this.putAction(pair(data, i + 1), token, end, index);\n }\n\n return possible_actions;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":994,"value":"action_triplet_to_object = ([action, token, nextEnd]) => ({\n token: parser.termNames[token],\n nextEnd,\n ...action_number_to_object(action)\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":203,"value":"reduce_to_object = (action) => {\n return {\n reducedepth: action >> Action.ReduceDepthShift,\n name: parser.termNames[action & Action.ValueMask],\n repeat: action & Action.RepeatFlag,\n stay: action & Action.StayFlag,\n type: action & Action.ValueMask\n // goto: Action.GotoFlag\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":195,"value":"action_number_to_object = (action) => {\n if (action & Action.ReduceFlag) {\n return {\n reduce: reduce_to_object(action)\n };\n } else {\n return {\n action: {\n goto: action & Action.GotoFlag,\n value: action & Action.ValueMask,\n stay: action & Action.StayFlag,\n repeat: action & Action.RepeatFlag\n }\n };\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":283,"value":"function pair(data, off) {\n return data[off] | (data[off + 1] << 16);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":179,"value":"md`\n## Constants\n\nTaken pretty much verbatim from [@lezer-parser/lr/constants.ts](https://github.com/lezer-parser/lr/blob/main/src/constants.ts)\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":73,"value":"ParseState = ({\n // Offsets into the record of the individual fields\n Flags: 0,\n Actions: 1,\n Skip: 2,\n TokenizerMask: 3,\n DefaultReduce: 4,\n ForcedReduce: 5\n // Total size of a state record\n // Size: 6\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":110,"value":"$34","pinned":false,"hidden":false,"mode":"ojs"},{"id":274,"value":"StateFlag = ({\n // Set if this state is part of a skip expression (which means nodes\n // produced by it should be moved out of any node reduced directly\n // after them).\n Skipped: 1,\n // Indicates whether this is an accepting state.\n Accepting: 2\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":288,"value":"Seq = ({\n // Used as end marker for most of the sequences stored in uint16\n // arrays\n End: 0xffff,\n Done: 0,\n Next: 1,\n Other: 2\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1156,"value":"md`## Modules`","pinned":false,"hidden":false,"mode":"ojs"},{"id":4,"value":"lr = import(\"https://cdn.skypack.dev/@lezer/lr@1.3.3?min\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":7,"value":"generator = import(\n \"https://unpkg.com/@lezer/generator@1.2.2/dist/index.js?module\"\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":581,"value":"ExternalTokenizer = lr.ExternalTokenizer","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W35","stdlib":"1","titling":"auto"}