2f:[] 2d:{"title":"Factorio Locale import and analysis","cells":[{"id":0,"value":"# Factorio Locale import and analysis\n\nUploading Factorio's game files would be a crime. But reading your filesystem on demand isn't!","pinned":false,"hidden":false,"mode":"md"},{"id":14,"value":"viewof base = { \n const form = Inputs.file({multiple:true, label: htl.html`Select folder Factorio/data/base/locale/`})\n const input = form.querySelector('input')\n input.webkitdirectory = true\n return form\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":76,"value":"viewof core = { \n const form = Inputs.file({multiple:true, label: htl.html`Select folder Factorio/data/core/locale/`})\n const input = form.querySelector('input')\n input.webkitdirectory = true\n return form\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":34,"value":"filenamePattern = /\\/(?[A-Za-z-]+)\\/(?base|core|info)\\.(?cfg|json)$/i","pinned":true,"hidden":false,"mode":"ojs"},{"id":31,"value":"baseFiles = Promise.all(base.map(async file => ({...file.webkitRelativePath.match(filenamePattern).groups, text: await file.text()})))","pinned":true,"hidden":false,"mode":"ojs"},{"id":84,"value":"coreFiles = Promise.all(core.filter(file => filenamePattern.test(file.webkitRelativePath)).map(async file => ({...file.webkitRelativePath.match(filenamePattern).groups, text: await file.text()})))","pinned":true,"hidden":false,"mode":"ojs"},{"id":50,"value":"function* strings(file) {\n const pattern = /^\\s*(?[^\\s=]+)\\s*=\\s*(?.*)$|\\s*\\[(?

.*)\\]\\s*$/gm;\n const { text, ...props } = file;\n let section = null;\n for (const {groups} of text.matchAll(pattern)) {\n if (groups.section) section = groups.section;\n else yield {\n ...props, ...groups, section,\n key: section + '.' + groups.name\n };\n }\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":54,"value":"allStrings = [...baseFiles, ...coreFiles].filter(it => it.type === 'cfg').flatMap(file => [...strings(file)])","pinned":true,"hidden":false,"mode":"ojs"},{"id":233,"value":"dt = aq.from(allStrings)\n // aq.from(offlineSample).derive({key:d=>d.section+'.'+d.name})","pinned":true,"hidden":false,"mode":"ojs"},{"id":365,"value":"allKeys = aq.agg(dt, op.array_agg_distinct('key'))","pinned":true,"hidden":false,"mode":"ojs"},{"id":369,"value":"allKeys.filter(it => /(?:-|\\b)(?:red|green|wires?|circuits?|networks?)(?:-|\\b)/.test(it))","pinned":true,"hidden":false,"mode":"ojs"},{"id":760,"value":"$${dt.filter(d=>d.key == 'item-name.red-wire' && op.includes(['ca','en'],d.lang)).select('value').toMarkdown()}","pinned":true,"hidden":false,"mode":"md"},{"id":138,"value":"function languageInfo(tag) {\n const locale = new Intl.Locale(tag).maximize()\n\n return {\n tag, locale,\n rtl: locale.getTextInfo().direction == 'rtl',\n graphemeSegmenter: new Intl.Segmenter(locale, { granularity: \"grapheme\" }),\n wordSegmenter: new Intl.Segmenter(locale, { granularity: \"word\" }),\n languageName: new Intl.DisplayNames([\"en\"], { type: \"language\" })\n .of(locale.language),\n nativeName: new Intl.DisplayNames([tag], { type: \"language\" })\n .of(locale.language),\n scriptName: new Intl.DisplayNames([\"en\"], { type: \"script\" })\n .of(locale.script),\n }\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":96,"value":"languages = Object.fromEntries(coreFiles.filter(it => it.file == 'info')\n .map(it => [it.lang, {\n gameName: JSON.parse(it.text)['language-name'],\n ...languageInfo(it.lang),\n name: () => this.gameName ?? this.nativeName,\n first(s) { return [...this.graphemeSegmenter.segment(s)].at(-this.rtl).segment },\n words(s) { return [...this.wordSegmenter.segment(s)].filter(it => it.segment.trim().length > 0).map(it => (it.segment))},\n firstEach(s) { return this.words(s).map(it => this.first(it)) },\n }]));","pinned":true,"hidden":false,"mode":"ojs"},{"id":140,"value":"scripts = Object.fromEntries(\n Object.entries(Object.groupBy(Object.values(languages), it => it.scriptName))\n .sort(([k1,v1],[k2,v2]) => v2.length - v1.length)\n .map(([k,v]) => ([k, v.map(it=>it.tag).join(' ')]))\n)","pinned":true,"hidden":false,"mode":"ojs"},{"id":174,"value":"allStrings.filter(it => it.section === 'item-name' && it.name === 'red-wire').map(it =>\n [...languages[it.lang].wordSegmenter.segment(it.value)].map(it => it.segment))","pinned":true,"hidden":false,"mode":"ojs"},{"id":188,"value":"allStrings.filter(it => it.section === 'shortcut' && it.name === 'make-red-wire').map(it =>\n [...languages[it.lang].wordSegmenter.segment(it.value)].map(it => it.segment))","pinned":true,"hidden":false,"mode":"ojs"},{"id":191,"value":"allStrings.filter(it => it.section === 'controls' && it.name === 'give-red-wire').map(it =>\n [...languages[it.lang].wordSegmenter.segment(it.value)].map(it => it.segment))","pinned":true,"hidden":false,"mode":"ojs"},{"id":393,"value":"The `[color-capital]` section gives perfect info for a handful of languages, but most of them specify `R` and `G` (possibly with good reason?)... We can also do a naïve attempt at getting the first letter of the words defined in `[color]`, but there's a chance the wire strings use different words, e.g. Japanese at one point had the equivalent of `red=Aka` but `red-wire=Reddo Wairu`.","pinned":false,"hidden":false,"mode":"md"},{"id":295,"value":"color_capital = dt\n .filter(d => op.startswith(d.section, 'color') && op.includes(['red','green'],d.name))\n .derive({ short: d => d.name + (op.endswith(d.section, 'capital') ? 'Cap' : '') })\n .groupby(\"lang\")\n .pivot(\"short\", \"value\")\n .derive({\n greenFirst: aq.escape((d) => languages[d.lang].first(d.green)),\n redFirst: aq.escape((d) => languages[d.lang].first(d.red)),\n })\n .derive({\n G: d => d.greenCap && d.greenCap != 'G' ? d.greenCap : d.greenFirst,\n R: d => d.redCap && d.redCap != 'R' ? d.redCap : d.redFirst,\n })","pinned":true,"hidden":false,"mode":"ojs"},{"id":353,"value":"Inputs.table(color_capital/*.filter(d => d.red !== 'R' || d.green !== 'G')*/, { select: false, layout:'auto' })","pinned":true,"hidden":false,"mode":"ojs"},{"id":659,"value":"We can do a scan of the strings that concern wires, and see if we can extract the first letter of each word, then do a Set symmetric-difference between the red and green version of each \"key pair\" (e.g. give-red-wire vs give-green-wire) as a heuristic approach.","pinned":false,"hidden":false,"mode":"md"},{"id":396,"value":"extracted = dt\n .orderby('lang')\n .filter(d => op.match(d.name, /(?:-|\\b)(?:red|green)-wire(?:-|\\b)/))\n .derive({\n grapheme: aq.escape((d) => languages[d.lang].firstEach(d.value)),\n c: d => op.match(d.name, /red|green/),\n key_stem: d => op.join(op.split(d.key, /red-|green-/),'')\n })\n .derive({\n cap_grapheme: aq.escape(d => d.grapheme.map(g => g.toLocaleUpperCase(languages[d.lang].locale)))\n })","pinned":true,"hidden":false,"mode":"ojs"},{"id":405,"value":"Inputs.table(extracted, opts)","pinned":true,"hidden":false,"mode":"ojs"},{"id":750,"value":"The `item-description` strings aren't that useful so we can filter them out.","pinned":true,"hidden":false,"mode":"md"},{"id":508,"value":"diffed = extracted\n .filter(d => d.key_stem != 'item-description.wire')\n .groupby('lang', 'key_stem')\n // Problems (aq bug?) with pivot, otherwise this would've been easier\n .rollup({\n value: op.object_agg('c','value'),\n grapheme: op.object_agg('c','cap_grapheme'),\n })\n .derive({\n diff: aq.escape(d => {\n const r = new Set(d.grapheme.red)\n const g = new Set(d.grapheme.green)\n return [...g.symmetricDifference(r)]\n })\n })","pinned":true,"hidden":false,"mode":"ojs"},{"id":668,"value":"Inputs.table(diffed, {...opts,format:{\n value: it => htl.html`${it.green}
${it.red}`,\n grapheme: it => htl.html`

${it.green}
${it.red}

`,\n}})","pinned":true,"hidden":false,"mode":"ojs"},{"id":725,"value":"diffedFull = diffed\n // Roundabout solution but it works!\n .select('lang','diff')\n .unroll('diff')\n .groupby('lang').rollup({diff: op.array_agg_distinct('diff')})\n .join_full(color_capital).orderby('lang')\n .derive({\n GOK:d=> op.includes(d.diff, d.G) ? '✅' : '❌',\n ROK:d=> op.includes(d.diff, d.R) ? '✅' : '❌',\n })","pinned":true,"hidden":false,"mode":"ojs"},{"id":728,"value":"Inputs.table(diffedFull.select(['lang','diff','G','R','GOK','ROK']), opts)","pinned":true,"hidden":false,"mode":"ojs"},{"id":10,"value":"## Imports and helpers","pinned":false,"hidden":false,"mode":"md"},{"id":17,"value":"Inputs = require('@observablehq/inputs@0.12.0/dist/inputs.min.js')","pinned":true,"hidden":false,"mode":"ojs"},{"id":283,"value":"aq = require('arquero@8.0.3/dist/arquero.min.js')","pinned":true,"hidden":false,"mode":"ojs"},{"id":302,"value":"op = aq.op","pinned":true,"hidden":false,"mode":"ojs"},{"id":734,"value":"opts = ({select:false, layout:'auto'})","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W2f","stdlib":"1","titling":"auto"}