Added fetching scripts

scripts/fetch/index.ts

@@ -0,0 +1,89 @@
+/* eslint-disable no-console */
+import fetch from 'node-fetch'
+import { HTMLElement, Node, parse, TextNode } from 'node-html-parser'
+import { writeFile } from 'fs/promises'
+import { promiseAllStepN } from '../utils/promiseAllStepN.js'
+import { retrieveRecipes } from '../utils/retrieveRecipes'
+import { Entity, Recipe, UnfetchedEntity } from '../utils/types'
+
+const OUT_FILE = './res/details.json'
+
+function parseRecipe(itemHref: string, recipeNodes: Node[]): Recipe {
+  enum State {
+    PRE_TOKEN_EXPECTED,
+    PLUS_OR_ARROW_EXPECTED,
+    PLUS_EXPECTED,
+    TARGET_EXPECTED
+  }
+  let status: State = State.PRE_TOKEN_EXPECTED
+  const output: Recipe = {
+    prerequisites: {},
+    time: 0,
+    output: {}
+  }
+  for (const node of recipeNodes) {
+    if (node instanceof TextNode && node.text.trim() === '') continue
+    if (status === State.PRE_TOKEN_EXPECTED) {
+      if (!(node instanceof HTMLElement)) throw Error(`${itemHref}: Prerequisite node expected!`)
+      const href = node.querySelector('a')?.attrs.href
+      const amountText = node.querySelector('.factorio-icon-text')?.innerText
+      if (!href || !amountText) throw Error(`${itemHref}: No amount or href present!`)
+      if (href === '/Time') output.time = parseFloat(amountText)
+      else output.prerequisites[href] = parseFloat(amountText)
+      status = State.PLUS_OR_ARROW_EXPECTED
+    } else if (status === State.PLUS_OR_ARROW_EXPECTED || status === State.PLUS_EXPECTED) {
+      if (!(node instanceof TextNode)) throw Error(`${itemHref}: Text node expected!`)
+      if (node.text.trim() === '+') {
+        status = State.PRE_TOKEN_EXPECTED
+      } else if (node.text.trim() === '→' && status === State.PLUS_OR_ARROW_EXPECTED) {
+        status = State.TARGET_EXPECTED
+      } else {
+        throw new Error(`${itemHref}: Token "${node.text.trim()}" unexpected!`)
+      }
+    } else if (status === State.TARGET_EXPECTED) {
+      if (!(node instanceof HTMLElement)) throw Error(`${itemHref}: Target node expected!`)
+      const href = node.querySelector('a')?.attrs.href
+      const amountText = node.querySelector('.factorio-icon-text')?.innerText
+      if (!href || !amountText) throw Error(`${itemHref}: No amount or href present!`)
+      output.output[href] = parseInt(amountText, 10)
+      status = State.PLUS_EXPECTED
+    }
+  }
+  return output
+}
+
+const retrieveDetails = async (entities: UnfetchedEntity[]) => {
+  const items: Entity[] = await promiseAllStepN(
+    3,
+    entities.map(entity => async () => {
+      const res = await fetch(new URL(entity.href, 'https://wiki.factorio.com/').href)
+      const html = await res.text()
+      const root = parse(html)
+      const normalTab = root
+        .querySelectorAll('div.tabbertab[title]')
+        .find(elem => elem.attrs.title?.includes('Normal mode'))
+      if (!normalTab) {
+        console.warn(`${entity.href}: No tab with normal recipe found! Assuming base entity...`)
+        return entity
+      }
+      const recipeRow = normalTab
+        .querySelectorAll('tr')
+        .find(row => row.querySelector('p')?.innerText.includes('Recipe'))?.nextElementSibling
+      if (!recipeRow) {
+        throw new Error(`${entity.href}: No recipe row found!`)
+      }
+      const recipeNodes = recipeRow.querySelector('td')?.childNodes ?? []
+      const recipe = parseRecipe(entity.href, recipeNodes)
+
+      const item: Entity = {
+        ...entity,
+        recipe
+      }
+      console.info(`${entity.href}: done`)
+      return item
+    })
+  )
+  await writeFile(OUT_FILE, JSON.stringify(items, null, 2), 'utf-8')
+}
+
+retrieveRecipes().then(retrieveDetails).catch(console.error)

scripts/index.ts

@@ -0,0 +1,5 @@
+export {}
+export { retrieveRecipes } from './utils/retrieveRecipes'
+export { Entity } from './utils/types'
+export { UnfetchedEntity } from './utils/types'
+export { Recipe } from './utils/types'

scripts/translations/index.ts

@@ -0,0 +1,18 @@
+/* eslint-disable no-console */
+import { writeFile } from 'fs/promises'
+import { retrieveRecipes } from '../utils/retrieveRecipes'
+
+const OUT_FILE = './res/translation-{lang}.json'
+const languages = ['de', 'nl']
+
+const retrieveTranslations = async () => {
+  for (const lang of languages) {
+    const entities = await retrieveRecipes(lang)
+    const items = Object.fromEntries(
+      entities.map(entity => [entity.href.replace(new RegExp(`/${lang}$`), ''), entity.name])
+    )
+    await writeFile(OUT_FILE.replace('{lang}', lang), JSON.stringify(items, null, 2), 'utf-8')
+  }
+}
+
+retrieveTranslations().catch(console.error)

scripts/utils/promiseAllStepN.ts

@@ -0,0 +1,18 @@
+export const promiseAllStepN = async <T>(n: number, list: (() => Promise<T>)[]) => {
+  const head = list.slice(0, n)
+  const tail = list.slice(n)
+  const result: T[] = []
+  const execute = async (promise: () => Promise<T>, i: number, runNext: () => Promise<void>) => {
+    result[i] = await promise()
+    await runNext()
+  }
+  const runNext = async () => {
+    const i = list.length - tail.length
+    const promise = tail.shift()
+    if (promise !== undefined) {
+      await execute(promise, i, runNext)
+    }
+  }
+  await Promise.all(head.map((promise, i) => execute(promise, i, runNext)))
+  return result
+}

scripts/utils/retrieveRecipes.ts

@@ -0,0 +1,19 @@
+import fetch from 'node-fetch'
+import { parse } from 'node-html-parser'
+import { UnfetchedEntity } from './types'
+
+export const retrieveRecipes = async (lang?: string) => {
+  const res = await fetch(
+    `https://wiki.factorio.com/Materials_and_recipes${lang ? `/${lang}` : ''}`
+  )
+  const html = await res.text()
+  const root = parse(html)
+  const icons = root.querySelectorAll('.tab > div > div.factorio-icon > a')
+  return icons
+    .map(icon => ({
+      name: icon.attrs.title,
+      href: icon.attrs.href,
+      image: icon.querySelector('img')?.attrs.src
+    }))
+    .filter((entity): entity is UnfetchedEntity => !!(entity.href && entity.name && entity.image))
+}

scripts/utils/types.ts

@@ -0,0 +1,15 @@
+export interface Recipe {
+  prerequisites: Record<string, number>
+  time: number
+  output: Record<string, number>
+}
+
+export interface UnfetchedEntity {
+  name: string
+  image: string
+  href: string
+}
+
+export interface Entity extends UnfetchedEntity {
+  recipe?: Recipe
+}