Added fetching scripts

scripts/fetch/index.ts

@@ -0,0 +1,89 @@
+/* eslint-disable no-console */
+import fetch from 'node-fetch'
+import { HTMLElement, Node, parse, TextNode } from 'node-html-parser'
+import { writeFile } from 'fs/promises'
+import { promiseAllStepN } from '../utils/promiseAllStepN.js'
+import { retrieveRecipes } from '../utils/retrieveRecipes'
+import { Entity, Recipe, UnfetchedEntity } from '../utils/types'
+
+const OUT_FILE = './res/details.json'
+
+function parseRecipe(itemHref: string, recipeNodes: Node[]): Recipe {
+  enum State {
+    PRE_TOKEN_EXPECTED,
+    PLUS_OR_ARROW_EXPECTED,
+    PLUS_EXPECTED,
+    TARGET_EXPECTED
+  }
+  let status: State = State.PRE_TOKEN_EXPECTED
+  const output: Recipe = {
+    prerequisites: {},
+    time: 0,
+    output: {}
+  }
+  for (const node of recipeNodes) {
+    if (node instanceof TextNode && node.text.trim() === '') continue
+    if (status === State.PRE_TOKEN_EXPECTED) {
+      if (!(node instanceof HTMLElement)) throw Error(`${itemHref}: Prerequisite node expected!`)
+      const href = node.querySelector('a')?.attrs.href
+      const amountText = node.querySelector('.factorio-icon-text')?.innerText
+      if (!href || !amountText) throw Error(`${itemHref}: No amount or href present!`)
+      if (href === '/Time') output.time = parseFloat(amountText)
+      else output.prerequisites[href] = parseFloat(amountText)
+      status = State.PLUS_OR_ARROW_EXPECTED
+    } else if (status === State.PLUS_OR_ARROW_EXPECTED || status === State.PLUS_EXPECTED) {
+      if (!(node instanceof TextNode)) throw Error(`${itemHref}: Text node expected!`)
+      if (node.text.trim() === '+') {
+        status = State.PRE_TOKEN_EXPECTED
+      } else if (node.text.trim() === '→' && status === State.PLUS_OR_ARROW_EXPECTED) {
+        status = State.TARGET_EXPECTED
+      } else {
+        throw new Error(`${itemHref}: Token "${node.text.trim()}" unexpected!`)
+      }
+    } else if (status === State.TARGET_EXPECTED) {
+      if (!(node instanceof HTMLElement)) throw Error(`${itemHref}: Target node expected!`)
+      const href = node.querySelector('a')?.attrs.href
+      const amountText = node.querySelector('.factorio-icon-text')?.innerText
+      if (!href || !amountText) throw Error(`${itemHref}: No amount or href present!`)
+      output.output[href] = parseInt(amountText, 10)
+      status = State.PLUS_EXPECTED
+    }
+  }
+  return output
+}
+
+const retrieveDetails = async (entities: UnfetchedEntity[]) => {
+  const items: Entity[] = await promiseAllStepN(
+    3,
+    entities.map(entity => async () => {
+      const res = await fetch(new URL(entity.href, 'https://wiki.factorio.com/').href)
+      const html = await res.text()
+      const root = parse(html)
+      const normalTab = root
+        .querySelectorAll('div.tabbertab[title]')
+        .find(elem => elem.attrs.title?.includes('Normal mode'))
+      if (!normalTab) {
+        console.warn(`${entity.href}: No tab with normal recipe found! Assuming base entity...`)
+        return entity
+      }
+      const recipeRow = normalTab
+        .querySelectorAll('tr')
+        .find(row => row.querySelector('p')?.innerText.includes('Recipe'))?.nextElementSibling
+      if (!recipeRow) {
+        throw new Error(`${entity.href}: No recipe row found!`)
+      }
+      const recipeNodes = recipeRow.querySelector('td')?.childNodes ?? []
+      const recipe = parseRecipe(entity.href, recipeNodes)
+
+      const item: Entity = {
+        ...entity,
+        recipe
+      }
+      console.info(`${entity.href}: done`)
+      return item
+    })
+  )
+  await writeFile(OUT_FILE, JSON.stringify(items, null, 2), 'utf-8')
+}
+
+retrieveRecipes().then(retrieveDetails).catch(console.error)