Implement most of external data seeding

.prettierignore

@@ -9,3 +9,6 @@ node_modules
 /postgres-data
 
 /app/styles/tailwind.css
+
+/app/lib/vendor
+/app/lib/dic

app/lib/files.server.ts

@@ -4,11 +4,27 @@ import path from "path";
 export const files = {
   kanjidicInput1: vendor("kanjidic/kanji_bank_1.json"),
   kanjidicInput2: vendor("kanjidic/kanji_bank_2.json"),
+  unihanReadings15: vendor("unihan/Unihan_Readings.txt"),
+  unihanVariants12: vendor("unihan/Unihan_Variants_12.txt"),
+  unihanVariants14: vendor("unihan/Unihan_Variants_14.txt"),
+  kanjiDbOldStyle: vendor("kanjiDb/jp-old-style.txt"),
+  kanjiDbHyogaiVariants: vendor("kanjiDb/hyogai-variants.txt"),
+  kanjiDbJinmeiVariants: vendor("kanjiDb/jinmei-variants.txt"),
+  kanjiDbBorrowedInput: vendor("kanjiDb/jp-borrowed.txt"),
+  kanjiDbTwEduVariants: vendor("kanjiDb/twedu-variants.txt"),
+  kanjiDbHanyuDaCidianVariants: vendor("kanjiDb/hydzd-variants.txt"),
+  kanjiDbIdsCdpTxt: vendor("kanjiDb/ids-cdp.txt"),
+  kanjiDbAnalysisTxt: vendor("kanjiDb/ids-analysis.txt"),
+  sbgyJson: dic("sbgy.json"),
+  sbgyNotesJson: dic("sbgyNotes.json"),
 };
 
 function vendor<S extends string>(string: S) {
   return path.resolve(__dirname, "vendor", string) as `${string}/vendor/${S}`;
 }
+function dic<S extends string>(string: S) {
+  return path.resolve(__dirname, "dic", string) as `${string}/vendor/${S}`;
+}
 
 function readTextFileSync<T>(filepath: string) {
   return fs.readFileSync(filepath, "utf-8") as T;

app/lib/forEachLine.server.ts

@@ -0,0 +1,25 @@
+import fs from "fs";
+import readline from "readline";
+
+export type LineReader = readline.Interface;
+
+export async function getLineReader(filename: string): Promise<LineReader> {
+  const fileStream = fs.createReadStream(filename);
+
+  const rl = readline.createInterface({
+    input: fileStream,
+    // recognize all instances of CR LF
+    // ('\r\n') in input as a single line break.
+    crlfDelay: Infinity,
+  });
+  return rl;
+}
+export async function forEachLine(
+  filename: string,
+  callback: (line: string) => void
+) {
+  const rl = await getLineReader(filename);
+  for await (const line of rl) {
+    callback(line);
+  }
+}