Release Unicode Tools 0.1.0

2026-09-01 02:38:25 +02:00
commit dca0cb0e79
75 changed files with 23916 additions and 0 deletions
@@ -0,0 +1,462 @@
+import { createHash } from "node:crypto";
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { unzipSync } from "fflate";
+
+const root = dirname(dirname(fileURLToPath(import.meta.url)));
+const cacheDirectory = join(root, ".cache", "unicode-17.0.0");
+const outputDirectory = join(root, "public", "data");
+const decoder = new TextDecoder();
+
+const sources = [
+  {
+    id: "ucd",
+    filename: "UCD.zip",
+    url: "https://www.unicode.org/Public/17.0.0/ucd/UCD.zip",
+    sha256: "2066d1909b2ea93916ce092da1c0ee4808ea3ef8407c94b4f14f5b7eb263d28e",
+  },
+  {
+    id: "unihan",
+    filename: "Unihan.zip",
+    url: "https://www.unicode.org/Public/17.0.0/ucd/Unihan.zip",
+    sha256: "f7a48b2b545acfaa77b2d607ae28747404ce02baefee16396c5d2d7a8ef34b5e",
+  },
+  {
+    id: "emoji-test",
+    filename: "emoji-test.txt",
+    url: "https://www.unicode.org/Public/17.0.0/emoji/emoji-test.txt",
+    sha256: "1d8a944f88d7952f7ef7c5167fef3c67995bcae24543949710231b03a201acda",
+  },
+  {
+    id: "emoji-sequences",
+    filename: "emoji-sequences.txt",
+    url: "https://www.unicode.org/Public/17.0.0/emoji/emoji-sequences.txt",
+    sha256: "12cc8267dc33cbd11ed32bcf6fc5dc2ad9c7a77bae1bdfba2f41b1b9b3ead8dd",
+  },
+  {
+    id: "emoji-zwj",
+    filename: "emoji-zwj-sequences.txt",
+    url: "https://www.unicode.org/Public/17.0.0/emoji/emoji-zwj-sequences.txt",
+    sha256: "5b25441daed2322b068c5e70cda522946a4f0274df864445a1965a92e5fc5cad",
+  },
+  {
+    id: "uts39",
+    filename: "uts39-data-17.0.0.zip",
+    url: "https://www.unicode.org/Public/security/latest/uts39-data-17.0.0.zip",
+    sha256: "349b8d9bbb4718e6bb58a8e8ee774857762e8121742e762e1bfb7152bd4c4238",
+  },
+  {
+    id: "license",
+    filename: "UNICODE-LICENSE-3.txt",
+    url: "https://www.unicode.org/license.txt",
+    sha256: "e7a93b009565cfce55919a381437ac4db883e9da2126fa28b91d12732bc53d96",
+  },
+];
+
+function hash(bytes) {
+  return createHash("sha256").update(bytes).digest("hex");
+}
+
+async function acquire(source) {
+  const target = join(cacheDirectory, source.filename);
+  try {
+    const cached = await readFile(target);
+    if (hash(cached) === source.sha256) return cached;
+  } catch {
+    /* Download below. */
+  }
+  const response = await fetch(source.url, { redirect: "follow" });
+  if (!response.ok)
+    throw new Error(
+      `Unable to download ${source.url}: HTTP ${response.status}`,
+    );
+  const bytes = Buffer.from(await response.arrayBuffer());
+  const actual = hash(bytes);
+  if (actual !== source.sha256)
+    throw new Error(
+      `SHA-256 mismatch for ${source.filename}: expected ${source.sha256}, got ${actual}`,
+    );
+  await writeFile(target, bytes);
+  return bytes;
+}
+
+function textFromArchive(archive, name) {
+  const value = archive[name];
+  if (!value) throw new Error(`Required Unicode data file is absent: ${name}`);
+  return decoder.decode(value);
+}
+
+function lines(text) {
+  return text.replaceAll("\r\n", "\n").split("\n");
+}
+
+function parseRange(token) {
+  const [start, end = start] = token.trim().split("..");
+  return [Number.parseInt(start, 16), Number.parseInt(end, 16)];
+}
+
+function parseRanges(text) {
+  const result = [];
+  for (const raw of lines(text)) {
+    const content = raw.split("#", 1)[0]?.trim();
+    if (!content) continue;
+    const [rangeToken, valueToken] = content
+      .split(";")
+      .map((value) => value.trim());
+    if (!rangeToken || !valueToken) continue;
+    const [start, end] = parseRange(rangeToken);
+    result.push({ start, end, value: valueToken });
+  }
+  return result.sort((left, right) => left.start - right.start);
+}
+
+function findRange(ranges, point, fallback) {
+  let low = 0;
+  let high = ranges.length - 1;
+  while (low <= high) {
+    const middle = (low + high) >>> 1;
+    const range = ranges[middle];
+    if (point < range.start) high = middle - 1;
+    else if (point > range.end) low = middle + 1;
+    else return range.value;
+  }
+  return fallback;
+}
+
+function parseDerivedNames(text) {
+  const values = new Map();
+  for (const raw of lines(text)) {
+    const content = raw.split("#", 1)[0]?.trim();
+    if (!content) continue;
+    const [rangeToken, nameToken] = content
+      .split(";")
+      .map((value) => value.trim());
+    if (!rangeToken || !nameToken) continue;
+    const [start, end] = parseRange(rangeToken);
+    for (let point = start; point <= end; point += 1) {
+      values.set(
+        point,
+        nameToken.includes("*")
+          ? nameToken.replace(
+              "*",
+              point.toString(16).toUpperCase().padStart(4, "0"),
+            )
+          : nameToken,
+      );
+    }
+  }
+  return values;
+}
+
+function parseAliases(text) {
+  const values = new Map();
+  for (const raw of lines(text)) {
+    const content = raw.split("#", 1)[0]?.trim();
+    if (!content) continue;
+    const [pointToken, alias, type] = content
+      .split(";")
+      .map((value) => value.trim());
+    const point = Number.parseInt(pointToken, 16);
+    if (!Number.isFinite(point) || !alias) continue;
+    const list = values.get(point) ?? [];
+    list.push(type ? `${alias} (${type})` : alias);
+    values.set(point, list);
+  }
+  return values;
+}
+
+function parseEmojiFlags(text) {
+  const flags = new Map();
+  const bitByProperty = new Map([
+    ["Emoji", 1],
+    ["Emoji_Presentation", 2],
+    ["Emoji_Modifier", 4],
+    ["Emoji_Modifier_Base", 8],
+    ["Extended_Pictographic", 16],
+  ]);
+  for (const range of parseRanges(text)) {
+    const bit = bitByProperty.get(range.value);
+    if (!bit) continue;
+    for (let point = range.start; point <= range.end; point += 1)
+      flags.set(point, (flags.get(point) ?? 0) | bit);
+  }
+  return flags;
+}
+
+function parseEmoji(text) {
+  const result = [];
+  let group = "Other";
+  let subgroup = "Other";
+  for (const raw of lines(text)) {
+    if (raw.startsWith("# group:")) {
+      group = raw.slice(8).trim();
+      continue;
+    }
+    if (raw.startsWith("# subgroup:")) {
+      subgroup = raw.slice(11).trim();
+      continue;
+    }
+    const [definition, annotation] = raw.split("#");
+    if (!definition || !annotation) continue;
+    const [pointsToken, statusToken] = definition
+      .split(";")
+      .map((value) => value.trim());
+    const status = statusToken?.trim();
+    if (status !== "fully-qualified" && status !== "component") continue;
+    const match = /^\s*(\S+)\s+E([0-9.]+)\s+(.+)$/u.exec(annotation);
+    if (!match || !pointsToken) continue;
+    const points = pointsToken
+      .split(/\s+/u)
+      .map((value) => Number.parseInt(value, 16));
+    result.push([
+      points,
+      match[1],
+      match[3],
+      group,
+      subgroup,
+      status,
+      match[2],
+    ]);
+  }
+  return result;
+}
+
+function parseNamedSequences(text) {
+  const result = [];
+  for (const raw of lines(text)) {
+    const content = raw.split("#", 1)[0]?.trim();
+    if (!content) continue;
+    const [name, pointsToken] = content.split(";").map((value) => value.trim());
+    if (!name || !pointsToken) continue;
+    const points = pointsToken
+      .split(/\s+/u)
+      .map((value) => Number.parseInt(value, 16));
+    result.push([name, points, String.fromCodePoint(...points)]);
+  }
+  return result;
+}
+
+function parseConfusables(text) {
+  const result = [];
+  for (const raw of lines(text)) {
+    const content = raw.split("#", 1)[0]?.trim();
+    if (!content) continue;
+    const [sourceToken, targetToken, type] = content
+      .split(";")
+      .map((value) => value.trim());
+    if (!sourceToken || !targetToken) continue;
+    const source = String.fromCodePoint(
+      ...sourceToken.split(/\s+/u).map((value) => Number.parseInt(value, 16)),
+    );
+    const target = String.fromCodePoint(
+      ...targetToken.split(/\s+/u).map((value) => Number.parseInt(value, 16)),
+    );
+    result.push([source, target, type]);
+  }
+  return result;
+}
+
+function parseUnihan(archive) {
+  const wanted = new Map([
+    ["kDefinition", 1],
+    ["kMandarin", 2],
+    ["kCantonese", 3],
+    ["kJapaneseOn", 4],
+    ["kJapaneseKun", 5],
+    ["kKorean", 6],
+    ["kTotalStrokes", 7],
+    ["kRSUnicode", 8],
+  ]);
+  const records = new Map();
+  for (const [name, bytes] of Object.entries(archive)) {
+    if (!name.endsWith(".txt")) continue;
+    for (const raw of lines(decoder.decode(bytes))) {
+      if (!raw.startsWith("U+")) continue;
+      const [pointToken, field, value] = raw.split("\t");
+      const index = wanted.get(field);
+      if (!index || !value) continue;
+      const point = Number.parseInt(pointToken.slice(2), 16);
+      const record = records.get(point) ?? [
+        point,
+        "",
+        "",
+        "",
+        "",
+        "",
+        "",
+        "",
+        "",
+      ];
+      record[index] = value;
+      records.set(point, record);
+    }
+  }
+  return [...records.values()].sort((left, right) => left[0] - right[0]);
+}
+
+function dictionaryIndex(dictionary, value) {
+  let index = dictionary.index.get(value);
+  if (index !== undefined) return index;
+  index = dictionary.values.length;
+  dictionary.values.push(value);
+  dictionary.index.set(value, index);
+  return index;
+}
+
+function scalarOrNull(value) {
+  return value ? Number.parseInt(value, 16) : null;
+}
+
+await mkdir(cacheDirectory, { recursive: true });
+await rm(outputDirectory, { recursive: true, force: true });
+await mkdir(outputDirectory, { recursive: true });
+const acquired = new Map();
+for (const source of sources) acquired.set(source.id, await acquire(source));
+
+const ucd = unzipSync(new Uint8Array(acquired.get("ucd")));
+const unihan = unzipSync(new Uint8Array(acquired.get("unihan")));
+const security = unzipSync(new Uint8Array(acquired.get("uts39")));
+const scripts = parseRanges(textFromArchive(ucd, "Scripts.txt"));
+const blocks = parseRanges(textFromArchive(ucd, "Blocks.txt"));
+const ages = parseRanges(textFromArchive(ucd, "DerivedAge.txt"));
+const names = parseDerivedNames(
+  textFromArchive(ucd, "extracted/DerivedName.txt"),
+);
+const aliases = parseAliases(textFromArchive(ucd, "NameAliases.txt"));
+const emojiFlags = parseEmojiFlags(
+  textFromArchive(ucd, "emoji/emoji-data.txt"),
+);
+
+const dictionaries = {
+  categories: { values: [], index: new Map() },
+  bidiClasses: { values: [], index: new Map() },
+  scripts: { values: [], index: new Map() },
+  blocks: { values: [], index: new Map() },
+  ages: { values: [], index: new Map() },
+};
+const planes = new Map();
+let pendingRange;
+
+function addRecord(point, fields) {
+  // Private-use code points and UTF-16 surrogates have no interoperable
+  // character identity. Their ranges remain visible through Blocks.txt, but
+  // emitting one synthetic search row per code point would add ~140,000
+  // misleading records and more than 10 MiB to every release.
+  if (fields[2] === "Co" || fields[2] === "Cs") return;
+  const plane = point >>> 16;
+  const list = planes.get(plane) ?? [];
+  list.push([
+    point,
+    names.get(point) ?? fields[1] ?? "",
+    dictionaryIndex(dictionaries.categories, fields[2] || "Cn"),
+    dictionaryIndex(dictionaries.scripts, findRange(scripts, point, "Unknown")),
+    dictionaryIndex(dictionaries.blocks, findRange(blocks, point, "No_Block")),
+    dictionaryIndex(dictionaries.ages, findRange(ages, point, "Unassigned")),
+    Number(fields[3] || 0),
+    dictionaryIndex(dictionaries.bidiClasses, fields[4] || "L"),
+    fields[5] || "",
+    fields[6] || "",
+    fields[8] || "",
+    fields[9] === "Y" ? 1 : 0,
+    scalarOrNull(fields[12]),
+    scalarOrNull(fields[13]),
+    scalarOrNull(fields[14]),
+    aliases.get(point) ?? null,
+    emojiFlags.get(point) ?? 0,
+  ]);
+  planes.set(plane, list);
+}
+
+for (const raw of lines(textFromArchive(ucd, "UnicodeData.txt"))) {
+  if (!raw) continue;
+  const fields = raw.split(";");
+  const point = Number.parseInt(fields[0], 16);
+  if (fields[1]?.endsWith(", First>")) {
+    pendingRange = { point, fields };
+    continue;
+  }
+  if (fields[1]?.endsWith(", Last>")) {
+    if (!pendingRange)
+      throw new Error(`Range end without start at U+${fields[0]}`);
+    for (let current = pendingRange.point; current <= point; current += 1)
+      addRecord(current, pendingRange.fields);
+    pendingRange = undefined;
+    continue;
+  }
+  addRecord(point, fields);
+}
+if (pendingRange) throw new Error("Unclosed range in UnicodeData.txt");
+
+const planeManifest = [];
+let characterCount = 0;
+for (const [plane, records] of [...planes].sort(
+  (left, right) => left[0] - right[0],
+)) {
+  const filename = `characters-${plane}.json`;
+  await writeFile(join(outputDirectory, filename), JSON.stringify(records));
+  planeManifest.push({ plane, filename, count: records.length });
+  characterCount += records.length;
+}
+
+const emoji = parseEmoji(decoder.decode(acquired.get("emoji-test")));
+const namedSequences = parseNamedSequences(
+  textFromArchive(ucd, "NamedSequences.txt"),
+);
+const confusables = parseConfusables(
+  textFromArchive(security, "confusables.txt"),
+);
+const unihanRecords = parseUnihan(unihan);
+await writeFile(join(outputDirectory, "emoji.json"), JSON.stringify(emoji));
+await writeFile(
+  join(outputDirectory, "named-sequences.json"),
+  JSON.stringify(namedSequences),
+);
+await writeFile(
+  join(outputDirectory, "confusables.json"),
+  JSON.stringify(confusables),
+);
+await writeFile(
+  join(outputDirectory, "unihan.json"),
+  JSON.stringify(unihanRecords),
+);
+
+const manifest = {
+  schemaVersion: 1,
+  unicodeVersion: "17.0.0",
+  emojiVersion: "17.0",
+  characterCount,
+  planes: planeManifest,
+  dictionaries: Object.fromEntries(
+    Object.entries(dictionaries).map(([key, value]) => [key, value.values]),
+  ),
+  blockRanges: blocks,
+  scriptRanges: scripts,
+  emojiCount: emoji.length,
+  namedSequenceCount: namedSequences.length,
+  confusableMappingCount: confusables.length,
+  unihanCount: unihanRecords.length,
+  sourceIdentity: sources.map(({ id, filename, url, sha256 }) => ({
+    id,
+    filename,
+    url,
+    sha256,
+  })),
+};
+await writeFile(
+  join(outputDirectory, "manifest.json"),
+  `${JSON.stringify(manifest, null, 2)}\n`,
+);
+await writeFile(
+  join(outputDirectory, "SOURCES.json"),
+  `${JSON.stringify({ schemaVersion: 1, unicodeVersion: "17.0.0", sources: manifest.sourceIdentity }, null, 2)}\n`,
+);
+await mkdir(join(root, "LICENSES"), { recursive: true });
+await writeFile(
+  join(root, "LICENSES", "UNICODE-3.0.txt"),
+  acquired.get("license"),
+);
+
+process.stdout.write(
+  `Generated ${characterCount.toLocaleString("en")} characters, ${emoji.length.toLocaleString("en")} emoji, ${unihanRecords.length.toLocaleString("en")} Unihan records, and ${confusables.length.toLocaleString("en")} confusable mappings.\n`,
+);