Preprocessing UCD to JSON lines

Author: fileformat

download.sh

@@ -70,10 +70,7 @@ else
 fi
 
 cd ${TMP_DIR}
-unzip -j UCD.zip --exclude "ReadMe.txt"
+unzip -j UCD.zip -x ReadMe.txt -x emoji/ReadMe.txt
 unzip -j ucd.all.flat.zip
 
-echo "INFO: converting ucd to json"
-yq -p=xml -o=json "${TMP_DIR}/ucd.all.flat.xml" > "${TMP_DIR}/ucd.all.flat.json"
-
 echo "INFO: download complete at $(date -u +%Y-%m-%dT%H:%M:%SZ)"

local-deploy.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+#
+# deploy the files to the local source repo
+#
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+echo "INFO: local deploy starting at $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+
+ORIG_DIR=./output
+if [ ! -d "${ORIG_DIR}" ]; then
+    echo "ERROR: output files not created in ${ORIG_DIR}"
+	exit 1
+fi
+
+DEST_DIR=~/workspace/ff/www.fileformat.info/src/main/java/org/unicode
+if [ ! -d "${ORIG_DIR}" ]; then
+	echo "ERROR: destination directory not found ${DEST_DIR}"
+	exit 2
+fi
+
+
+gzip \
+	--stdout \
+	"${ORIG_DIR}/ucd-lines.json" \
+	> "${DEST_DIR}/ucd-lines.json.gz"
+
+cp -p "${ORIG_DIR}/Unihan.txt.gz" "${DEST_DIR}/Unihan.txt.gz"
+
+echo "INFO: local deploy complete at $(date -u +%Y-%m-%dT%H:%M:%SZ)"

run.sh

@@ -7,10 +7,20 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-
 echo "INFO: run starting at $(date -u +%Y-%m-%dT%H:%M:%SZ)"
 
+if [ ! -d "./output" ]; then
+	echo "INFO: creating output directory ./output"
+	mkdir -p ./output
+fi
+
+if [ ! -d "./node_modules" ]; then
+	echo "INFO: installing npm dependencies"
+	npm install
+fi
+
+
 npx tsc
 node dist/main.js
 
-echo "INFO: run complete at $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+echo "INFO: run complete at $(date -u +%Y-%m-%dT%H:%M:%SZ)"

src/CodepointData.ts

@@ -1,17 +1,24 @@
 type CodepointData = {
-	code: string;
-	name: string;
 	age: string;
+	approximations?: string[];
+	bidi: string;
 	block: string;
+	caseVariants?: { [key: string]: string[] };
 	category: string;
-	script: string;
-	approximations?: string[];
+	code: string;
+	combine: string;
 	comments?: string[];
+	decomposition?: string[];
 	indexEntries?: string[];
+	mirror?: string;
+	name: string;
 	notes?: string[];
+	oldname?: string;
 	related?: string[];
+	script: string;
 	scriptExtensions?: string[];
 	tags?: string[];
+	title: string;
 	variants?: string[];
 };
 

src/ProcessIndexTxt.ts

@@ -36,11 +36,6 @@ async function ProcessIndexTxt( filePath: string, cpMap: {[code: string]: Codepo
 				continue;
 			}
 
-			if (cpData.name == indexEntry) {
-				// console.log(`DEBUG: skipping index entry ${indexEntry} for code ${code} since it is the same as the name`);
-				continue;
-			}
-
 			if (!cpData.indexEntries) {
 				cpData.indexEntries = [];
 			}

src/ProcessScriptExtensionsTxt.ts

@@ -40,7 +40,7 @@ async function ProcessScriptExtensionsTxt(
 				endCode = startCode;
 			}
 
-			console.log(`DEBUG: processing codepoints ${codes} (${startCode}-${endCode}) for scripts ${scripts.join(", ")}`);
+			//console.log(`DEBUG: processing codepoints ${codes} (${startCode}-${endCode}) for scripts ${scripts.join(", ")}`);
 
 			for (let cp = startCode; cp <= endCode; cp++) {
 				const codeHex = cp.toString(16).toUpperCase().padStart(4, "0");

src/XmlToMap.ts

@@ -142,6 +142,9 @@ async function XmlToMap(xmlPath: string) {
 		var name = charData.na || charData.na1;
 		if (!name && charData['name-alias']) {
 			name = charData['name-alias'][0].alias;
+			console.log(
+				`WARN: codepoint ${charData.cp} missing name, using name-alias '${name}'`
+			);
 		}
 
 		if (name.endsWith('#')) {
@@ -185,19 +188,59 @@ async function XmlToMap(xmlPath: string) {
 			console.log(`INFO: name-alias data: ${JSON.stringify(charData['name-alias'])}`);
 		}
 
+		if (charData.JSN && charData.JSN.length > 0) {
+			notes.push(`Hangul Syllable Type: ${charData.JSN}`);
+		}
+
+		const caseVariants: { [key: string]: string[] } = {};
+		if (charData.suc && charData.suc.length > 0 && charData.suc !== "#") {
+			caseVariants["uppercase"] = [ charData.ucp ];
+		}
+		if (charData.slc && charData.slc.length > 0 && charData.slc !== "#") {
+			caseVariants["lowercase"] = [ charData.lcp ];
+		}
+		if (charData.stc && charData.stc.length > 0 && charData.stc !== "#") {
+			caseVariants["titlecase"] = [ charData.tcp ];
+		}
+		if (charData.uc && charData.uc.length > 0 && charData.uc !== "#") {
+			caseVariants["uppercase"] = charData.uc.split(" ");
+		}
+		if (charData.lc && charData.lc.length > 0 && charData.lc !== "#") {
+			caseVariants["lowercase"] = charData.lc.split(" ");
+		}
+		if (charData.tc && charData.tc.length > 0 && charData.tc !== "#") {
+			caseVariants["titlecase"] = charData.tc.split(" ");
+		}
+		var decomposition: string[] | undefined = undefined;
+		if (charData.dm && charData.dm.length > 0 && charData.dm !== "#") {
+			decomposition = charData.dm.split(" ");
+		}
+
+		if (charData.bpt && charData.bpt.length > 0 && charData.bpt !== 'n') {
+			notes.push(`Bidi Paired Bracket Type: ${charData.bpt == 'o' ? 'Open' : 'Closed'}`);
+			notes.push(`Bidi Paired Bracket: U+${charData.bpb}`);
+		}
+
 		var cpData: CodepointData = {
-			code: charData.cp,
 			name,
+			title: charData.na1 || name,
 			age: charData.age,
+			bidi: charData.bc,
 			block: charData.blk,
+			caseVariants: Object.keys(caseVariants).length > 0 ? caseVariants : undefined,
 			category: charData.gc,
+			code: charData.cp,
+			combine: charData.ccc, 
+			decomposition,
+			mirror: charData.bmg === '' ? undefined : charData.bmg,
+			oldname: charData.na1 === '' ? undefined : charData.na1,
 			script: charData.sc,
 			tags,
 			notes,
 			comments: [],
 			indexEntries: [],
-			related : [],
-			variants : [],
+			related: [],
+			variants: [],
 		};
 
 		cpMap[charData.cp] = cpData;