Merge pull request #1035 from googlefonts/ur

Support unicode range and codepage range bit assignments in source
googlefonts · Oct 15, 2024 · 37143a0 · 37143a0
2 parents 099c01c + d119a73
commit 37143a0
Show file tree

Hide file tree

Showing 31 changed files with 778 additions and 20 deletions.
diff --git a/fontbe/src/os2.rs b/fontbe/src/os2.rs
@@ -306,11 +306,18 @@ fn add_unicode_range_bits(add_to: &mut HashSet<u32>, codepoint: u32) {
 }
 
 /// <https://github.com/fonttools/fonttools/blob/47813b217c1f8bc343c094776020b4f32fc024b0/Lib/fontTools/ttLib/tables/O_S_2f_2.py#L317-L334>
-fn apply_unicode_range(os2: &mut Os2, codepoints: &HashSet<u32>) {
-    let mut bits = HashSet::new();
-    for codepoint in codepoints {
-        add_unicode_range_bits(&mut bits, *codepoint);
-    }
+fn apply_unicode_range(
+    os2: &mut Os2,
+    assigned_bits: Option<HashSet<u32>>,
+    codepoints: &HashSet<u32>,
+) {
+    let bits = assigned_bits.unwrap_or_else(|| {
+        let mut bits = HashSet::new();
+        for codepoint in codepoints {
+            add_unicode_range_bits(&mut bits, *codepoint);
+        }
+        bits
+    });
 
     let mut unicode_range = [0u32; 4];
     for bit in bits {
@@ -331,7 +338,7 @@ fn apply_unicode_range(os2: &mut Os2, codepoints: &HashSet<u32>) {
 /// This is a direct translation from FontTools
 ///   <https://github.com/googlefonts/ufo2ft/blob/main/Lib/ufo2ft/util.py#L357-L449>
 /// FontTools is in turn a translation of <https://github.com/fontforge/fontforge/blob/7b2c074/fontforge/tottf.c#L3158>
-fn codepage_range_bits(codepoints: &HashSet<u32>) -> HashSet<usize> {
+fn codepage_range_bits(codepoints: &HashSet<u32>) -> HashSet<u32> {
     let mut bits = HashSet::new();
 
     let chars = codepoints
@@ -459,14 +466,18 @@ fn codepage_range_bits(codepoints: &HashSet<u32>) -> HashSet<usize> {
     bits
 }
 
-fn apply_codepage_range(os2: &mut Os2, codepoints: &HashSet<u32>) {
-    let bits = codepage_range_bits(codepoints);
+fn apply_codepage_range(
+    os2: &mut Os2,
+    assigned_bits: Option<HashSet<u32>>,
+    codepoints: &HashSet<u32>,
+) {
+    let bits = assigned_bits.unwrap_or_else(|| codepage_range_bits(codepoints));
     let mut codepage_range = [0u32; 2];
     for bit in bits {
         let idx = bit / 32;
         let bit = bit - idx * 32;
         assert!(bit <= 32, "{bit}");
-        codepage_range[idx] |= 1 << bit;
+        codepage_range[idx as usize] |= 1 << bit;
     }
     os2.ul_code_page_range_1 = Some(codepage_range[0]);
     os2.ul_code_page_range_2 = Some(codepage_range[1]);
@@ -573,8 +584,16 @@ impl Work<Context, AnyWorkId, Error> for Os2Work {
         };
         apply_panose(&mut os2, static_metadata.misc.panose.as_ref());
         apply_metrics(&mut os2, &metrics);
-        apply_unicode_range(&mut os2, &codepoints);
-        apply_codepage_range(&mut os2, &codepoints);
+        apply_unicode_range(
+            &mut os2,
+            static_metadata.misc.unicode_range_bits.clone(),
+            &codepoints,
+        );
+        apply_codepage_range(
+            &mut os2,
+            static_metadata.misc.codepage_range_bits.clone(),
+            &codepoints,
+        );
         apply_min_max_char_index(&mut os2, &codepoints);
 
         apply_max_context(&mut os2, context);

diff --git a/fontc/src/lib.rs b/fontc/src/lib.rs
@@ -3217,4 +3217,51 @@ mod tests {
             )
         );
     }
+
+    fn assert_expected_unicode_ranges(source: &str) {
+        let compile = TestCompile::compile_source(source);
+        let font = compile.font();
+        let os2 = font.os2().unwrap();
+
+        assert_eq!(
+            [1 | 1 << 28, 1 << 20 | 1 << 24, 0, 1 << 26],
+            [
+                os2.ul_unicode_range_1(),
+                os2.ul_unicode_range_2(),
+                os2.ul_unicode_range_3(),
+                os2.ul_unicode_range_4()
+            ]
+        );
+    }
+
+    #[test]
+    fn obeys_source_unicode_ranges_glyphs() {
+        assert_expected_unicode_ranges("glyphs3/WghtVar_OS2.glyphs");
+    }
+
+    #[test]
+    fn obeys_source_unicode_ranges_designspace() {
+        assert_expected_unicode_ranges("designspace_from_glyphs/WghtVarOS2.designspace");
+    }
+
+    fn assert_expected_codepage_ranges(source: &str) {
+        let compile = TestCompile::compile_source(source);
+        let font = compile.font();
+        let os2 = font.os2().unwrap();
+
+        assert_eq!(
+            [Some(1 | 1 << 1 | 1 << 19), Some(1 << 31)],
+            [os2.ul_code_page_range_1(), os2.ul_code_page_range_2(),]
+        );
+    }
+
+    #[test]
+    fn obeys_source_codepage_ranges_glyphs() {
+        assert_expected_codepage_ranges("glyphs3/WghtVar_OS2.glyphs");
+    }
+
+    #[test]
+    fn obeys_source_codepage_ranges_designspace() {
+        assert_expected_codepage_ranges("designspace_from_glyphs/WghtVarOS2.designspace");
+    }
 }
diff --git a/fontir/src/ir.rs b/fontir/src/ir.rs
@@ -124,6 +124,12 @@ pub struct MiscMetadata {
     pub created: Option<DateTime<Utc>>,
 
     pub panose: Option<Panose>,
+
+    // Allows source to explicitly control bits. <https://github.com/googlefonts/fontc/issues/1027>
+    pub unicode_range_bits: Option<HashSet<u32>>,
+
+    // Allows source to explicitly control bits. <https://github.com/googlefonts/fontc/issues/1027>
+    pub codepage_range_bits: Option<HashSet<u32>>,
 }
 
 /// PANOSE bytes
@@ -464,6 +470,8 @@ impl StaticMetadata {
                 head_flags: 3,
                 created: None,
                 panose: None,
+                unicode_range_bits: None,
+                codepage_range_bits: None,
             },
         })
     }
@@ -2124,6 +2132,8 @@ mod tests {
                 lowest_rec_ppm: 42,
                 created: None,
                 panose: None,
+                unicode_range_bits: None,
+                codepage_range_bits: None,
             },
             number_values: Default::default(),
         }

diff --git a/glyphs-reader/src/error.rs b/glyphs-reader/src/error.rs
@@ -20,4 +20,6 @@ pub enum Error {
     NotAGlyphsPackage(PathBuf),
     #[error("Invalid plist")]
     WorstPlistEver(#[from] crate::plist::Error),
+    #[error("Invalid code page {0}")]
+    InvalidCodePage(u32),
 }
diff --git a/glyphs-reader/src/font.rs b/glyphs-reader/src/font.rs
@@ -86,6 +86,8 @@ pub struct Font {
     pub superscript_y_offset: Option<i64>,
     pub superscript_y_size: Option<i64>,
 
+    pub unicode_range_bits: Option<BTreeSet<u32>>,
+    pub codepage_range_bits: Option<BTreeSet<u32>>,
     pub panose: Option<Vec<i64>>,
 }
 
@@ -420,6 +422,20 @@ impl CustomParameters {
         Some(bits)
     }
 
+    fn unicode_range(&self) -> Option<&Vec<i64>> {
+        let Some(CustomParameterValue::UnicodeRange(bits)) = self.get("unicodeRanges") else {
+            return None;
+        };
+        Some(bits)
+    }
+
+    fn codepage_range(&self) -> Option<&Vec<i64>> {
+        let Some(CustomParameterValue::CodepageRange(bits)) = self.get("codePageRanges") else {
+            return None;
+        };
+        Some(bits)
+    }
+
     fn panose(&self) -> Option<&Vec<i64>> {
         let Some(CustomParameterValue::Panose(values)) = self.get("panose") else {
             return None;
@@ -439,6 +455,8 @@ enum CustomParameterValue {
     GlyphOrder(Vec<SmolStr>),
     VirtualMaster(Vec<AxisLocation>),
     FsType(Vec<i64>),
+    UnicodeRange(Vec<i64>),
+    CodepageRange(Vec<i64>),
     Panose(Vec<i64>),
 }
 
@@ -537,6 +555,20 @@ impl FromPlist for CustomParameters {
                                 };
                                 value = Some(CustomParameterValue::FsType(tokenizer.parse()?));
                             }
+                            _ if name == Some(String::from("unicodeRanges")) => {
+                                let Token::OpenParen = peek else {
+                                    return Err(Error::UnexpectedChar('('));
+                                };
+                                value =
+                                    Some(CustomParameterValue::UnicodeRange(tokenizer.parse()?));
+                            }
+                            _ if name == Some(String::from("codePageRanges")) => {
+                                let Token::OpenParen = peek else {
+                                    return Err(Error::UnexpectedChar('('));
+                                };
+                                value =
+                                    Some(CustomParameterValue::CodepageRange(tokenizer.parse()?));
+                            }
                             _ if name == Some(String::from("panose")) => {
                                 let Token::OpenParen = peek else {
                                     return Err(Error::UnexpectedChar('('));
@@ -2154,6 +2186,48 @@ impl Instance {
     }
 }
 
+/// Glyphs appears to use code page identifiers rather than bits
+///
+/// <https://learn.microsoft.com/en-us/typography/opentype/spec/os2#ulcodepagerange>
+fn codepage_range_bit(codepage: u32) -> Result<u32, Error> {
+    Ok(match codepage {
+        1252 => 0,  // Latin 1
+        1250 => 1,  // Latin 2: Eastern Europe
+        1251 => 2,  // Cyrillic
+        1253 => 3,  // Greek
+        1254 => 4,  // Turkish
+        1255 => 5,  // Hebrew
+        1256 => 6,  // Arabic
+        1257 => 7,  // Windows Baltic
+        1258 => 8,  // Vietnamese
+        874 => 16,  // Thai
+        932 => 17,  // JIS/Japan
+        936 => 18,  // Chinese: Simplified PRC and Singapore
+        949 => 19,  // Korean Wansung
+        950 => 20,  // Chinese: Traditional Taiwan and Hong Kong SAR
+        1361 => 21, // Korean Johab
+        869 => 48,  // IBM Greek
+        866 => 49,  // MS-DOS Russian
+        865 => 50,  // MS-DOS Nordic
+        864 => 51,  // Arabic
+        863 => 52,  //	MS-DOS Canadian French
+        862 => 53,  //		Hebrew
+        861 => 54,  //		MS-DOS Icelandic
+        860 => 55,  //		MS-DOS Portuguese
+        857 => 56,  //		IBM Turkish
+        855 => 57,  //	IBM Cyrillic; primarily Russian
+        852 => 58,  //		Latin 2
+        775 => 59,  //		MS-DOS Baltic
+        737 => 60,  //	Greek; former 437 G
+        708 => 61,  //	Arabic; ASMO 708
+        850 => 62,  //	WE/Latin 1
+        437 => 63,  //	US
+
+        v if v < 64 => v, // an actual bit
+        _ => return Err(Error::InvalidCodePage(codepage)),
+    })
+}
+
 impl TryFrom<RawFont> for Font {
     type Error = Error;
 
@@ -2227,6 +2301,21 @@ impl TryFrom<RawFont> for Font {
             .fs_type()
             .map(|bits| bits.iter().map(|bit| 1 << bit).sum());
 
+        let unicode_range_bits = from
+            .custom_parameters
+            .unicode_range()
+            .map(|bits| bits.iter().map(|b| *b as u32).collect());
+
+        let codepage_range_bits = from
+            .custom_parameters
+            .codepage_range()
+            .map(|bits| {
+                bits.iter()
+                    .map(|b| codepage_range_bit(*b as u32))
+                    .collect::<Result<_, Error>>()
+            })
+            .transpose()?;
+
         let panose = from.custom_parameters.panose().cloned();
 
         let mut names = BTreeMap::new();
@@ -2372,6 +2461,8 @@ impl TryFrom<RawFont> for Font {
             superscript_x_size,
             superscript_y_offset,
             superscript_y_size,
+            unicode_range_bits,
+            codepage_range_bits,
             panose,
         })
     }
@@ -2651,19 +2742,21 @@ mod tests {
         let _ = env_logger::builder().is_test(true).try_init();
         let filename = format!("{name}.glyphs");
         let pkgname = format!("{name}.glyphspackage");
-        let g2 = Font::load(&glyphs2_dir().join(filename.clone())).unwrap();
-        let g3 = Font::load(&glyphs3_dir().join(filename.clone())).unwrap();
+        let g2_file = glyphs2_dir().join(filename.clone());
+        let g3_file = glyphs3_dir().join(filename.clone());
+        let g2 = Font::load(&g2_file).unwrap();
+        let g3 = Font::load(&g3_file).unwrap();
 
         // Handy if troubleshooting
         std::fs::write("/tmp/g2.glyphs.txt", format!("{g2:#?}")).unwrap();
         std::fs::write("/tmp/g3.glyphs.txt", format!("{g3:#?}")).unwrap();
 
         // Assert fields that often don't match individually before doing the whole struct for nicer diffs
-        assert_eq!(g2.axes, g3.axes);
+        assert_eq!(g2.axes, g3.axes, "axes mismatch {g2_file:?} vs {g3_file:?}");
         for (g2m, g3m) in g2.masters.iter().zip(g3.masters.iter()) {
-            assert_eq!(g2m, g3m);
+            assert_eq!(g2m, g3m, "master mismatch {g2_file:?} vs {g3_file:?}");
         }
-        assert_eq!(g2, g3, "g2 should match g3");
+        assert_eq!(g2, g3, "g2 should match g3 {g2_file:?} vs {g3_file:?}");
 
         if has_package {
             let g2_pkg = Font::load(&glyphs2_dir().join(pkgname.clone())).unwrap();

diff --git a/glyphs2fontir/src/source.rs b/glyphs2fontir/src/source.rs
@@ -127,6 +127,8 @@ impl GlyphsIrSource {
             superscript_x_size: font.superscript_x_size,
             superscript_y_offset: font.superscript_y_offset,
             superscript_y_size: font.superscript_y_size,
+            unicode_range_bits: font.unicode_range_bits.clone(),
+            codepage_range_bits: font.codepage_range_bits.clone(),
             panose: font.panose.clone(),
         };
         state.track_memory("/font_master".to_string(), &font)?;
@@ -177,6 +179,8 @@ impl GlyphsIrSource {
             superscript_x_size: font.superscript_x_size,
             superscript_y_offset: font.superscript_y_offset,
             superscript_y_size: font.superscript_y_size,
+            unicode_range_bits: None,
+            codepage_range_bits: None,
             panose: None,
         };
         state.track_memory("/font_master".to_string(), &font)?;
@@ -453,6 +457,15 @@ impl Work<Context, WorkId, Error> for StaticMetadataWork {
         // Default per <https://github.com/googlefonts/glyphsLib/blob/cb8a4a914b0a33431f0a77f474bf57eec2f19bcc/Lib/glyphsLib/builder/custom_params.py#L1117-L1119>
         static_metadata.misc.fs_type = font.fs_type.or(Some(1 << 3));
 
+        static_metadata.misc.unicode_range_bits = font
+            .unicode_range_bits
+            .as_ref()
+            .map(|bits| bits.iter().copied().collect());
+        static_metadata.misc.codepage_range_bits = font
+            .codepage_range_bits
+            .as_ref()
+            .map(|bits| bits.iter().copied().collect());
+
         if let Some(raw_panose) = font.panose.as_ref() {
             let mut bytes = [0u8; 10];
             bytes