From 25b3bc60b6b793cd0ef15c25f760de9fef7a6750 Mon Sep 17 00:00:00 2001 From: garikkh <77762367+garikkh@users.noreply.github.com> Date: Mon, 14 Oct 2024 05:32:39 -0700 Subject: [PATCH] feat: adds custom prefix support for gettext po (#2004) --- packages/format-po-gettext/README.md | 11 +- .../src/__snapshots__/po-gettext.test.ts.snap | 107 ++++++++++++++++++ .../format-po-gettext/src/po-gettext.test.ts | 74 ++++++++++++ packages/format-po-gettext/src/po-gettext.ts | 24 ++-- website/docs/ref/catalog-formats.md | 22 +++- 5 files changed, 227 insertions(+), 11 deletions(-) diff --git a/packages/format-po-gettext/README.md b/packages/format-po-gettext/README.md index dbd4eb423..30c6940c0 100644 --- a/packages/format-po-gettext/README.md +++ b/packages/format-po-gettext/README.md @@ -13,7 +13,7 @@ > **Warning** > This formatter is made for compatibility with translation management systems, which do not support ICU expressions in PO files. -> +> > It does not support all features of LinguiJS and should be carefully considered over other formats. > > Not supported features (native gettext doesn't support this): @@ -72,10 +72,17 @@ export type PoGettextFormatterOptions = { /** * Disable warning about unsupported `Select` feature encountered in catalogs - * + * * @default false */ disableSelectWarning?: boolean + + /** + * Overrides the default prefix for icu and plural comments in the final PO catalog. + * + * @default "js-lingui:" + */ + customICUPrefix?: string } ``` diff --git a/packages/format-po-gettext/src/__snapshots__/po-gettext.test.ts.snap b/packages/format-po-gettext/src/__snapshots__/po-gettext.test.ts.snap index 4f84d62e7..d241214f1 100644 --- a/packages/format-po-gettext/src/__snapshots__/po-gettext.test.ts.snap +++ b/packages/format-po-gettext/src/__snapshots__/po-gettext.test.ts.snap @@ -180,3 +180,110 @@ exports[`po-gettext format should convert gettext plurals to ICU plural messages }, } `; + +exports[`po-gettext format using custom prefix handles custom prefix 1`] = ` +msgid "" +msgstr "" +"POT-Creation-Date: 2018-08-27 10:00+0000\\n" +"MIME-Version: 1.0\\n" +"Content-Type: text/plain; charset=utf-8\\n" +"Content-Transfer-Encoding: 8bit\\n" +"X-Generator: @lingui/cli\\n" +"Language: en\\n" +"Project-Id-Version: \\n" +"Report-Msgid-Bugs-To: \\n" +"PO-Revision-Date: \\n" +"Last-Translator: \\n" +"Language-Team: \\n" +"Plural-Forms: \\n" + +#. This is a comment by the developers about how the content must be localized. +#. js-lingui-explicit-id +#. custom-prefix:pluralize_on=someCount +msgid "message_with_id" +msgid_plural "message_with_id_plural" +msgstr[0] "Singular case with id" +msgstr[1] "Case number {someCount} with id" + +#. custom-prefix:icu=%7BanotherCount%2C+plural%2C+one+%7BSingular+case%7D+other+%7BCase+number+%7BanotherCount%7D%7D%7D&pluralize_on=anotherCount +msgid "Singular case" +msgid_plural "Case number {anotherCount}" +msgstr[0] "Singular case" +msgstr[1] "Case number {anotherCount}" + +`; + +exports[`po-gettext format using custom prefix warns and falls back to using count if prefix is not found 1`] = ` +{ + lO3l+X: { + comments: [ + js-lingui:icu=%7BanotherCount%2C+plural%2C+one+%7BSingular+case%7D+other+%7BCase+number+%7BanotherCount%7D%7D%7D&pluralize_on=anotherCount, + ], + context: null, + extra: { + flags: [], + translatorComments: [], + }, + message: Singular case, + obsolete: false, + origin: [], + translation: {count, plural, one {Singular case} other {Case number {anotherCount}}}, + }, + maCaRp: { + comments: [ + js-lingui:icu=%7Bcount%2C+plural%2C+one+%7BSingular%7D+other+%7BPlural%7D%7D&pluralize_on=count, + ], + context: null, + extra: { + flags: [], + translatorComments: [], + }, + message: Singular, + obsolete: false, + origin: [], + translation: , + }, + message_with_id: { + comments: [ + js-lingui:pluralize_on=someCount, + js-lingui-explicit-id, + ], + context: null, + extra: { + flags: [], + translatorComments: [], + }, + obsolete: false, + origin: [], + translation: {count, plural, one {Singular case} other {Case number {someCount}}}, + }, + message_with_id_but_without_translation: { + comments: [ + Comment made by the developers., + js-lingui:pluralize_on=count, + js-lingui-explicit-id, + ], + context: null, + extra: { + flags: [], + translatorComments: [], + }, + obsolete: false, + origin: [], + translation: , + }, + static: { + comments: [ + js-lingui-explicit-id, + ], + context: null, + extra: { + flags: [], + translatorComments: [], + }, + obsolete: false, + origin: [], + translation: Static message, + }, +} +`; diff --git a/packages/format-po-gettext/src/po-gettext.test.ts b/packages/format-po-gettext/src/po-gettext.test.ts index 40233886e..564203350 100644 --- a/packages/format-po-gettext/src/po-gettext.test.ts +++ b/packages/format-po-gettext/src/po-gettext.test.ts @@ -230,4 +230,78 @@ msgstr[2] "# dní" expect(catalog).toMatchSnapshot() }) + + describe("using custom prefix", () => { + it("parses plurals correctly", () => { + const defaultProfile = fs + .readFileSync(path.join(__dirname, "fixtures/messages_plural.po")) + .toString() + const customProfile = defaultProfile.replace( + /js-lingui:/g, + "custom-prefix:" + ) + + const defaultPrefix = createFormat() + const customPrefix = createFormat({ customICUPrefix: "custom-prefix:" }) + + const defaultCatalog = defaultPrefix.parse( + defaultProfile, + defaultParseCtx + ) + const customCatalog = customPrefix.parse(customProfile, defaultParseCtx) + + expect(defaultCatalog).toEqual(customCatalog) + }) + + it("warns and falls back to using count if prefix is not found", () => { + const defaultProfile = fs + .readFileSync(path.join(__dirname, "fixtures/messages_plural.po")) + .toString() + + const usingInvalidPrefix = createFormat({ + customICUPrefix: "invalid-prefix:", + }) + mockConsole((console) => { + const catalog = usingInvalidPrefix.parse( + defaultProfile, + defaultParseCtx + ) + expect(console.warn).toHaveBeenCalledWith( + expect.stringContaining( + "should be stored in a comment starting with" + ), + expect.anything() + ) + expect(catalog).toMatchSnapshot() + }) + }) + + it("handles custom prefix", () => { + const format = createFormat({ customICUPrefix: "custom-prefix:" }) + + const catalog: CatalogType = { + message_with_id: { + message: + "{someCount, plural, one {Singular case with id\ + and linebreak} other {Case number {someCount} with id}}", + translation: + "{someCount, plural, one {Singular case with id} other {Case number {someCount} with id}}", + comments: [ + "This is a comment by the developers about how the content must be localized.", + "js-lingui-explicit-id", + ], + }, + WGI12K: { + message: + "{anotherCount, plural, one {Singular case} other {Case number {anotherCount}}}", + translation: + "{anotherCount, plural, one {Singular case} other {Case number {anotherCount}}}", + }, + } + + const pofile = format.serialize(catalog, defaultSerializeCtx) + + expect(pofile).toMatchSnapshot() + }) + }) }) diff --git a/packages/format-po-gettext/src/po-gettext.ts b/packages/format-po-gettext/src/po-gettext.ts index 6d45b30a6..63c1fb447 100644 --- a/packages/format-po-gettext/src/po-gettext.ts +++ b/packages/format-po-gettext/src/po-gettext.ts @@ -13,6 +13,7 @@ type POItem = InstanceType export type PoGettextFormatterOptions = PoFormatterOptions & { disableSelectWarning?: boolean + customICUPrefix?: string } // Attempts to turn a single tokenized ICU plural case back into a string. @@ -40,7 +41,7 @@ const ICU_SELECT_REGEX = /^{.*, select(Ordinal)?, .*}$/ const LINE_ENDINGS = /\r?\n/g // Prefix that is used to identitify context information used by this module in PO's "extracted comments". -const CTX_PREFIX = "js-lingui:" +const DEFAULT_CTX_PREFIX = "js-lingui:" function serializePlurals( item: POItem, @@ -52,6 +53,7 @@ function serializePlurals( // Depending on whether custom ids are used by the developer, the (potential plural) "original", untranslated ICU // message can be found in `message.message` or in the item's `key` itself. const icuMessage = message.message + const ctxPrefix = options.customICUPrefix || DEFAULT_CTX_PREFIX if (!icuMessage) { return item @@ -99,7 +101,7 @@ function serializePlurals( } ctx.sort() - item.extractedComments.push(CTX_PREFIX + ctx.toString()) + item.extractedComments.push(ctxPrefix + ctx.toString()) // If there is a translated value, parse that instead of the original message to prevent overriding localized // content with the original message. If there is no translated value, don't touch msgstr, since marking item as @@ -161,7 +163,8 @@ const getPluralCases = (lang: string): string[] | undefined => { const convertPluralsToICU = ( item: POItem, pluralForms: string[], - lang: string + lang: string, + ctxPrefix: string = DEFAULT_CTX_PREFIX ) => { const translationCount = item.msgstr.length const messageKey = item.msgid @@ -181,13 +184,13 @@ const convertPluralsToICU = ( } const contextComment = item.extractedComments - .find((comment) => comment.startsWith(CTX_PREFIX)) - ?.substr(CTX_PREFIX.length) + .find((comment) => comment.startsWith(ctxPrefix)) + ?.substring(ctxPrefix.length) const ctx = new URLSearchParams(contextComment) if (contextComment != null) { item.extractedComments = item.extractedComments.filter( - (comment) => !comment.startsWith(CTX_PREFIX) + (comment) => !comment.startsWith(ctxPrefix) ) } @@ -229,7 +232,7 @@ const convertPluralsToICU = ( let pluralizeOn = ctx.get("pluralize_on") if (!pluralizeOn) { console.warn( - `Unable to determine plural placeholder name for item with key "%s" in language "${lang}" (should be stored in a comment starting with "#. ${CTX_PREFIX}"), assuming "count".`, + `Unable to determine plural placeholder name for item with key "%s" in language "${lang}" (should be stored in a comment starting with "#. ${ctxPrefix}"), assuming "count".`, messageKey ) pluralizeOn = "count" @@ -262,7 +265,12 @@ export function formatter( let pluralForms = getPluralCases(po.headers.Language) po.items.forEach((item) => { - convertPluralsToICU(item, pluralForms, po.headers.Language) + convertPluralsToICU( + item, + pluralForms, + po.headers.Language, + options.customICUPrefix + ) }) return formatter.parse(po.toString(), ctx) as CatalogType diff --git a/website/docs/ref/catalog-formats.md b/website/docs/ref/catalog-formats.md index cdc75853e..10466018b 100644 --- a/website/docs/ref/catalog-formats.md +++ b/website/docs/ref/catalog-formats.md @@ -146,7 +146,7 @@ export default { ### Configuration {#po-gettext-configuration} -PO Gettext formatter accepts the following options: +The PO Gettext formatter accepts the following options: ```ts export type PoGettextFormatterOptions = { @@ -170,6 +170,13 @@ export type PoGettextFormatterOptions = { * @default false */ disableSelectWarning?: boolean; + + /** + * Overrides the default prefix for icu and plural comments in the final PO catalog. + * + * @default "js-lingui:" + */ + customICUPrefix?: string; }; ``` @@ -203,6 +210,19 @@ With this format, plural messages are exported in the following ways, depending Note how `msgid` and `msgid_plural` were extracted from the original message. +- Message **with a custom comment prefix**. + + Some TMS might modify the ICU comment by attempting to split lines to be 80 characters or less, or have trouble reading lingui comments because of the `js-lingui:` prefix. To change the prefix, set `customICUPrefix` to modify the prefix for ICU comments. + + ```po + # with default prefix + #. js- + #. lingui:icu=%7BanotherCount%2C+plural%2C+one+%7BSingular+case%7D+other+%7BCase+number+%7BanotherCount%7D%7D%7D&pluralize_on=anotherCount + + # customICUPrefix = jsi18n: + #. jsi18n:icu=%7BanotherCount%2C+plural%2C+one+%7BSingular+case%7D+other+%7BCase+number+%7BanotherCount%7D%7D%7D&pluralize_on=anotherCount + ``` + ### Limitations {#po-gettext-limitations} This format comes with several caveats and should only be used when using ICU plurals in PO files is not an option: