FEATURE: Allow plugins to register a new locale

.eslintignore

@@ -1,7 +1,5 @@
 app/assets/javascripts/env.js
-app/assets/javascripts/main_include.js
 app/assets/javascripts/main_include_admin.js
-app/assets/javascripts/pagedown_custom.js
 app/assets/javascripts/vendor.js
 app/assets/javascripts/locales/i18n.js
 app/assets/javascripts/ember-addons/
@@ -11,11 +9,9 @@ lib/javascripts/messageformat.js
 lib/javascripts/moment.js
 lib/javascripts/moment_locale/
 lib/highlight_js/
+plugins/**/lib/javascripts/locale
 public/javascripts/
-spec/phantom_js/smoke_test.js
 vendor/
 test/javascripts/test_helper.js
-test/javascripts/test_helper.js
 test/javascripts/fixtures
 test/javascripts/helpers/assertions.js
-app/assets/javascripts/ember-addons/

app/assets/javascripts/locales/i18n.js

@@ -15,6 +15,7 @@ I18n.pluralizationRules = {
 
 // Set current locale to null
 I18n.locale = null;
+I18n.fallbackLocale = null;
 
 // Set the placeholder format. Accepts `{{placeholder}}` and `%{placeholder}`.
 I18n.PLACEHOLDER = /(?:\{\{|%\{)(.*?)(?:\}\}?)/gm;
@@ -143,6 +144,10 @@ I18n.translate = function(scope, options) {
   var translation = this.lookup(scope, options);
 
   if (!this.noFallbacks) {
+    if (!translation && this.fallbackLocale) {
+      options.locale = this.fallbackLocale;
+      translation = this.lookup(scope, options);
+    }
     if (!translation && this.currentLocale() !== this.defaultLocale) {
       options.locale = this.defaultLocale;
       translation = this.lookup(scope, options);

app/models/locale_site_setting.rb

@@ -7,9 +7,12 @@ def self.valid_value?(val)
   end
 
   def self.values
-    supported_locales.map do |l|
-      lang = language_names[l] || language_names[l[0..1]]
-      { name: lang ? lang['nativeName'] : l, value: l }
+    @values ||= supported_locales.map do |locale|
+      lang = language_names[locale] || language_names[locale.split("_")[0]]
+      {
+        name: lang ? lang['nativeName'] : locale,
+        value: locale
+      }
     end
   end
 
@@ -19,43 +22,41 @@ def self.language_names
     return @language_names if @language_names
 
     @lock.synchronize do
-      @language_names ||= YAML.load(File.read(File.join(Rails.root, 'config', 'locales', 'names.yml')))
+      @language_names ||= begin
+        names = YAML.load(File.read(File.join(Rails.root, 'config', 'locales', 'names.yml')))
+
+        DiscoursePluginRegistry.locales.each do |locale, options|
+          if !names.key?(locale) && options[:name] && options[:nativeName]
+            names[locale] = { "name" => options[:name], "nativeName" => options[:nativeName] }
+          end
+        end
+
+        names
+      end
     end
   end
 
   def self.supported_locales
     @lock.synchronize do
       @supported_locales ||= begin
-        app_client_files = Dir.glob(
+        locales = Dir.glob(
           File.join(Rails.root, 'config', 'locales', 'client.*.yml')
-        )
-
-        unless ignore_plugins?
-          app_client_files += Dir.glob(
-            File.join(Rails.root, 'plugins', '*', 'config', 'locales', 'client.*.yml')
-          )
-        end
+        ).map { |x| x.split('.')[-2] }
 
-        app_client_files.map { |x| x.split('.')[-2] }
-          .uniq
-          .select { |locale| valid_locale?(locale) }
-          .sort
+        locales += DiscoursePluginRegistry.locales.keys
+        locales.uniq.sort
       end
     end
   end
 
-  def self.valid_locale?(locale)
-    assets = Rails.configuration.assets
-
-    assets.precompile.grep(/locales\/#{locale}(?:\.js)?/).present? &&
-      (Dir.glob(File.join(Rails.root, 'app', 'assets', 'javascripts', 'locales', "#{locale}.js.erb")).present? ||
-        Dir.glob(File.join(Rails.root, 'plugins', '*', 'assets', 'locales', "#{locale}.js.erb")).present?)
+  def self.reset!
+    @lock.synchronize do
+      @values = @language_names = @supported_locales = nil
+    end
   end
 
-  def self.ignore_plugins?
-    Rails.env.test? && ENV['LOAD_PLUGINS'] != "1"
+  def self.fallback_locale(locale)
+    plugin_locale = DiscoursePluginRegistry.locales[locale.to_s]
+    plugin_locale ? plugin_locale[:fallbackLocale]&.to_sym : nil
   end
-
-  private_class_method :valid_locale?
-  private_class_method :ignore_plugins?
 end

app/models/translation_override.rb

@@ -21,7 +21,8 @@ def self.upsert!(locale, key, value)
 
     data = { value: value }
     if key.end_with?('_MF')
-      data[:compiled_js] = JsLocaleHelper.compile_message_format(locale, value)
+      _, filename = JsLocaleHelper.find_message_format_locale(['en'], false)
+      data[:compiled_js] = JsLocaleHelper.compile_message_format(filename, locale, value)
     end
 
     translation_override = find_or_initialize_by(params)

config/locales/names.yml

@@ -119,9 +119,6 @@ eo:
 es:
   name: Spanish
   nativeName: Español
-es_MX:
-  name: Spanish
-  nativeName: Español (MX)
 et:
   name: Estonian
   nativeName: eesti

lib/discourse_plugin_registry.rb

@@ -14,6 +14,7 @@ class << self
     attr_writer :handlebars
     attr_writer :serialized_current_user_fields
     attr_writer :seed_data
+    attr_writer :locales
     attr_accessor :custom_html
 
     def plugins
@@ -65,6 +66,10 @@ def seed_data
       @seed_data ||= HashWithIndifferentAccess.new({})
     end
 
+    def locales
+      @locales ||= HashWithIndifferentAccess.new({})
+    end
+
     def html_builders
       @html_builders ||= {}
     end
@@ -92,6 +97,10 @@ def register_css(filename)
     self.class.stylesheets << filename
   end
 
+  def self.register_locale(locale, options = {})
+    self.locales[locale] = options
+  end
+
   def register_archetype(name, options = {})
     Archetype.register(name, options)
   end
@@ -171,6 +180,10 @@ def self.seed_paths
     result.uniq
   end
 
+  def locales
+    self.class.locales
+  end
+
   def javascripts
     self.class.javascripts
   end
@@ -207,6 +220,7 @@ def self.clear
     self.desktop_stylesheets = nil
     self.sass_variables = nil
     self.handlebars = nil
+    self.locales = nil
   end
 
   def self.reset!
@@ -222,6 +236,7 @@ def self.reset!
     html_builders.clear
     vendored_pretty_text.clear
     seed_path_builders.clear
+    locales.clear
   end
 
   def self.setup(plugin_class)

lib/freedom_patches/translate_accelerator.rb

@@ -39,6 +39,13 @@ def load_locale(locale)
         if @loaded_locales.empty?
           # load all rb files
           I18n.backend.load_translations(I18n.load_path.grep(/\.rb$/))
+
+          # load plural rules from plugins
+          DiscoursePluginRegistry.locales.each do |locale, options|
+            if options[:plural]
+              I18n.backend.store_translations(locale, i18n: { plural: options[:plural] })
+            end
+          end
         end
 
         # load it

lib/i18n/backend/fallback_locale_list.rb

@@ -3,7 +3,8 @@ module Backend
     # Configure custom fallback order
     class FallbackLocaleList < Hash
       def [](locale)
-        [locale, SiteSetting.default_locale.to_sym, :en].uniq.compact
+        fallback_locale = LocaleSiteSetting.fallback_locale(locale)
+        [locale, fallback_locale, SiteSetting.default_locale.to_sym, :en].uniq.compact
       end
     end
   end

lib/js_locale_helper.rb

@@ -85,6 +85,7 @@ def self.deep_delete_matches(deleting_from, checking_hashes)
   end
 
   def self.load_translations_merged(*locales)
+    locales = locales.compact
     @loaded_merges ||= {}
     @loaded_merges[locales.join('-')] ||= begin
       all_translations = {}
@@ -101,9 +102,10 @@ def self.load_translations_merged(*locales)
   end
 
   def self.translations_for(locale_str)
-    current_locale = I18n.locale
-    locale_sym     = locale_str.to_sym
-    site_locale    = SiteSetting.default_locale.to_sym
+    current_locale  = I18n.locale
+    locale_sym      = locale_str.to_sym
+    site_locale     = SiteSetting.default_locale.to_sym
+    fallback_locale = LocaleSiteSetting.fallback_locale(locale_str)
 
     I18n.locale = locale_sym
 
@@ -113,9 +115,9 @@ def self.translations_for(locale_str)
       elsif locale_sym == :en
         load_translations(locale_sym)
       elsif locale_sym == site_locale || site_locale == :en
-        load_translations_merged(locale_sym, :en)
+        load_translations_merged(locale_sym, fallback_locale, :en)
       else
-        load_translations_merged(locale_sym, site_locale, :en)
+        load_translations_merged(locale_sym, fallback_locale, site_locale, :en)
       end
 
     I18n.locale = current_locale
@@ -125,11 +127,13 @@ def self.translations_for(locale_str)
 
   def self.output_locale(locale)
     locale_str = locale.to_s
+    fallback_locale_str = LocaleSiteSetting.fallback_locale(locale_str)&.to_s
     translations = Marshal.load(Marshal.dump(translations_for(locale_str)))
 
     message_formats = strip_out_message_formats!(translations[locale_str]['js'])
     message_formats.merge!(strip_out_message_formats!(translations[locale_str]['admin_js']))
-    result = generate_message_format(message_formats, locale_str)
+    mf_locale, mf_filename = find_message_format_locale([locale_str], true)
+    result = generate_message_format(message_formats, mf_locale, mf_filename)
 
     translations.keys.each do |l|
       translations[l].keys.each do |k|
@@ -140,7 +144,8 @@ def self.output_locale(locale)
     # I18n
     result << "I18n.translations = #{translations.to_json};\n"
     result << "I18n.locale = '#{locale_str}';\n"
-    result << "I18n.pluralizationRules.#{locale_str} = MessageFormat.locale.#{locale_str};\n" if locale_str != "en"
+    result << "I18n.fallbackLocale = '#{fallback_locale_str}';\n" if fallback_locale_str && fallback_locale_str != "en"
+    result << "I18n.pluralizationRules.#{locale_str} = MessageFormat.locale.#{mf_locale};\n" if mf_locale != "en"
 
     # moment
     result << File.read("#{Rails.root}/lib/javascripts/moment.js")
@@ -150,6 +155,41 @@ def self.output_locale(locale)
     result
   end
 
+  def self.find_moment_locale(locale_chain)
+    path = "#{Rails.root}/lib/javascripts/moment_locale"
+
+    # moment.js uses a different naming scheme for locale files
+    locale_chain = locale_chain.map { |l| l.tr('_', '-').downcase }
+
+    find_locale(locale_chain, path, :moment_js, false)
+  end
+
+  def self.find_message_format_locale(locale_chain, fallback_to_english)
+    path = "#{Rails.root}/lib/javascripts/locale"
+    find_locale(locale_chain, path, :message_format, fallback_to_english)
+  end
+
+  def self.find_locale(locale_chain, path, type, fallback_to_english)
+    locale_chain.each do |locale|
+      plugin_locale = DiscoursePluginRegistry.locales[locale]
+      return plugin_locale[type] if plugin_locale&.has_key?(type)
+
+      filename = File.join(path, "#{locale}.js")
+      return [locale, filename] if File.exist?(filename)
+    end
+
+    # try again, but this time only with the language itself
+    locale_chain = locale_chain.map { |l| l.split(/[-_]/)[0] }
+      .uniq.reject { |l| locale_chain.include?(l) }
+    unless locale_chain.empty?
+      locale_data = find_locale(locale_chain, path, type, false)
+      return locale_data if locale_data
+    end
+
+    # English should alyways work
+    ["en", File.join(path, "en.js")] if fallback_to_english
+  end
+
   def self.moment_formats
     result = ""
     result << moment_format_function('short_date_no_year')
@@ -163,23 +203,13 @@ def self.moment_format_function(name)
     "moment.fn.#{name.camelize(:lower)} = function(){ return this.format('#{format}'); };\n"
   end
 
-  def self.moment_locale(locale_str)
-    # moment.js uses a different naming scheme for locale files
-    locale_str = locale_str.tr('_', '-').downcase
-    filename = "#{Rails.root}/lib/javascripts/moment_locale/#{locale_str}.js"
-
-    # try the language without the territory
-    locale_str = locale_str.split("-")[0]
-    filename = "#{Rails.root}/lib/javascripts/moment_locale/#{locale_str}.js" unless File.exists?(filename)
-
-    File.exists?(filename) ? File.read(filename) << "\n" : ""
+  def self.moment_locale(locale)
+    _, filename = find_moment_locale([locale])
+    filename && File.exist?(filename) ? File.read(filename) << "\n" : ""
   end
 
-  def self.generate_message_format(message_formats, locale_str)
-    formats = message_formats.map { |k, v| k.inspect << " : " << compile_message_format(locale_str, v) }.join(", ")
-
-    filename = "#{Rails.root}/lib/javascripts/locale/#{locale_str}.js"
-    filename = "#{Rails.root}/lib/javascripts/locale/en.js" unless File.exists?(filename)
+  def self.generate_message_format(message_formats, locale, filename)
+    formats = message_formats.map { |k, v| k.inspect << " : " << compile_message_format(filename, locale, v) }.join(", ")
 
     result = "MessageFormat = {locale: {}};\n"
     result << "I18n._compiledMFs = {#{formats}};\n"
@@ -203,10 +233,9 @@ def self.with_context
     end
   end
 
-  def self.compile_message_format(locale, format)
+  def self.compile_message_format(path, locale, format)
     with_context do |ctx|
-      path = "#{Rails.root}/lib/javascripts/locale/#{locale}.js"
-      ctx.load(path) if File.exists?(path)
+      ctx.load(path) if File.exist?(path)
       ctx.eval("mf = new MessageFormat('#{locale}');")
       ctx.eval("mf.precompile(mf.parse(#{format.inspect}))")
     end