Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow dev i18n to be more concurrent #20159

Merged
merged 8 commits into from
Jul 4, 2022
Merged

Allow dev i18n to be more concurrent #20159

Changes from 2 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
f8c10fd
Fix races in i18n code
zeripath Jun 28, 2022
23f2a3b
Update modules/translation/i18n/i18n.go
zeripath Jun 28, 2022
bd7861c
Merge remote-tracking branch 'origin/main' into fix-races-in-i18n
zeripath Jun 28, 2022
879f49e
further improvements
zeripath Jun 28, 2022
edb5dd5
Update modules/translation/i18n/i18n.go
zeripath Jun 29, 2022
0c5a0b5
Merge remote-tracking branch 'origin/main' into fix-races-in-i18n
zeripath Jun 29, 2022
e596b70
Merge branch 'main' into fix-races-in-i18n
6543 Jul 3, 2022
3ff6d5f
Merge branch 'main' into fix-races-in-i18n
6543 Jul 4, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
242 changes: 170 additions & 72 deletions modules/translation/i18n/i18n.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,57 +25,75 @@ var (
)

type locale struct {
// This mutex will be set if we have live-reload enabled (e.g. dev mode)
reloadMu *sync.RWMutex

store *LocaleStore
langName string
textMap map[int]string // the map key (idx) is generated by store's textIdxMap

idxToMsgMap map[int]string // the map idx is generated by store's trKeyToIdxMap

sourceFileName string
sourceFileInfo os.FileInfo
lastReloadCheckTime time.Time
}

type LocaleStore struct {
reloadMu *sync.Mutex // for non-prod(dev), use a mutex for live-reload. for prod, no mutex, no live-reload.
// This mutex will be set if we have live-reload enabled (e.g. dev mode)
reloadMu *sync.RWMutex

langNames []string
langDescs []string

localeMap map[string]*locale
textIdxMap map[string]int
// these need to be locked when live-reloading
localeMap map[string]*locale
trKeyToIdxMap map[string]int

defaultLang string
}

func NewLocaleStore(isProd bool) *LocaleStore {
ls := &LocaleStore{localeMap: make(map[string]*locale), textIdxMap: make(map[string]int)}
store := &LocaleStore{localeMap: make(map[string]*locale), trKeyToIdxMap: make(map[string]int)}
if !isProd {
ls.reloadMu = &sync.Mutex{}
store.reloadMu = &sync.RWMutex{}
}
return ls
return store
}

// AddLocaleByIni adds locale by ini into the store
// if source is a string, then the file is loaded. in dev mode, the file can be live-reloaded
// if source is a string, then the file is loaded. In dev mode, this file will be checked for live-reloading
// if source is a []byte, then the content is used
func (ls *LocaleStore) AddLocaleByIni(langName, langDesc string, source interface{}) error {
if _, ok := ls.localeMap[langName]; ok {
func (store *LocaleStore) AddLocaleByIni(langName, langDesc string, source interface{}) error {
if store.reloadMu != nil {
store.reloadMu.Lock()
defer store.reloadMu.Unlock()
}
if _, ok := store.localeMap[langName]; ok {
return ErrLocaleAlreadyExist
}

lc := &locale{store: ls, langName: langName}
l := &locale{store: store, langName: langName}
if store.reloadMu != nil {
l.reloadMu = &sync.RWMutex{}
l.reloadMu.Lock()
defer l.reloadMu.Unlock()
}

if fileName, ok := source.(string); ok {
lc.sourceFileName = fileName
lc.sourceFileInfo, _ = os.Stat(fileName) // live-reload only works for regular files. the error can be ignored
l.sourceFileName = fileName
l.sourceFileInfo, _ = os.Stat(fileName) // live-reload only works for regular files. the error can be ignored
}

ls.langNames = append(ls.langNames, langName)
ls.langDescs = append(ls.langDescs, langDesc)
ls.localeMap[lc.langName] = lc
store.langNames = append(store.langNames, langName)
store.langDescs = append(store.langDescs, langDesc)
store.localeMap[l.langName] = l

return ls.reloadLocaleByIni(langName, source)
return store.reloadLocaleByIni(langName, source)
}

func (ls *LocaleStore) reloadLocaleByIni(langName string, source interface{}) error {
// store.reloadMu and l.reloadMu must be locked if it exists before calling this function
// this function arguably belongs to locale rather than store but both need to be locked
func (store *LocaleStore) reloadLocaleByIni(langName string, source interface{}) error {
iniFile, err := ini.LoadSources(ini.LoadOptions{
IgnoreInlineComment: true,
UnescapeValueCommentSymbols: true,
Expand All @@ -85,114 +103,194 @@ func (ls *LocaleStore) reloadLocaleByIni(langName string, source interface{}) er
}
iniFile.BlockMode = false

lc := ls.localeMap[langName]
lc.textMap = make(map[int]string)
l := store.localeMap[langName]
l.idxToMsgMap = make(map[int]string)

for _, section := range iniFile.Sections() {
for _, key := range section.Keys() {

var trKey string
if section.Name() == "" || section.Name() == "DEFAULT" {
trKey = key.Name()
} else {
trKey = section.Name() + "." + key.Name()
}
textIdx, ok := ls.textIdxMap[trKey]

// Instead of storing the key strings in multiple different maps we compute a idx which will act as numeric code for key
// This reduces the size of the locale idxToMsgMaps
idx, ok := store.trKeyToIdxMap[trKey]
if !ok {
textIdx = len(ls.textIdxMap)
ls.textIdxMap[trKey] = textIdx
idx = len(store.trKeyToIdxMap)
store.trKeyToIdxMap[trKey] = idx
}
lc.textMap[textIdx] = key.Value()
l.idxToMsgMap[idx] = key.Value()
}
}
iniFile = nil
return nil
}

func (ls *LocaleStore) HasLang(langName string) bool {
_, ok := ls.localeMap[langName]
func (store *LocaleStore) HasLang(langName string) bool {
if store.reloadMu != nil {
store.reloadMu.RLock()
defer store.reloadMu.RUnlock()
}

_, ok := store.localeMap[langName]
return ok
}

func (ls *LocaleStore) ListLangNameDesc() (names, desc []string) {
return ls.langNames, ls.langDescs
func (store *LocaleStore) ListLangNameDesc() (names, desc []string) {
return store.langNames, store.langDescs
}

// SetDefaultLang sets default language as a fallback
func (ls *LocaleStore) SetDefaultLang(lang string) {
ls.defaultLang = lang
func (store *LocaleStore) SetDefaultLang(lang string) {
store.defaultLang = lang
}

// Tr translates content to target language. fall back to default language.
func (ls *LocaleStore) Tr(lang, trKey string, trArgs ...interface{}) string {
l, ok := ls.localeMap[lang]
func (store *LocaleStore) Tr(lang, trKey string, trArgs ...interface{}) string {
if store.reloadMu != nil {
store.reloadMu.RLock()
}

l, ok := store.localeMap[lang]
if !ok {
l, ok = ls.localeMap[ls.defaultLang]
l, ok = store.localeMap[store.defaultLang]
}
if store.reloadMu != nil {
store.reloadMu.RUnlock()
}
if ok {
return l.Tr(trKey, trArgs...)
}
return trKey
}

// this function will assume that the l.reloadMu has been RLocked if it already exists
func (l *locale) reloadIfNeeded() {
if l.store.reloadMu == nil {
return
}

now := time.Now()
if now.Sub(l.lastReloadCheckTime) < time.Second || l.sourceFileInfo == nil || l.sourceFileName == "" {
return
}

l.reloadMu.RUnlock()
l.reloadMu.Lock() // (NOTE: a pre-emption can occur between these two locks so we need to recheck)

if now.Sub(l.lastReloadCheckTime) < time.Second || l.sourceFileInfo == nil || l.sourceFileName == "" {
l.reloadMu.Unlock()
l.reloadMu.RLock()
return
}

l.lastReloadCheckTime = now
sourceFileInfo, err := os.Stat(l.sourceFileName)
if err != nil || sourceFileInfo.ModTime().Equal(l.sourceFileInfo.ModTime()) {
l.reloadMu.Unlock()
l.reloadMu.RLock()
return
}

l.store.reloadMu.Lock()
err = l.store.reloadLocaleByIni(l.langName, l.sourceFileName)
l.store.reloadMu.Unlock()

if err == nil {
l.sourceFileInfo = sourceFileInfo
} else {
log.Error("unable to live-reload the locale file %q, err: %v", l.sourceFileName, err)
}

l.reloadMu.Unlock()
l.reloadMu.RLock()
}

// Tr translates content to locale language. fall back to default language.
func (l *locale) Tr(trKey string, trArgs ...interface{}) string {
if l.store.reloadMu != nil {
l.store.reloadMu.Lock()
defer l.store.reloadMu.Unlock()
now := time.Now()
if now.Sub(l.lastReloadCheckTime) >= time.Second && l.sourceFileInfo != nil && l.sourceFileName != "" {
l.lastReloadCheckTime = now
if sourceFileInfo, err := os.Stat(l.sourceFileName); err == nil && !sourceFileInfo.ModTime().Equal(l.sourceFileInfo.ModTime()) {
if err = l.store.reloadLocaleByIni(l.langName, l.sourceFileName); err == nil {
l.sourceFileInfo = sourceFileInfo
} else {
log.Error("unable to live-reload the locale file %q, err: %v", l.sourceFileName, err)
}
}
}
if l.reloadMu != nil {
l.reloadMu.RLock()
defer l.reloadMu.RUnlock()
l.reloadIfNeeded()
}

msg, _ := l.tryTr(trKey, trArgs...)
return msg
}

func (l *locale) tryTr(trKey string, trArgs ...interface{}) (msg string, found bool) {
trMsg := trKey
textIdx, ok := l.store.textIdxMap[trKey]

if l.store.reloadMu != nil {
l.store.reloadMu.RLock()
}
idx, ok := l.store.trKeyToIdxMap[trKey]
if l.store.reloadMu != nil {
l.store.reloadMu.RUnlock()
}

if ok {
if msg, found = l.textMap[textIdx]; found {
if msg, found = l.idxToMsgMap[idx]; found {
trMsg = msg // use current translation
} else if l.langName != l.store.defaultLang {
if def, ok := l.store.localeMap[l.store.defaultLang]; ok {
return def.tryTr(trKey, trArgs...)

// attempt to get the default language from the locale store
if l.store.reloadMu != nil {
l.store.reloadMu.RLock()
}
def, ok := l.store.localeMap[l.store.defaultLang]
if l.store.reloadMu != nil {
l.store.reloadMu.RUnlock()
}

if ok {
if def.reloadMu != nil {
def.reloadMu.RLock()
}
if msg, found = def.idxToMsgMap[idx]; found {
trMsg = msg // use current translation
}
if def.reloadMu != nil {
def.reloadMu.RUnlock()
}
zeripath marked this conversation as resolved.
Show resolved Hide resolved
}
} else if !setting.IsProd {
log.Error("missing i18n translation key: %q", trKey)
}
}

if len(trArgs) > 0 {
fmtArgs := make([]interface{}, 0, len(trArgs))
for _, arg := range trArgs {
val := reflect.ValueOf(arg)
if val.Kind() == reflect.Slice {
// before, it can accept Tr(lang, key, a, [b, c], d, [e, f]) as Sprintf(msg, a, b, c, d, e, f), it's an unstable behavior
// now, we restrict the strange behavior and only support:
// 1. Tr(lang, key, [slice-items]) as Sprintf(msg, items...)
// 2. Tr(lang, key, args...) as Sprintf(msg, args...)
if len(trArgs) == 1 {
for i := 0; i < val.Len(); i++ {
fmtArgs = append(fmtArgs, val.Index(i).Interface())
}
} else {
log.Error("the args for i18n shouldn't contain uncertain slices, key=%q, args=%v", trKey, trArgs)
break
if !found && !setting.IsProd {
log.Error("missing i18n translation key: %q", trKey)
}

if len(trArgs) == 0 {
return trMsg, found
}

fmtArgs := make([]interface{}, 0, len(trArgs))
for _, arg := range trArgs {
val := reflect.ValueOf(arg)
if val.Kind() == reflect.Slice {
// before, it can accept Tr(lang, key, a, [b, c], d, [e, f]) as Sprintf(msg, a, b, c, d, e, f), it's an unstable behavior
// now, we restrict the strange behavior and only support:
// 1. Tr(lang, key, [slice-items]) as Sprintf(msg, items...)
// 2. Tr(lang, key, args...) as Sprintf(msg, args...)
if len(trArgs) == 1 {
for i := 0; i < val.Len(); i++ {
fmtArgs = append(fmtArgs, val.Index(i).Interface())
}
} else {
fmtArgs = append(fmtArgs, arg)
log.Error("the args for i18n shouldn't contain uncertain slices, key=%q, args=%v", trKey, trArgs)
break
}
} else {
fmtArgs = append(fmtArgs, arg)
}
return fmt.Sprintf(trMsg, fmtArgs...), found
}
return trMsg, found

return fmt.Sprintf(trMsg, fmtArgs...), found
}

// ResetDefaultLocales resets the current default locales
Expand Down