Skip to content

Commit

Permalink
Add en fallback in coreLocalizer instead of utilDetect.browserLocales
Browse files Browse the repository at this point in the history 
Include all preferred and fallback langauges in coreLocalizer.localeCodes even if higher-priority ones have 100% string coverage
Fallback to the user's preferred languages instead of directly to English when querying OSM wikibase documentation and Wikidata (re: #7996)
Add `lang` attribute to tag documentation text loaded from OSM wikibase or Wikidata (re: #7963)
  • Loading branch information
@quincylvania
quincylvania committed Sep 23, 2020
1 parent 6b5a553 commit bbbf401
Show file tree
Hide file tree
Showing 5 changed files with 81 additions and 88 deletions.
43 changes: 20 additions & 23 deletions modules/core/localizer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,14 +94,20 @@ export function coreLocalizer() {
.then(() => {
let requestedLocales = (_preferredLocaleCodes || [])
// List of locales preferred by the browser in priority order.
// This always includes an `en` fallback, so we know at least one is valid.
.concat(utilDetect().browserLocales);
.concat(utilDetect().browserLocales)
// fallback to English since it's the only guaranteed complete language
.concat(['en']);

_localeCodes = localesToUseFrom(requestedLocales);
// run iD in the highest-priority locale; the rest are fallbacks
// Run iD in the highest-priority locale; the rest are fallbacks
_localeCode = _localeCodes[0];

const loadStringsPromises = _localeCodes.map(function(code) {
// Will always return the index for `en` if nothing else
const fullCoverageIndex = _localeCodes.findIndex(function(locale) {
return _dataLocales[locale].pct === 1;
});
// We only need to load locales up until we find one with full coverage
const loadStringsPromises = _localeCodes.slice(0, fullCoverageIndex + 1).map(function(code) {
return localizer.loadLocale(code);
});
return Promise.all(loadStringsPromises);
Expand All @@ -116,32 +122,19 @@ export function coreLocalizer() {
function localesToUseFrom(requestedLocales) {
let supportedLocales = _dataLocales;

let toLoad = [];

let toUse = [];
for (let i in requestedLocales) {
let locale = requestedLocales[i];
if (supportedLocales[locale]) {
toLoad.push(locale);
}
if (supportedLocales[locale]) toUse.push(locale);

if (locale.includes('-')) {
// Full locale ('es-ES'), add fallback to the base ('es')
let langPart = locale.split('-')[0];
if (supportedLocales[langPart]) {
toLoad.push(langPart);
}
if (supportedLocales[langPart]) toUse.push(langPart);
}
}

toLoad = utilArrayUniq(toLoad);

// this is guaranteed to always return an index since `en` is always listed
// and `en` always has full coverage
let fullCoverageIndex = toLoad.findIndex(function(locale) {
return supportedLocales[locale].pct === 1;
});
// we only need to load locales up until we find one with full coverage
return toLoad.slice(0, fullCoverageIndex + 1);
// remove duplicates
return utilArrayUniq(toUse);
}

function updateForCurrentLocale() {
Expand Down Expand Up @@ -329,7 +322,11 @@ export function coreLocalizer() {
// Returns the localized text wrapped in an HTML element encoding the locale info
localizer.t.html = function(stringId, replacements, locale) {
const info = localizer.tInfo(stringId, replacements, locale);
return `<span class="localized-text" lang="${info.locale || 'unknown'}">${info.text}</span>`;
return localizer.htmlForLocalizedText(info.text, info.locale);
};

localizer.htmlForLocalizedText = function(text, localeCode) {
return `<span class="localized-text" lang="${localeCode || 'unknown'}">${text}</span>`;
};

localizer.languageName = (code, options) => {
Expand Down
92 changes: 45 additions & 47 deletions modules/services/osm_wikibase.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,22 +32,6 @@ function request(url, callback) {
}


/**
* Get the best string value from the descriptions/labels result
* Note that if mediawiki doesn't recognize language code, it will return all values.
* In that case, fallback to use English.
* @param values object - either descriptions or labels
* @param langCode String
* @returns localized string
*/
function localizedToString(values, langCode) {
if (values) {
values = values[langCode] || values.en;
}
return values ? values.value : '';
}


export default {

init: function() {
Expand Down Expand Up @@ -137,12 +121,16 @@ export default {
var tagSitelink = (params.key && params.value) ? this.toSitelink(params.key, params.value) : false;
var localeSitelink;

if (params.langCode && _localeIDs[params.langCode] === undefined) {
// If this is the first time we are asking about this locale,
// fetch corresponding entity (if it exists), and cache it.
// If there is no such entry, cache `false` value to avoid re-requesting it.
localeSitelink = ('Locale:' + params.langCode).replace(/_/g, ' ').trim();
titles.push(localeSitelink);
if (params.langCodes) {
params.langCodes.forEach(function(langCode) {
if (_localeIDs[langCode] === undefined) {
// If this is the first time we are asking about this locale,
// fetch corresponding entity (if it exists), and cache it.
// If there is no such entry, cache `false` value to avoid re-requesting it.
localeSitelink = ('Locale:' + langCode).replace(/_/g, ' ').trim();
titles.push(localeSitelink);
}
});
}

if (rtypeSitelink) {
Expand Down Expand Up @@ -183,7 +171,7 @@ export default {
action: 'wbgetentities',
sites: 'wiki',
titles: titles.join('|'),
languages: params.langCode,
languages: params.langCodes.join('|'),
languagefallback: 1,
origin: '*',
format: 'json',
Expand All @@ -202,9 +190,6 @@ export default {
var localeID = false;
Object.values(d.entities).forEach(function(res) {
if (res.missing !== '') {
// Simplify access to the localized values
res.description = localizedToString(res.descriptions, params.langCode);
res.label = localizedToString(res.labels, params.langCode);

var title = res.sitelinks.wiki.title;
if (title === rtypeSitelink) {
Expand All @@ -226,7 +211,7 @@ export default {

if (localeSitelink) {
// If locale ID is not found, store false to prevent repeated queries
that.addLocale(params.langCode, localeID);
that.addLocale(params.langCodes[0], localeID);
}

callback(null, result);
Expand All @@ -253,8 +238,10 @@ export default {
//
getDocs: function(params, callback) {
var that = this;
var langCode = localizer.localeCode().toLowerCase();
params.langCode = langCode;
var langCodes = localizer.localeCodes().map(function(code) {
return code.toLowerCase();
});
params.langCodes = langCodes;

this.getEntity(params, function(err, data) {
if (err) {
Expand All @@ -268,21 +255,33 @@ export default {
return;
}

var i;
var description;
for (i in langCodes) {
let code = langCodes[i];
if (entity.descriptions[code] && entity.descriptions[code].language === code) {
description = entity.descriptions[code];
break;
}
}
if (!description && Object.values(entity.descriptions).length) description = Object.values(entity.descriptions)[0];

// prepare result
var result = {
title: entity.title,
description: entity.description,
description: description ? description.value : '',
descriptionLocaleCode: description ? description.language : '',
editURL: 'https://wiki.openstreetmap.org/wiki/' + entity.title
};

// add image
if (entity.claims) {
var imageroot;
var image = that.claimToValue(entity, 'P4', langCode);
var image = that.claimToValue(entity, 'P4', langCodes[0]);
if (image) {
imageroot = 'https://commons.wikimedia.org/w/index.php';
} else {
image = that.claimToValue(entity, 'P28', langCode);
image = that.claimToValue(entity, 'P28', langCodes[0]);
if (image) {
imageroot = 'https://wiki.openstreetmap.org/w/index.php';
}
Expand All @@ -302,21 +301,20 @@ export default {
var tagWiki = that.monolingualClaimToValueObj(data.tag, 'P31');
var keyWiki = that.monolingualClaimToValueObj(data.key, 'P31');

// If exact language code does not exist, try to find the first part before the '-'
// BUG: in some cases, a more elaborate fallback logic might be needed
var langPrefix = langCode.split('-', 2)[0];

// use the first acceptable wiki page
result.wiki =
getWikiInfo(rtypeWiki, langCode, 'inspector.wiki_reference') ||
getWikiInfo(rtypeWiki, langPrefix, 'inspector.wiki_reference') ||
getWikiInfo(rtypeWiki, 'en', 'inspector.wiki_en_reference') ||
getWikiInfo(tagWiki, langCode, 'inspector.wiki_reference') ||
getWikiInfo(tagWiki, langPrefix, 'inspector.wiki_reference') ||
getWikiInfo(tagWiki, 'en', 'inspector.wiki_en_reference') ||
getWikiInfo(keyWiki, langCode, 'inspector.wiki_reference') ||
getWikiInfo(keyWiki, langPrefix, 'inspector.wiki_reference') ||
getWikiInfo(keyWiki, 'en', 'inspector.wiki_en_reference');
var wikis = [rtypeWiki, tagWiki, keyWiki];
for (i in wikis) {
var wiki = wikis[i];
for (var j in langCodes) {
var code = langCodes[j];
var referenceId = (langCodes[0].split('-')[0] !== 'en' && code.split('-')[0] === 'en') ? 'inspector.wiki_en_reference' : 'inspector.wiki_reference';
var info = getWikiInfo(wiki, code, referenceId);
if (info) {
result.wiki = info;
break;
}
}
if (result.wiki) break;
}

callback(null, result);

Expand Down
26 changes: 13 additions & 13 deletions modules/services/wikidata.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { json as d3_json } from 'd3-fetch';

import { utilArrayUniq, utilQsString } from '../util';
import { utilQsString } from '../util';
import { localizer } from '../core/localizer';

var apibase = 'https://www.wikidata.org/w/api.php?';
Expand Down Expand Up @@ -85,15 +85,9 @@ export default {


languagesToQuery: function() {
var localeCode = localizer.localeCode().toLowerCase();
// HACK: en-us isn't a wikidata language. We should really be filtering by
// the languages known to be supported by wikidata.
if (localeCode === 'en-us') localeCode = 'en';
return utilArrayUniq([
localeCode,
localizer.languageCode().toLowerCase(),
'en'
]);
return localizer.localeCodes().map(function(code) {
return code.toLowerCase();
});
},


Expand Down Expand Up @@ -157,14 +151,20 @@ export default {

var i;
var description;
if (entity.descriptions && Object.keys(entity.descriptions).length > 0) {
description = entity.descriptions[Object.keys(entity.descriptions)[0]].value;
for (i in langs) {
let code = langs[i];
if (entity.descriptions[code] && entity.descriptions[code].language === code) {
description = entity.descriptions[code];
break;
}
}
if (!description && Object.values(entity.descriptions).length) description = Object.values(entity.descriptions)[0];

// prepare result
var result = {
title: entity.id,
description: description,
description: description ? description.value : '',
descriptionLocaleCode: description ? description.language : '',
editURL: 'https://www.wikidata.org/wiki/' + entity.id
};

Expand Down
4 changes: 2 additions & 2 deletions modules/ui/tag_reference.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {
select as d3_select
} from 'd3-selection';

import { t } from '../core/localizer';
import { t, localizer } from '../core/localizer';
import { services } from '../services';
import { svgIcon } from '../svg/icon';

Expand Down Expand Up @@ -64,7 +64,7 @@ export function uiTagReference(what) {
_body
.append('p')
.attr('class', 'tag-reference-description')
.html(docs.description || t.html('inspector.no_documentation_key'))
.html(docs.description ? localizer.htmlForLocalizedText(docs.description, docs.descriptionLocaleCode) : t.html('inspector.no_documentation_key'))
.append('a')
.attr('class', 'tag-reference-edit')
.attr('target', '_blank')
Expand Down
4 changes: 1 addition & 3 deletions modules/util/detect.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,7 @@ export function utilDetect(refresh) {
.concat(navigator.languages || [])
.concat([
// old property for backwards compatibility
navigator.userLanguage,
// fallback to English
'en'
navigator.userLanguage
])
// remove any undefined values
.filter(Boolean)
Expand Down

0 comments on commit bbbf401

Please sign in to comment.