Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

CLDR-15071 Fix dtd tests #2702

Merged
merged 5 commits into from
Feb 9, 2023
Merged

CLDR-15071 Fix dtd tests #2702

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
610d490
CLDR-15071 Fix dtd tests
macchiati Feb 8, 2023
2630bf7
CLDR-15071 Added extra value for cr@alt
macchiati Feb 8, 2023
79dd20b
CLDR-15071 Patch for JSON
macchiati Feb 8, 2023
99a6839
CLDR-15071 fix test to check all paths
macchiati Feb 8, 2023
52cc0cc
CLDR-15071 Gradually fixing problems that only show up on the server.
macchiati Feb 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions common/dtd/ldml.dtd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ CLDR data files are interpreted according to the LDML specification (http://unic
<!--@METADATA-->
<!--@DEPRECATED:true, false-->
<!ATTLIST language references CDATA #IMPLIED >
<!--@MATCH:any-->
<!--@METADATA-->

<!ELEMENT script ( #PCDATA ) >
Expand Down Expand Up @@ -465,6 +466,7 @@ CLDR data files are interpreted according to the LDML specification (http://unic
<!--@METADATA-->
<!--@DEPRECATED-->
<!ATTLIST exemplarCharacters references CDATA #IMPLIED >
<!--@MATCH:any-->
<!--@METADATA-->
<!ATTLIST exemplarCharacters validSubLocales CDATA #IMPLIED >
<!--@VALUE-->
Expand Down Expand Up @@ -2825,10 +2827,11 @@ CLDR data files are interpreted according to the LDML specification (http://unic

<!ELEMENT cr ( #PCDATA ) >
<!ATTLIST cr alt NMTOKENS #IMPLIED >
<!--@MATCH:literal/variant-->
<!--@MATCH:literal/variant, proposed, short-->
<!ATTLIST cr draft (approved | contributed | provisional | unconfirmed) #IMPLIED >
<!--@METADATA-->
<!ATTLIST cr references CDATA #IMPLIED >
<!--@MATCH:any-->
<!--@METADATA-->

<!-- # Use the cr element instead, with ICU syntax. -->
Expand Down Expand Up @@ -3123,7 +3126,7 @@ CLDR data files are interpreted according to the LDML specification (http://unic
<!--@MATCH:or/range/-1.0E20~1.0E20||literal/-x, 0, 0.x, NaN, -Inf, Inf, x,x, x.x-->
<!--@VALUE-->
<!ATTLIST rbnfrule radix CDATA #IMPLIED >
<!--@MATCH:literal/1,000, 100, 1000, 100000, 20-->
<!--@MATCH:literal/1,000, 100, 1000, 100000, 5, 20, 400, 8000, 160,000, 3,200,000, 64,000,000-->
<!--@VALUE-->
<!ATTLIST rbnfrule decexp CDATA #IMPLIED >
<!--@VALUE-->
Expand Down
6 changes: 3 additions & 3 deletions common/dtd/ldmlSupplemental.dtd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ CLDR data files are interpreted according to the LDML specification (http://unic

<!ELEMENT era EMPTY >
<!ATTLIST era type NMTOKEN #REQUIRED >
<!--@MATCH:range/0~250-->
<!--@MATCH:range/-2~250-->
<!ATTLIST era start CDATA #IMPLIED >
<!--@MATCH:time/yyyy-MM-dd-->
<!--@VALUE-->
Expand All @@ -254,6 +254,7 @@ CLDR data files are interpreted according to the LDML specification (http://unic
<!--@MATCH:regex/[a-z0-9]{3,8}(\-[a-z0-9]{3,8})*-->
<!--@VALUE-->
<!ATTLIST era aliases NMTOKENS #IMPLIED >
<!--@MATCH:set/regex/[a-z0-9]{2,8}(\-[a-z0-9]{3,8})*-->
<!--@VALUE-->
<!ATTLIST era named (true | false) #IMPLIED >
<!--@VALUE-->
Expand Down Expand Up @@ -444,7 +445,6 @@ CLDR data files are interpreted according to the LDML specification (http://unic
<!--@DEPRECATED-->
<!ATTLIST unitPreferences draft (approved | contributed | provisional | unconfirmed) #IMPLIED >
<!--@METADATA-->
<!--@DEPRECATED-->

<!ELEMENT unitPreference ( #PCDATA ) >
<!--@ORDERED-->
Expand Down Expand Up @@ -929,7 +929,7 @@ CLDR data files are interpreted according to the LDML specification (http://unic
<!--@MATCH:validity/locale-->
<!--@VALUE-->
<!ATTLIST likelySubtag origin NMTOKENS #IMPLIED >
<!--@MATCH:set/literal/sil22,wikidata,special-->
<!--@MATCH:set/literal/sil1, wikidata, special-->
<!--@METADATA-->

<!ELEMENT metazoneInfo ( timezone* ) >
Expand Down
1 change: 1 addition & 0 deletions tools/cldr-code/src/main/java/org/unicode/cldr/util/DtdData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ private Attribute(DtdType dtdType, Element element2, String aName, Mode mode2, S
element = element2;
name = aName.intern();
if (name.equals("draft") // normally never permitted on elements with children, but special cases...
&& dtdType == DtdType.ldml
&& !DRAFT_ON_NON_LEAF_ALLOWED.contains(element.getName())) {
int elementChildrenCount = element.getChildren().size();
if (elementChildrenCount > 1
Expand Down
1 change: 1 addition & 0 deletions tools/cldr-code/src/test/java/org/unicode/cldr/json/LdmlConvertRulesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ void testSplittableAttributes() {

// Handled manually
jsonSplittableAttrs.add(Pair.of("defaultContent", "locales"));
jsonSplittableAttrs.add(Pair.of("era", "aliases"));

// handled as calendarPreferenceData
jsonSplittableAttrs.add(Pair.of("calendarPreference", "ordering"));
Expand Down
82 changes: 50 additions & 32 deletions tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPaths.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,15 @@
import org.unicode.cldr.util.PathHeader.SectionId;
import org.unicode.cldr.util.PathStarrer;
import org.unicode.cldr.util.StandardCodes;
import org.unicode.cldr.util.SupplementalDataInfo;
import org.unicode.cldr.util.XMLFileReader;
import org.unicode.cldr.util.XPathParts;

import com.google.common.collect.ImmutableSet;

public class TestPaths extends TestFmwkPlus {
static CLDRConfig testInfo = CLDRConfig.getInstance();
static final CLDRConfig testInfo = CLDRConfig.getInstance();
static final SupplementalDataInfo SDI = testInfo.getSupplementalDataInfo(); // Load first, before XPartPaths is called

public static void main(String[] args) {
new TestPaths().run(args);
Expand Down Expand Up @@ -294,7 +296,8 @@ public CheckDeprecated(TestPaths testPaths) {
static final Set<String> ALLOWED = new HashSet<>(Arrays.asList("postalCodeData", "postCodeRegex"));
static final Set<String> OK_IF_MISSING = new HashSet<>(Arrays.asList("alt", "draft", "references"));

public boolean check(DtdData dtdData, XPathParts parts, String fullName) {
public boolean check(XPathParts parts, String fullName) {
DtdData dtdData = parts.getDtdData();
for (int i = 0; i < parts.size(); ++i) {
String elementName = parts.getElement(i);
if (dtdData.isDeprecated(elementName, "*", "*")) {
Expand All @@ -313,17 +316,17 @@ public boolean check(DtdData dtdData, XPathParts parts, String fullName) {
if (dtdData.isDeprecated(elementName, attributeName, "*")) {
if (attributeName.equals("draft")) {
testPaths.errln("Deprecated attribute in data: "
+ dtdData.dtdType
+ ":" + elementName
+ ":" + attributeName
+ " \t;" + fullName +
" - consider adding to DtdData.DRAFT_ON_NON_LEAF_ALLOWED if you are sure this is ok.");
+ dtdData.dtdType
+ ":" + elementName
+ ":" + attributeName
+ " \t;" + fullName +
" - consider adding to DtdData.DRAFT_ON_NON_LEAF_ALLOWED if you are sure this is ok.");
} else {
testPaths.errln("Deprecated attribute in data: "
+ dtdData.dtdType
+ ":" + elementName
+ ":" + attributeName
+ " \t;" + fullName);
+ dtdData.dtdType
+ ":" + elementName
+ ":" + attributeName
+ " \t;" + fullName);
}
return true;
}
Expand Down Expand Up @@ -435,31 +438,26 @@ public void TestNonLdml() {
if (++count > maxPerDirectory) {
break;
}
DtdType type = null;
DtdData dtdData = null;
String fullName = dir2 + "/" + file;
for (Pair<String, String> pathValue : XMLFileReader.loadPathValues(fullName, new ArrayList<Pair<String, String>>(), true)) {
String path = pathValue.getFirst();
final String value = pathValue.getSecond();
XPathParts parts = XPathParts.getFrozenInstance(path);
if (dtdData == null) {
type = DtdType.valueOf(parts.getElement(0));
dtdData = DtdData.getInstance(type);
}
DtdData dtdData = parts.getDtdData();
DtdType type = dtdData.dtdType;

XPathParts pathParts = XPathParts.getFrozenInstance(path);
String finalElementString = pathParts.getElement(-1);
String finalElementString = parts.getElement(-1);
Element finalElement = dtdData.getElementFromName().get(finalElementString);
if (!haveErrorsAlready.contains(finalElement)) {
ElementType elementType = finalElement.getType();
// HACK!!
if (pathParts.size() > 1 && "identity".equals(pathParts.getElement(1))) {
if (parts.size() > 1 && "identity".equals(parts.getElement(1))) {
elementType = ElementType.EMPTY;
logKnownIssue("cldrbug:9784", "fix TODO's in Attribute validity tests");
} else if (pathParts.size() > 2
&& "validity".equals(pathParts.getElement(2))
} else if (parts.size() > 2
&& "validity".equals(parts.getElement(2))
&& value.isEmpty()) {
String typeValue = pathParts.getAttributeValue(-1, "type");
String typeValue = parts.getAttributeValue(-1, "type");
if ("TODO".equals(typeValue)
|| "locale".equals(typeValue)) {
elementType = ElementType.EMPTY;
Expand All @@ -479,24 +477,27 @@ public void TestNonLdml() {
}
}

if (checkDeprecated.check(dtdData, parts, fullName)) {
if (checkDeprecated.check(parts, fullName)) {
break;
}

String last = parts.getElement(-1);
if (skipLast.contains(last)) {
continue;
}

checkParts(fileName + "/" + file, parts);

String dpath = CLDRFile.getDistinguishingXPath(path, normalizedPath);
if (!dpath.equals(path)) {
checkParts(dpath, dtdData);
checkParts(fileName + "/" + file, dpath);
}
if (!normalizedPath.equals(path) && !normalizedPath[0].equals(dpath)) {
checkParts(normalizedPath[0], dtdData);
checkParts(fileName + "/" + file, normalizedPath[0]);
}
parts = parts.cloneAsThawed();
counter = removeNonDistinguishing(parts, dtdData, counter, removed, nonFinalValues);
String cleaned = parts.toString();
XPathParts mutableParts = parts.cloneAsThawed();
counter = removeNonDistinguishing(mutableParts, dtdData, counter, removed, nonFinalValues);
String cleaned = mutableParts.toString();
Pair<String, String> pair = Pair.of(type == DtdType.ldml ? file : type.toString(), cleaned);
if (seen.contains(pair)) {
// parts.set(path);
Expand Down Expand Up @@ -526,8 +527,12 @@ public void TestNonLdml() {
checkDeprecated.show(getInclusion());
}

private void checkParts(String path, DtdData dtdData) {
XPathParts parts = XPathParts.getFrozenInstance(path);
private void checkParts(String file, String path) {
checkParts(file, XPathParts.getFrozenInstance(path));
}

public void checkParts(String file, XPathParts parts) {
DtdData dtdData = parts.getDtdData();
Element current = dtdData.ROOT;
for (int i = 0; i < parts.size(); ++i) {
String elementName = parts.getElement(i);
Expand All @@ -538,13 +543,26 @@ private void checkParts(String path, DtdData dtdData) {
if (!assertNotNull("element", current)) {
return; // failed
}
assertFalse(file + "/" + elementName + " deprecated", current.isDeprecated());
}
for (String attributeName : parts.getAttributeKeys(i)) {
Attribute attribute = current.getAttributeNamed(attributeName);
if (!assertNotNull("attribute", attribute)) {
return; // failed
}
// later, check values
assertFalse(file + "/" + elementName + "@" + attributeName + " deprecated", attribute.isDeprecated());

String value = parts.getAttributeValue(i, attributeName);
switch(attribute.getValueStatus(value)) {
case valid:
break;
default:
errln(file + "/" + elementName + "@" + attributeName
+ ", expected match to: " + attribute.getMatchString()
+ " actual: «" + value + "»");
attribute.getValueStatus(value);
break;
}
}
}
}
Expand Down