Skip to content

Fix memory leaks and update PDF/A algorithm for non-document products #845

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Mar 2, 2024
Merged

Fix memory leaks and update PDF/A algorithm for non-document products #845

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9d3bcc6
add unit test first
Feb 26, 2024
d224804
fixed memory leak(s)
Feb 29, 2024
67efb71
Merge branch 'main' into issue_824
al-niessner Feb 29, 2024
469b0b9
check PDF only if documentation
Mar 1, 2024
d2cf090
Merge branch 'issue_824' of github.com:NASA-PDS/validate into issue_824
Mar 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 54 additions & 55 deletions src/main/java/gov/nasa/pds/tools/label/LabelValidator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParserFactory;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.transform.dom.DOMResult;
Expand All @@ -46,8 +45,6 @@
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Attr;
Expand Down Expand Up @@ -98,15 +95,15 @@ public class LabelValidator {
private Map<String, Boolean> configurations = new HashMap<>();
private List<URL> userSchemaFiles;
private List<URL> userSchematronFiles;
private List<Transformer> userSchematronTransformers;
private List<String> userSchematronTransformers;
private XMLReader cachedParser;
private ValidatorHandler cachedValidatorHandler;
private List<Transformer> cachedSchematron;
private List<String> cachedSchematron;
private XMLCatalogResolver resolver;
private Boolean useLabelSchema;
private Boolean useLabelSchematron;
private Boolean skipProductValidation;
private Map<String, Transformer> cachedLabelSchematrons;
private Map<String, String> cachedLabelSchematrons;

public static final String SCHEMA_CHECK = "gov.nasa.pds.tools.label.SchemaCheck";
public static final String SCHEMATRON_CHECK = "gov.nasa.pds.tools.label.SchematronCheck";
Expand All @@ -120,7 +117,6 @@ public class LabelValidator {
private SchemaFactory schemaFactory;
private Schema validatingSchema;
private SchematronTransformer schematronTransformer;
private XPathFactory xPathFactory;

private long filesProcessed = 0;
private double totalTimeElapsed = 0.0;
Expand Down Expand Up @@ -195,8 +191,6 @@ public LabelValidator() throws ParserConfigurationException, TransformerConfigur

documentValidators.add(new DefaultDocumentValidator());
schematronTransformer = new SchematronTransformer();

xPathFactory = new net.sf.saxon.xpath.XPathFactoryImpl();
}

/**
Expand All @@ -215,7 +209,7 @@ public void setSchema(List<URL> schemaFiles) {
*
* @param schematrons A list of transformed schematrons.
*/
public void setSchematrons(List<Transformer> schematrons) {
public void setSchematrons(List<String> schematrons) {
userSchematronTransformers = schematrons;
LOG.debug("setSchematrons:schematrons.size(),schematrons {},{}", schematrons.size(),
schematrons);
Expand All @@ -227,7 +221,7 @@ public void setSchematrons(List<Transformer> schematrons) {
*
* @param schematronMap
*/
public void setLabelSchematrons(Map<String, Transformer> schematronMap) {
public void setLabelSchematrons(Map<String, String> schematronMap) {
cachedLabelSchematrons = schematronMap;
}

Expand Down Expand Up @@ -573,12 +567,9 @@ public synchronized Document parseAndValidate(ProblemHandler handler, URL url)
LOG.debug("parseAndValidate:0003:url,useLabelSchematron,cachedSchematron.size() {},{},{}",
url, useLabelSchematron, cachedSchematron.size());
} else if (userSchematronFiles != null) {
List<Transformer> transformers = new ArrayList<>();
List<String> transformers = new ArrayList<>();
for (URL schematron : userSchematronFiles) {
StreamSource source = new StreamSource(schematron.toString());
source.setSystemId(schematron.toString());
Transformer transformer = schematronTransformer.transform(source, handler);
transformers.add(transformer);
transformers.add(schematronTransformer.fetch(schematron, handler));
}
cachedSchematron = transformers;
LOG.debug("parseAndValidate:0004:url,useLabelSchematron,cachedSchematron.size() {},{},{}",
Expand Down Expand Up @@ -613,31 +604,37 @@ public synchronized Document parseAndValidate(ProblemHandler handler, URL url)
LOG.debug("parseAndValidate:url,skipProductValidation,validateAgainstSchematronFlag {},{},{}",
url, skipProductValidation, validateAgainstSchematronFlag);

for (Transformer schematron : cachedSchematron) {
long singleSchematronStartTime = System.currentTimeMillis();
if (!validateAgainstSchematronFlag) {
continue; // Skip the validation if validateAgainstSchematronFlag is not true.
}
DOMResult result = new DOMResult();
DOMSource domSource = new DOMSource(xml);
LOG.debug("parseAndValidate:VALIDATING_SCHEMATRON_URL:START {} against schematron", url);
domSource.setSystemId(url.toString());
// Apply the rules specified in the schematron file
schematron.transform(domSource, result);
// Output is svrl:schematron-output document
// Select out svrl:failed-assert nodes and put into problem container
Document reportDoc = (Document) result.getNode();
NodeList nodes =
reportDoc.getElementsByTagNameNS("http://purl.oclc.org/dsdl/svrl", "failed-assert");
for (int i = 0; i < nodes.getLength(); i++) {
Node node = nodes.item(i);
// Add an error for each failed asssert
handler.addProblem(processFailedAssert(url, node, xml));
}
long singleSchematronFinishTime = System.currentTimeMillis();
long singleSchematronTimeElapsed = singleSchematronFinishTime - singleSchematronStartTime;
LOG.debug("parseAndValidate:VALIDATING_SCHEMATRON_URL:ELAPSED {} {}", url,
singleSchematronTimeElapsed);
for (String schematron : cachedSchematron) {
try {
long singleSchematronStartTime = System.currentTimeMillis();
if (!validateAgainstSchematronFlag) {
continue; // Skip the validation if validateAgainstSchematronFlag is not true.
}
DOMResult result = new DOMResult();
DOMSource domSource = new DOMSource(xml);
LOG.debug("parseAndValidate:VALIDATING_SCHEMATRON_URL:START {} against schematron", url);
domSource.setSystemId(url.toString());
// Apply the rules specified in the schematron file
schematronTransformer.transform(schematron).transform(domSource, result);
// Output is svrl:schematron-output document
// Select out svrl:failed-assert nodes and put into problem container
Document reportDoc = (Document) result.getNode();
NodeList nodes =
reportDoc.getElementsByTagNameNS("http://purl.oclc.org/dsdl/svrl", "failed-assert");
for (int i = 0; i < nodes.getLength(); i++) {
Node node = nodes.item(i);
// Add an error for each failed asssert
handler.addProblem(processFailedAssert(url, node, xml));
}
long singleSchematronFinishTime = System.currentTimeMillis();
long singleSchematronTimeElapsed = singleSchematronFinishTime - singleSchematronStartTime;
LOG.debug("parseAndValidate:VALIDATING_SCHEMATRON_URL:ELAPSED {} {}", url,
singleSchematronTimeElapsed);
} catch (TransformerException te) {
// should never get here
handler.addProblem(new ValidationProblem(
new ProblemDefinition(ExceptionType.FATAL, ProblemType.SCHEMATRON_ERROR, te.getLocalizedMessage()),
url)); }
}
LOG.debug("parseAndValidate:VALIDATING_SCHEMATRON_URL:DONE {} against schematron", url);
}
Expand Down Expand Up @@ -872,12 +869,14 @@ public List<String> getSchematrons(NodeList nodeList, URL url, ProblemHandler ha
return results;
}

private List<Transformer> loadLabelSchematrons(List<String> schematronSources, URL url,
private List<String> loadLabelSchematrons(List<String> schematronSources, URL url,
ProblemHandler handler) {
List<Transformer> transformers = new ArrayList<>();
List<String> transformers = new ArrayList<>();
LOG.debug("loadLabelSchematrons:resolver,schematronSources {},{}", resolver, schematronSources);
for (String source : schematronSources) {
try {
String document;

if (resolver != null) {
try {
String absoluteUrl = Utility.makeAbsolute(Utility.getParent(url).toString(), source);
Expand All @@ -893,20 +892,20 @@ private List<Transformer> loadLabelSchematrons(List<String> schematronSources, U
+ "' through the catalog: " + io.getMessage());
}
}
Transformer transformer = cachedLabelSchematrons.get(source);
if (transformer != null) {
transformers.add(transformer);
LOG.debug("loadLabelSchematrons:transformers.add:source {}", source);
} else {
if (!this.cachedLabelSchematrons.containsKey(source)) {
URL sourceUrl = new URL(source);
LOG.debug("loadLabelSchematrons:sourceUrl {}", sourceUrl);
try {
transformer = schematronTransformer.transform(sourceUrl);
cachedLabelSchematrons.put(source, transformer);
} catch (TransformerException te) {
throw new Exception("Schematron '" + source + "' error: " + te.getMessage());
}
document = schematronTransformer.fetch(sourceUrl);
cachedLabelSchematrons.put(source, document);
}
document = cachedLabelSchematrons.get(source);
transformers.add(document);
LOG.debug("loadLabelSchematrons:transformers.add:source {}", source);
} catch (TransformerException te) {
String message = "Schematron '" + source + "' error: " + te.getMessage();
handler.addProblem(new ValidationProblem(
new ProblemDefinition(ExceptionType.ERROR, ProblemType.SCHEMATRON_ERROR, message),
url));
} catch (Exception e) {
String message = "Error occurred while loading schematron: " + e.getMessage();
handler.addProblem(new ValidationProblem(
Expand Down Expand Up @@ -948,7 +947,7 @@ private ValidationProblem processFailedAssert(URL url, Node node, Document doc)
String location = ((Attr) node.getAttributes().getNamedItem("location")).getValue();
SourceLocation sourceLoc = null;
try {
XPath documentPath = xPathFactory.newXPath();
XPath documentPath = new net.sf.saxon.xpath.XPathFactoryImpl().newXPath();
Node failureNode = (Node) documentPath.evaluate(location, doc, XPathConstants.NODE);
sourceLoc = (SourceLocation) failureNode.getUserData(SourceLocation.class.getName());
} catch (XPathExpressionException e) {
Expand Down
6 changes: 2 additions & 4 deletions src/main/java/gov/nasa/pds/tools/label/LocationValidator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import java.util.List;
import java.util.Map;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerConfigurationException;
import org.apache.commons.chain.Catalog;
import org.apache.commons.chain.CatalogFactory;
Expand Down Expand Up @@ -72,7 +71,6 @@ public class LocationValidator {
private RuleContext ruleContext;
private String validationRule;
private String labelExtension;
private String pdfErrorDir;

/*
* Sets the report object in BundleManager static class.
Expand Down Expand Up @@ -349,7 +347,7 @@ public void setSchema(List<URL> schemaFiles) {
LOG.debug("setSchema:schemaFiles {}", schemaFiles);
}

public void setSchematrons(List<Transformer> schematrons) {
public void setSchematrons(List<String> schematrons) {
labelValidator.setSchematrons(schematrons);
}

Expand Down Expand Up @@ -379,7 +377,7 @@ public void addValidator(DocumentValidator validator) {
labelValidator.addValidator(validator);
}

public void setLabelSchematrons(Map<String, Transformer> labelSchematrons) {
public void setLabelSchematrons(Map<String, String> labelSchematrons) {
labelValidator.setLabelSchematrons(labelSchematrons);
}

Expand Down
Loading