Snakeyaml 1.33 and exit handling refactor

core/src/main/java/oracle/weblogic/deploy/yaml/AbstractYamlTranslator.java

@@ -36,6 +36,7 @@ public abstract class AbstractYamlTranslator {
 
     private final boolean useOrderedDict;
     private final String fileName;
+    private final int codePointsLimit;
 
     protected abstract PlatformLogger getLogger();
     protected abstract String getClassName();
@@ -46,9 +47,10 @@ public abstract class AbstractYamlTranslator {
     // override to write a list of documents as Python dictionaries to the YAML
     public abstract void dumpDocuments(List<?> documents) throws YamlException;
 
-    protected AbstractYamlTranslator(String fileName, boolean useOrderedDict) {
+    protected AbstractYamlTranslator(String fileName, boolean useOrderedDict, int codePointsLimit) {
         this.fileName = fileName;
         this.useOrderedDict = useOrderedDict;
+        this.codePointsLimit = codePointsLimit;
     }
 
     /**
@@ -120,7 +122,7 @@ protected void dumpInternal(List<?> data, Writer outputWriter) throws YamlExcept
 
         if (outputWriter != null) {
             DumperOptions dumperOptions = getDefaultDumperOptions();
-            YamlRepresenter representer = new YamlRepresenter();
+            YamlRepresenter representer = new YamlRepresenter(dumperOptions);
             Yaml yaml = new Yaml(representer, dumperOptions);
 
             try {
@@ -139,6 +141,11 @@ private LoaderOptions getDefaultLoaderOptions() {
         // Turning on setProcessComments seems to trigger a parsing bug when dealing with
         // tags with no value so leave it off...
         //
+        if (this.codePointsLimit > 0) {
+            result.setCodePointLimit(this.codePointsLimit);
+        } else if (this.codePointsLimit < 0 ){
+            getLogger().fine("WLSDPLY-18111", this.codePointsLimit);
+        }
         return result;
     }
 

core/src/main/java/oracle/weblogic/deploy/yaml/YamlRepresenter.java

@@ -27,6 +27,10 @@
  */
 public class YamlRepresenter extends Representer {
 
+  public YamlRepresenter(DumperOptions dumperOptions) {
+    super(dumperOptions);
+  }
+
   @Override
   protected Node representMapping(Tag tag, Map<?, ?> mapping, DumperOptions.FlowStyle flowStyle) {
     MappingNode node = (MappingNode) super.representMapping(tag, mapping, flowStyle);

core/src/main/java/oracle/weblogic/deploy/yaml/YamlStreamTranslator.java

@@ -43,7 +43,22 @@ public YamlStreamTranslator(String streamFileName, InputStream yamlStream) {
      * @param useOrderedDict whether or not to use an ordered dictionary to maintain the order
      */
     public YamlStreamTranslator(String streamFileName, InputStream yamlStream, boolean useOrderedDict) {
-        super(streamFileName, useOrderedDict);
+        super(streamFileName, useOrderedDict, 0);
+        this.streamFileName = streamFileName;
+        this.yamlStream = yamlStream;
+        this.yamlOutputWriter = null;
+    }
+
+    /**
+     * The constructor that allows control of ordering.
+     *
+     * @param streamFileName the name of the file used to create the InputStream (used only for logging purposes)
+     * @param yamlStream the input stream
+     * @param useOrderedDict whether or not to use an ordered dictionary to maintain the order
+     * @param maxCodePoints the maximum number of characters that the parser will accept
+     */
+    public YamlStreamTranslator(String streamFileName, InputStream yamlStream, boolean useOrderedDict, int maxCodePoints) {
+        super(streamFileName, useOrderedDict, maxCodePoints);
         this.streamFileName = streamFileName;
         this.yamlStream = yamlStream;
         this.yamlOutputWriter = null;
@@ -56,7 +71,7 @@ public YamlStreamTranslator(String streamFileName, InputStream yamlStream, boole
      * @param yamlOutputWriter the Writer to use for writing the YAML output
      */
     public YamlStreamTranslator(String streamFileName, Writer yamlOutputWriter) {
-        super(streamFileName, true);
+        super(streamFileName, true, 0);
         this.streamFileName = streamFileName;
         this.yamlStream = null;
         this.yamlOutputWriter = yamlOutputWriter;

core/src/main/java/oracle/weblogic/deploy/yaml/YamlTranslator.java

@@ -32,7 +32,7 @@ public class YamlTranslator extends AbstractYamlTranslator {
      * @throws IllegalArgumentException if the file name is null or does not point to a valid, existing file.
      */
     public YamlTranslator(String fileName) {
-        this(fileName, false);
+        this(fileName, false, 0);
     }
 
     /**
@@ -43,7 +43,21 @@ public YamlTranslator(String fileName) {
      * @throws IllegalArgumentException if the file name is null or does not point to a valid, existing file.
      */
     public YamlTranslator(String fileName, boolean useOrderedDict) {
-        super(fileName, useOrderedDict);
+        super(fileName, useOrderedDict, 0);
+        this.yamlFile = FileUtils.validateExistingFile(fileName);
+    }
+
+    /**
+     * Constructor for parsing YAML file into a Python dictionary, controlling ordering, and
+     * controlling the maximum file size.
+     *
+     * @param fileName the name of the existing YAML file to parse
+     * @param useOrderedDict whether or not to use an ordered dictionary to maintain the order
+     * @param maxCodePoints the maximum number of code points for the input file, or zero to accept the default
+     * @throws IllegalArgumentException if the file name is null or does not point to a valid, existing file.
+     */
+    public YamlTranslator(String fileName, boolean useOrderedDict, int maxCodePoints) {
+        super(fileName, useOrderedDict, maxCodePoints);
         this.yamlFile = FileUtils.validateExistingFile(fileName);
     }
 

core/src/main/python/wlsdeploy/exception/exception_helper.py

@@ -486,4 +486,4 @@ def __handle_unexpected_exception(ex, program_name, class_name, logger):
         # Note: since this is Python 2, it seems we can only get the traceback object via sys.exc_info, and of course only
         # while in the except block handling code
         logger.finer('WLSDPLY-20036', program_name, traceback.format_exception(type(ex), ex, sys.exc_info()[2]))
-    tool_exit.end(logger, ExitCode.ERROR, class_name)
+    tool_exit.end(ExitCode.ERROR, class_name)

core/src/main/python/wlsdeploy/util/model_config.py

@@ -16,6 +16,7 @@
 
 _logger = PlatformLogger('wlsdeploy.config')
 _class_name = 'ModelConfig'
+_config_object = None
 
 # Tool Properties for configuration and default values if properties not loaded
 
@@ -42,10 +43,21 @@
 WLST_EDIT_LOCK_RELEASE_TIMEOUT_DEFAULT = '-1'
 WLST_EDIT_LOCK_EXCLUSIVE_PROP = 'wlst.edit.lock.exclusive'
 WLST_EDIT_LOCK_EXCLUSIVE_DEFAULT = 'false'
+YAML_FILE_MAX_CODE_POINTS_PROP = 'yaml.max.file.size'
+YAML_FILE_MAX_CODE_POINTS_DEFAULT = '0'
 
 # System Property overrides for WLST timeout properties
 SYS_PROP_PREFIX = 'wdt.config.'
 
+# This method is used to get the model configuration singleton object.
+# There is an implicit assumption that the object is created by the
+# model context
+def get_model_config(program_name='unknown'):
+    global _config_object
+    if _config_object is None:
+        _config_object = ModelConfiguration(program_name)
+    return _config_object
+
 
 class ModelConfiguration(object):
     """
@@ -137,6 +149,13 @@ def get_wlst_edit_lock_exclusive(self):
         """
         return self._get_from_dict(WLST_EDIT_LOCK_EXCLUSIVE_PROP, WLST_EDIT_LOCK_EXCLUSIVE_DEFAULT)
 
+    def get_yaml_file_max_code_points(self):
+        """
+        Returns the exclusive value for startEdit from tool properties
+        :return: the string 'true' or 'false' (default)
+        """
+        return self._get_from_dict_as_long(YAML_FILE_MAX_CODE_POINTS_PROP, YAML_FILE_MAX_CODE_POINTS_DEFAULT)
+
     def _get_from_dict(self, name, default_value=None):
         _method_name = '_get_from_dict'
         _logger.entering(name, default_value, class_name=_class_name, method_name=_method_name)
@@ -173,5 +192,13 @@ def _load_properties_file():
     except IOException, ioe:
         _logger.warning('WLSDPLY-01570', wlsdeploy_path, ioe.getMessage(),
                         class_name=_class_name, method_name=_method_name)
+
+        # Return an empty dict so that failing to load the tool.properties file does
+        # not prevent the code above from working using the default values.  The WLST
+        # unit tests are depending on this behavior until they are refactored to all
+        # copy the tool.properties file into the target/unit_tests/config directory
+        # and setting the WDT_CUSTOM_CONFIG environment variable to point to it.
+        #
+        result = dict()
     _logger.exiting(class_name=_class_name, method_name=_method_name)
     return result

core/src/main/python/wlsdeploy/util/model_context.py

@@ -19,7 +19,7 @@
 from wlsdeploy.util.cla_utils import CommandLineArgUtil
 from wlsdeploy.util import path_utils
 from wlsdeploy.util import string_utils
-from wlsdeploy.util.model_config import ModelConfiguration
+from wlsdeploy.util import model_config
 from wlsdeploy.util.target_configuration import TargetConfiguration
 from wlsdeploy.util.validate_configuration import ValidateConfiguration
 from wlsdeploy.util.weblogic_helper import WebLogicHelper
@@ -52,6 +52,7 @@ def __init__(self, program_name, arg_map):
         self._program_name = program_name
         self._logger = platform_logger.PlatformLogger('wlsdeploy.util')
         self._wls_helper = WebLogicHelper(self._logger)
+        self._model_config = model_config.get_model_config(self._program_name)
 
         self._oracle_home = None
         self._wl_home = None
@@ -100,7 +101,6 @@ def __init__(self, program_name, arg_map):
         self._rcu_db_user = self.DB_USER_DEFAULT
         self._discard_current_edit = False
         self._wait_for_edit_lock = False
-        self._model_config = None
         self._remote = False
         self._skip_archive = False
 
@@ -370,11 +370,8 @@ def __copy__(self):
     def get_model_config(self):
         """
         Return the encapsulated tool properties configuration instance.
-        This will load the ModelConfiguration from the tool properties on the first request
         :return: model configuration instance
         """
-        if self._model_config is None:
-            self._model_config = ModelConfiguration(self._program_name)
         return self._model_config
 
     def get_program_name(self):

core/src/main/python/wlsdeploy/util/model_translator.py

@@ -10,6 +10,7 @@
 
 from wlsdeploy.logging import platform_logger
 from wlsdeploy.exception import exception_helper
+from wlsdeploy.util import model_config
 
 
 class FileToPython(object):
@@ -23,6 +24,7 @@ def __init__(self, file_name, use_ordering=False):
         self.file_name = file_name
         self.use_ordering = use_ordering
         self.logger = platform_logger.PlatformLogger('wlsdeploy.translator')
+        self.model_config = model_config.get_model_config()
 
     def parse(self):
         """
@@ -70,11 +72,12 @@ def _parse_yaml(self):
         """
         _method_name = '_parse_yaml'
 
-        from wlsdeploy.yaml.yaml_translator import YamlToPython as JYamlToPython
+        from wlsdeploy.yaml.yaml_translator import YamlToPython
         self.logger.finer('WLSDPLY-01711', 'YAML', self.file_name,
                           class_name=self._class_name, method_name=_method_name)
         try:
-            return JYamlToPython(self.file_name, self.use_ordering).parse()
+            max_size = self.model_config.get_yaml_file_max_code_points()
+            return YamlToPython(self.file_name, self.use_ordering, max_size).parse()
         except JYamlException, ye:
             translate_ex = exception_helper.create_translate_exception('WLSDPLY-01710', self.file_name,
                                                                        ye.getLocalizedMessage(), error=ye)

core/src/main/python/wlsdeploy/yaml/yaml_translator.py

@@ -33,14 +33,14 @@ class YamlToPython(object):
     """
     _class_name = 'YamlToPython'
 
-    def __init__(self, file_name, use_ordering=False):
+    def __init__(self, file_name, use_ordering=False, max_size=0):
         _method_name = '__init__'
 
         self._file_name = file_name
         self._use_ordering = use_ordering
         self._logger = PlatformLogger('wlsdeploy.yaml')
         try:
-            self._translator = JYamlTranslator(self._file_name, self._use_ordering)
+            self._translator = JYamlTranslator(self._file_name, self._use_ordering, max_size)
         except JIllegalArgumentException, iae:
             yaml_ex = \
                 exception_helper.create_yaml_exception('WLSDPLY-18008', file_name, iae.getLocalizedMessage(), error=iae)

core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties

@@ -1469,6 +1469,7 @@ WLSDPLY-18107=An error occurred while writing the YAML output file {0}: {1}
 WLSDPLY-18108=An error occurred while creating the input stream for YAML input file {0}: {1}
 WLSDPLY-18109=An error occurred while creating the output writer for YAML output file {0}: {1}
 WLSDPLY-18110=An error occurred while closing the yaml output writer for yaml file {0}...continuing: {1}
+WLSDPLY-18111=The YAML parser received an invalid value "{0}" for the maximum file size so ignoring the value...
 
 # New PythonToJava type conversion code
 WLSDPLY-18200=An error occurred because the top-level type to be converted was not a Python dictionary

core/src/test/java/oracle/weblogic/deploy/yaml/YamlTranslatorTest.java

@@ -467,6 +467,16 @@ public void testSecurityConfigModel2() throws Exception {
         assertTrue(((PyRealBoolean) value).getValue(), "RejectEqualOrContainReverseUsername value should be true");
     }
 
+    @Test
+    public void testLargeFileIsSuccessful() throws Exception {
+        File yamlFile = new File("src/test/resources/yaml/big-model.yaml").getAbsoluteFile();
+        YamlTranslator yamlTranslator = new YamlTranslator(yamlFile.getAbsolutePath(), true, 20000000);
+
+        PyDictionary actual = yamlTranslator.parse();
+
+        assertNotNull(actual, "big model file should return dict");
+    }
+
     /**
      * Verify that a lexical error will throw an Exception
      */

installer/src/main/lib/tool.properties

@@ -12,3 +12,7 @@ set.server.groups.timeout=30000
 wlst.edit.lock.acquire.timeout=0
 wlst.edit.lock.release.timeout=-1
 wlst.edit.lock.exclusive=false
+#
+# 0 means to accept the default value, which is 3145728 (i.e., 3 MB)
+#
+yaml.max.file.size=0

pom.xml

@@ -45,7 +45,7 @@
         <weblogic-deploy-installer-name>weblogic-deploy</weblogic-deploy-installer-name>
         <skipTests>false</skipTests>
         <antlr.version>4.9.3</antlr.version>
-        <snakeyaml.version>1.31</snakeyaml.version>
+        <snakeyaml.version>1.33</snakeyaml.version>
 
         <sonar.organization>oracle</sonar.organization>
         <sonar.host.url>https://sonarcloud.io</sonar.host.url>