Add try/except blocks around external text file read operations to catch encoding-related exceptions

nsiqcppstyle_checker.py

@@ -343,8 +343,13 @@ def __init__(self, filename, data=None):
         lexer = nsiqcppstyle_lexer.lex()
         self.data = data
         if data is None:
-            f = open(filename)
-            self.data = f.read()
+            with open(filename) as f:
+                try:
+                    self.data = f.read()
+                except UnicodeDecodeError as ex:
+                    console.Out.Ci("[ERROR] UnicodeDecodeError in CppLexerNavigator: " + str(ex))
+                    console.Out.Ci("[ERROR] Exception occurred reading file '%s', convert from UTF16LE to UTF8" % (filename))
+                    raise ex
         self.lines = self.data.splitlines()
         lexer.input(self.data)
         index = 0
@@ -983,7 +988,15 @@ def Peek(self):
 
 def ProcessFile(ruleManager, file, data=None):
     #    print file
-    lexer = CppLexerNavigator(file, data)
+    try:
+        lexer = CppLexerNavigator(file, data)
+    except UnicodeDecodeError:
+        # If an exception was thrown (i.e., UnicodeDecodeError), it was
+        # caught, process, logged to stdout, and the exception was raised
+        # again.  At this point in the code, there is nothing else to do
+        # other than skip the processing of the file, which is why we
+        # just return.
+        return
     ContructContextInfo(lexer)
     # Run Rules
     lexer.Reset()

nsiqcppstyle_exe.py

@@ -580,9 +580,9 @@ def __init__(self, targetDir):
         if os.path.isdir(targetDir):
             fsrc = os.path.join(targetDir, "basefilelist.txt")
             if os.path.exists(fsrc):
-                f = open(fsrc)
-                for line in f.readlines():
-                    self.baseFileList[line.strip()] = True
+                with open(fsrc) as f:
+                    for line in f.readlines():
+                        self.baseFileList[line.strip()] = True
 
     def IsNewOrChanged(self, filename):
         item = os.path.basename(filename) + str(os.path.getsize(filename))

nsiqcppstyle_lexer.py

@@ -169,14 +169,10 @@ def clone(self, object=None):
         return c
 
     # ------------------------------------------------------------
-    # writetab() - Write lexer information to a table file
+    # actual_writetab() - Perform actual file write of lexer
+    #                     information to a table file
     # ------------------------------------------------------------
-    def writetab(self, tabfile, outputdir=""):
-        if isinstance(tabfile, types.ModuleType):
-            return
-        basetabfilename = tabfile.split(".")[-1]
-        filename = os.path.join(outputdir, basetabfilename) + ".py"
-        tf = open(filename, "w")
+    def actual_writetab(self, tabfile, tf):
         tf.write("# %s.py. This file automatically created by PLY (version %s). Don't edit!\n" % (
             tabfile, __version__))
         tf.write("_tabversion   = %s\n" % repr(__version__))
@@ -211,7 +207,17 @@ def writetab(self, tabfile, outputdir=""):
             else:
                 taberr[key] = None
         tf.write("_lexstateerrorf = %s\n" % repr(taberr))
-        tf.close()
+
+    # ------------------------------------------------------------
+    # writetab() - Write lexer information to a table file
+    # ------------------------------------------------------------
+    def writetab(self, tabfile, outputdir=""):
+        if isinstance(tabfile, types.ModuleType):
+            return
+        basetabfilename = tabfile.split(".")[-1]
+        filename = os.path.join(outputdir, basetabfilename) + ".py"
+        with open(filename, "w") as tf:
+            self.actual_writetab(tabfile, tf)
 
     # ------------------------------------------------------------
     # readtab() - Read lexer information from a tab file
@@ -885,9 +891,12 @@ def validate_file(self, filename):
             return         # No idea what the file is. Return OK
 
         try:
-            f = open(filename)
-            lines = f.readlines()
-            f.close()
+            with open(filename) as f:
+                lines = f.readlines()
+        except UnicodeDecodeError as ex:
+            console.Out.Ci("[ERROR] UnicodeDecodeError in validate_file: " + str(ex))
+            console.Out.Ci("[ERROR] Exception occurred reading file '%s', convert from UTF16LE to UTF8" % (filename))
+            raise ex
         except IOError:
             return                      # Couldn't find the file.  Don't worry about it
 
@@ -1082,9 +1091,8 @@ def runmain(lexer=None, data=None):
     if not data:
         try:
             filename = sys.argv[1]
-            f = open(filename)
-            data = f.read()
-            f.close()
+            with open(filename) as f:
+                data = f.read()
         except IndexError:
             sys.stdout.write(
                 "Reading from standard input (type EOF to end):\n")