Merge pull request #81 from robotpy/extern-c

Fix extern C and add option to remove preprocessing

Author: virtuald

CppHeaderParser/CppHeaderParser.py

@@ -139,6 +139,9 @@ def trace_print(*args):
 #: Symbols to ignore, usually special macros
 ignoreSymbols = ["Q_OBJECT"]
 
+_BRACE_REASON_OTHER = 0
+_BRACE_REASON_NS = 1
+_BRACE_REASON_EXTERN = 2
 
 # Track what was added in what order and at what depth
 parseHistory = []
@@ -1477,7 +1480,6 @@ class Resolver(object):
     C_KEYWORDS = "extern virtual static explicit inline friend constexpr".split()
     C_KEYWORDS = set(C_KEYWORDS)
 
-    SubTypedefs = {}  # TODO deprecate?
     NAMESPACES = []
     CLASSES = {}
 
@@ -1506,6 +1508,11 @@ def cur_namespace(self, add_double_colon=False):
             i += 1
         return rtn
 
+    def cur_linkage(self):
+        if len(self.linkage_stack):
+            return self.linkage_stack[-1]
+        return ""
+
     def guess_ctypes_type(self, string):
         pointers = string.count("*")
         string = string.replace("*", "")
@@ -1870,21 +1877,6 @@ def finalize_vars(self):
                             var["ctypes_type"] = "ctypes.c_void_p"
                             var["unresolved"] = True
 
-                        elif tag in self.SubTypedefs:  # TODO remove SubTypedefs
-                            if (
-                                "property_of_class" in var
-                                or "property_of_struct" in var
-                            ):
-                                trace_print(
-                                    "class:", self.SubTypedefs[tag], "tag:", tag
-                                )
-                                var["typedef"] = self.SubTypedefs[tag]  # class name
-                                var["ctypes_type"] = "ctypes.c_void_p"
-                            else:
-                                trace_print("WARN-this should almost never happen!")
-                                trace_print(var)
-                                var["unresolved"] = True
-
                         elif tag in self._template_typenames:
                             var["typename"] = tag
                             var["ctypes_type"] = "ctypes.c_void_p"
@@ -2061,10 +2053,6 @@ def finalize(self):
                         trace_print("meth returns class:", meth["returns"])
                         meth["returns_class"] = True
 
-                    elif meth["returns"] in self.SubTypedefs:
-                        meth["returns_class"] = True
-                        meth["returns_nested"] = self.SubTypedefs[meth["returns"]]
-
                     elif meth["returns"] in cls._public_enums:
                         enum = cls._public_enums[meth["returns"]]
                         meth["returns_enum"] = enum.get("type")
@@ -2372,6 +2360,7 @@ def _evaluate_method_stack(self):
                     self._get_location(self.nameStack),
                 )
                 newMethod["parent"] = None
+                newMethod["linkage"] = self.cur_linkage()
                 self.functions.append(newMethod)
 
             # Reset template once it has been used
@@ -2472,7 +2461,6 @@ def _evaluate_property_stack(self, clearStack=True, addToVar=None):
                 klass["typedefs"][self.curAccessSpecifier].append(name)
                 if self.curAccessSpecifier == "public":
                     klass._public_typedefs[name] = typedef["type"]
-                Resolver.SubTypedefs[name] = self.curClass
             else:
                 assert 0
         elif self.curClass:
@@ -2524,6 +2512,7 @@ def _evaluate_property_stack(self, clearStack=True, addToVar=None):
                 self._get_location(self.nameStack),
             )
             newVar["namespace"] = self.current_namespace()
+            newVar["linkage"] = self.cur_linkage()
             if self.curClass:
                 klass = self.classes[self.curClass]
                 klass["properties"][self.curAccessSpecifier].append(newVar)
@@ -2542,6 +2531,7 @@ def _evaluate_property_stack(self, clearStack=True, addToVar=None):
                 self._get_location(self.nameStack),
             )
             newVar["namespace"] = self.cur_namespace(False)
+            newVar["linkage"] = self.cur_linkage()
             if addToVar:
                 newVar.update(addToVar)
             self.variables.append(newVar)
@@ -2600,6 +2590,7 @@ def _evaluate_class_stack(self):
             )
             self.curTemplate = None
         newClass["declaration_method"] = self.nameStack[0]
+        newClass["linkage"] = self.cur_linkage()
         self.classes_order.append(newClass)  # good idea to save ordering
         self.stack = []  # fixes if class declared with ';' in closing brace
         self.stmtTokens = []
@@ -2702,7 +2693,14 @@ def show(self):
         for className in list(self.classes.keys()):
             self.classes[className].show()
 
-    def __init__(self, headerFileName, argType="file", encoding=None, **kwargs):
+    def __init__(
+        self,
+        headerFileName,
+        argType="file",
+        encoding=None,
+        preprocessed=False,
+        **kwargs
+    ):
         """Create the parsed C++ header file parse tree
 
         headerFileName - Name of the file to parse OR actual file contents (depends on argType)
@@ -2775,6 +2773,7 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs):
         self.curAccessSpecifier = "private"  # private is default
         self.curTemplate = None
         self.accessSpecifierStack = []
+        self.linkage_stack = []
         debug_print(
             "curAccessSpecifier changed/defaulted to %s", self.curAccessSpecifier
         )
@@ -2802,31 +2801,24 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs):
                 "[ ]+", " ", supportedAccessSpecifier[i]
             ).strip()
 
-        # Change multi line #defines and expressions to single lines maintaining line nubmers
-        # Based from http://stackoverflow.com/questions/2424458/regular-expression-to-match-cs-multiline-preprocessor-statements
-        matches = re.findall(r"(?m)^(?:.*\\\r?\n)+.*$", headerFileStr)
-        is_define = re.compile(r"[ \t\v]*#[Dd][Ee][Ff][Ii][Nn][Ee]")
-        for m in matches:
-            # Keep the newlines so that linecount doesnt break
-            num_newlines = len([a for a in m if a == "\n"])
-            if is_define.match(m):
-                new_m = m.replace("\n", "<CppHeaderParser_newline_temp_replacement>\\n")
-            else:
-                # Just expression taking up multiple lines, make it take 1 line for easier parsing
-                new_m = m.replace("\\\n", " ")
-            if num_newlines > 0:
-                new_m += "\n" * (num_newlines)
-            headerFileStr = headerFileStr.replace(m, new_m)
-
-        # Filter out Extern "C" statements.  These are order dependent
-        matches = re.findall(
-            re.compile(r'extern[\t ]+"[Cc]"[\t \n\r]*{', re.DOTALL), headerFileStr
-        )
-        for m in matches:
-            # Keep the newlines so that linecount doesnt break
-            num_newlines = len([a for a in m if a == "\n"])
-            headerFileStr = headerFileStr.replace(m, "\n" * num_newlines)
-        headerFileStr = re.sub(r'extern[ ]+"[Cc]"[ ]*', "", headerFileStr)
+        if not preprocessed:
+            # Change multi line #defines and expressions to single lines maintaining line nubmers
+            # Based from http://stackoverflow.com/questions/2424458/regular-expression-to-match-cs-multiline-preprocessor-statements
+            matches = re.findall(r"(?m)^(?:.*\\\r?\n)+.*$", headerFileStr)
+            is_define = re.compile(r"[ \t\v]*#[Dd][Ee][Ff][Ii][Nn][Ee]")
+            for m in matches:
+                # Keep the newlines so that linecount doesnt break
+                num_newlines = len([a for a in m if a == "\n"])
+                if is_define.match(m):
+                    new_m = m.replace(
+                        "\n", "<CppHeaderParser_newline_temp_replacement>\\n"
+                    )
+                else:
+                    # Just expression taking up multiple lines, make it take 1 line for easier parsing
+                    new_m = m.replace("\\\n", " ")
+                if num_newlines > 0:
+                    new_m += "\n" * (num_newlines)
+                headerFileStr = headerFileStr.replace(m, new_m)
 
         # Filter out any ignore symbols that end with "()" to account for #define magic functions
         for ignore in ignoreSymbols:
@@ -2866,6 +2858,8 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs):
                     )
 
         self.braceDepth = 0
+        self.braceReason = []
+        self.lastBraceReason = _BRACE_REASON_OTHER
 
         lex = Lexer(self.headerFileName)
         lex.input(headerFileStr)
@@ -2938,23 +2932,20 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs):
                     continue
 
                 if parenDepth == 0 and tok.type == "{":
+                    self.lastBraceReason = _BRACE_REASON_OTHER
                     if len(self.nameStack) >= 2 and is_namespace(
                         self.nameStack
                     ):  # namespace {} with no name used in boost, this sets default?
-                        if (
-                            self.nameStack[1]
-                            == "__IGNORED_NAMESPACE__CppHeaderParser__"
-                        ):  # Used in filtering extern "C"
-                            self.nameStack[1] = ""
                         self.nameSpaces.append("".join(self.nameStack[1:]))
                         ns = self.cur_namespace()
                         self.stack = []
                         self.stmtTokens = []
                         if ns not in self.namespaces:
                             self.namespaces.append(ns)
+                        self.lastBraceReason = _BRACE_REASON_NS
                     # Detect special condition of macro magic before class declaration so we
                     # can filter it out
-                    if "class" in self.nameStack and self.nameStack[0] != "class":
+                    elif "class" in self.nameStack and self.nameStack[0] != "class":
                         classLocationNS = self.nameStack.index("class")
                         classLocationS = self.stack.index("class")
                         if (
@@ -2987,14 +2978,20 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs):
                         self.stmtTokens = []
                     if not self.braceHandled:
                         self.braceDepth += 1
+                        self.braceReason.append(self.lastBraceReason)
 
                 elif parenDepth == 0 and tok.type == "}":
                     if self.braceDepth == 0:
                         continue
-                    if self.braceDepth == len(self.nameSpaces):
-                        tmp = self.nameSpaces.pop()
+                    reason = self.braceReason.pop()
+                    if reason == _BRACE_REASON_NS:
+                        self.nameSpaces.pop()
                         self.stack = []  # clear stack when namespace ends?
                         self.stmtTokens = []
+                    elif reason == _BRACE_REASON_EXTERN:
+                        self.linkage_stack.pop()
+                        self.stack = []  # clear stack when linkage ends?
+                        self.stmtTokens = []
                     else:
                         self._evaluate_stack()
                     self.braceDepth -= 1
@@ -3358,8 +3355,10 @@ def _evaluate_stack(self, token=None):
             pass
         elif len(self.nameStack) == 2 and self.nameStack[0] == "extern":
             debug_print("trace extern")
+            self.linkage_stack.append(self.nameStack[1].strip('"'))
             self.stack = []
             self.stmtTokens = []
+            self.lastBraceReason = _BRACE_REASON_EXTERN
         elif (
             len(self.nameStack) == 2 and self.nameStack[0] == "friend"
         ):  # friend class declaration
@@ -3667,12 +3666,14 @@ def _parse_enum(self):
     def _install_enum(self, newEnum, instancesData):
         if len(self.curClass):
             newEnum["namespace"] = self.cur_namespace(False)
+            newEnum["linkage"] = self.cur_linkage()
             klass = self.classes[self.curClass]
             klass["enums"][self.curAccessSpecifier].append(newEnum)
             if self.curAccessSpecifier == "public" and "name" in newEnum:
                 klass._public_enums[newEnum["name"]] = newEnum
         else:
             newEnum["namespace"] = self.cur_namespace(True)
+            newEnum["linkage"] = self.cur_linkage()
             self.enums.append(newEnum)
             if "name" in newEnum and newEnum["name"]:
                 self.global_enums[newEnum["name"]] = newEnum

test/test_CppHeaderParser.py

@@ -4183,5 +4183,51 @@ def test_fn3(self):
         self.assertEqual(fn["pure_virtual"], True)
 
 
+class ExternCQuirk(unittest.TestCase):
+    # bug where extern "C" reset the namespace
+    def setUp(self):
+        self.cppHeader = CppHeaderParser.CppHeader(
+            """
+namespace cs {
+extern "C" {
+	struct InCSAndExtern {};
+	void FnInCSAndExtern(InCSAndExtern *n);
+}
+
+class InCS {};
+
+}
+
+void FnNotInCSOrExtern();
+
+""",
+            "string",
+        )
+
+    def test_fn(self):
+
+        # NotCS should be in namespace cs, extern C
+        c = self.cppHeader.classes["InCSAndExtern"]
+        self.assertEqual(c["namespace"], "cs")
+        self.assertEqual(c["linkage"], "C")
+
+        # FnNotCS should be in namespace cs, extern C
+        fn = self.cppHeader.functions[0]
+        self.assertEqual(fn["name"], "FnInCSAndExtern")
+        self.assertEqual(fn["namespace"], "cs::")
+        self.assertEqual(fn["linkage"], "C")
+
+        # InCS should be in namespace cs
+        c = self.cppHeader.classes["InCS"]
+        self.assertEqual(c["namespace"], "cs")
+        self.assertEqual(c["linkage"], "")
+
+        # FnNotCS should not in namespace cs nor extern C
+        fn = self.cppHeader.functions[1]
+        self.assertEqual(fn["name"], "FnNotInCSOrExtern")
+        self.assertEqual(fn["namespace"], "")
+        self.assertEqual(fn["linkage"], "")
+
+
 if __name__ == "__main__":
     unittest.main()