Merge pull request #415 from madsmtm/header-translator-fixes

A few `header-translator` fixes

crates/header-translator/Cargo.toml

@@ -9,6 +9,7 @@ license = "Zlib OR Apache-2.0 OR MIT"
 
 [dependencies]
 clang = { version = "2.0", features = ["runtime", "clang_10_0"] }
+clang-sys = { version = "1.4.0" }
 toml = "0.5.9"
 serde = { version = "1.0.144", features = ["derive"] }
 apple-sdk = { version = "0.4.0", default-features = false }

crates/header-translator/README.md

@@ -11,12 +11,20 @@ cargo run --bin header-translator
 
 Make sure you have the same XCode version installed as the one documented in [`crates/icrate/README.md`](../icrate/README.md).
 
-If you have the SDK stored in a different directory, you can specify that as the first argument:
+If you use a different operating system than macOS, or simply have multiple SDKs installed, you can specify the directory as the first argument:
 
 ```console
 cargo run --bin header-translator -- /Applications/Xcode_new.app/Contents/Developer
 ```
 
+Note that you will likely need to use a newer `libclang`, such as [the one distributed with Swift `5.7.2`](https://github.com/apple/llvm-project/tree/swift-5.7.2-RELEASE). You can use a different version as follows (details might vary between operating systems):
+
+```console
+export LIBCLANG_PATH=/path/to/custom/installation/usr/lib/libclang.dylib
+export CPATH=/path/to/custom/installation/usr/lib/clang/14.0.0/include/
+cargo run --bin header-translator -- /Applications/Xcode_new.app/Contents/Developer
+```
+
 We do not redistribute SDKs, to hopefully avoid a license violation. You should download XCode (which contain the SDKs) yourself from [Apple's website](https://developer.apple.com/download/all/?q=xcode) (requires an Apple ID).
 
 

crates/header-translator/src/context.rs

@@ -10,7 +10,7 @@ use crate::config::Config;
 
 pub struct Context<'a> {
     config: &'a Config,
-    pub macro_invocations: HashMap<Location<'a>, String>,
+    pub macro_invocations: HashMap<Location<'a>, Entity<'a>>,
     framework_dir: PathBuf,
     include_dir: PathBuf,
     system_headers: HashSet<&'static Path>,

crates/header-translator/src/main.rs

@@ -44,6 +44,9 @@ fn main() -> Result<(), BoxError> {
 
     let config = load_config(manifest_dir);
 
+    clang_sys::load()?;
+    info!(clang_version = clang::get_version());
+
     let clang = Clang::new()?;
     let index = Index::new(&clang, true, true);
 
@@ -249,11 +252,10 @@ fn parse_sdk(index: &Index<'_>, sdk: &SdkPath, llvm_target: &str, config: &Confi
                         }
                     }
                     EntityKind::MacroExpansion if preprocessing => {
-                        let name = entity.get_name().expect("macro name");
                         let location = entity.get_location().expect("macro location");
                         context
                             .macro_invocations
-                            .insert(location.get_spelling_location(), name);
+                            .insert(location.get_spelling_location(), entity);
                     }
                     EntityKind::MacroDefinition if preprocessing => {
                         // let name = entity.get_name().expect("macro def name");
@@ -312,9 +314,13 @@ fn get_translation_unit<'i: 'tu, 'tu>(
             "-fobjc-arc-exceptions",
             "-fobjc-abi-version=2", // 3??
             // "-fparse-all-comments",
+            // TODO: "-fretain-comments-from-system-headers"
             "-fapinotes",
             "-isysroot",
             sdk.path.to_str().unwrap(),
+            // See ClangImporter.cpp and Foundation/NSObjCRuntime.h
+            "-D",
+            "__SWIFT_ATTR_SUPPORTS_SENDABLE_DECLS=1",
         ])
         .parse()
         .unwrap();

crates/header-translator/src/rust_type.rs

@@ -7,6 +7,7 @@ use serde::Deserialize;
 
 use crate::context::Context;
 use crate::id::ItemIdentifier;
+use crate::unexposed_attr::UnexposedAttr;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
 enum ParsePosition {
@@ -353,7 +354,9 @@ impl IdType {
                         }
 
                         let mut parser = AttributeParser::new(pointer_name, &pointee_name);
+                        *is_kindof = parser.is_kindof(ParsePosition::Prefix);
                         lifetime.update(parser.lifetime(ParsePosition::Prefix));
+                        lifetime.update(parser.lifetime(ParsePosition::Suffix));
                         parser.set_inner_pointer();
 
                         Self::Class {
@@ -1787,21 +1790,19 @@ fn parse_unexposed_tokens(s: &str) -> String {
     let mut iter = tokens.into_iter().peekable();
     if let Some(TokenTree::Ident(ident)) = iter.peek() {
         let ident = ident.to_string();
-        match &*ident {
-            "NS_RETURNS_INNER_POINTER" | "NS_REFINED_FOR_SWIFT" | "NS_SWIFT_UI_ACTOR" => {
-                iter.next();
+        if let Ok(attr) = UnexposedAttr::from_name(&ident, || {
+            iter.next();
+            if let Some(TokenTree::Group(group)) = iter.peek() {
+                Some(group)
+            } else {
+                error!(?ident, "expected group in macro");
+                None
             }
-            "API_AVAILABLE"
-            | "API_UNAVAILABLE"
-            | "NS_SWIFT_NAME"
-            | "API_DEPRECATED"
-            | "API_DEPRECATED_WITH_REPLACEMENT" => {
-                iter.next();
-                if let Some(TokenTree::Group(_)) = iter.peek() {
-                    iter.next();
-                }
+        }) {
+            if let Some(attr) = attr {
+                error!(?attr, "unknown attribute");
             }
-            _ => {}
+            iter.next();
         }
     }
     TokenStream::from_iter(iter).to_string()

crates/header-translator/src/stmt.rs

@@ -118,6 +118,9 @@ fn parse_objc_decl(
         EntityKind::ObjCSuperClassRef | EntityKind::TypeRef if superclass => {
             // Parsed in parse_superclass
         }
+        EntityKind::ObjCSubclassingRestricted if superclass => {
+            // TODO: https://clang.llvm.org/docs/AttributeReference.html#objc-subclassing-restricted
+        }
         EntityKind::ObjCRootClass => {
             debug!("parsing root class");
         }
@@ -910,6 +913,11 @@ impl Stmt {
                 );
                 vec![]
             }
+            EntityKind::UnexposedDecl => {
+                // `@compatibility_alias`, can be ignored (since we don't
+                // need to support older SDK versions).
+                vec![]
+            }
             _ => {
                 error!("unknown");
                 vec![]

crates/header-translator/src/unexposed_attr.rs

@@ -1,4 +1,5 @@
-use clang::source::SourceLocation;
+use clang::source::{SourceLocation, SourceRange};
+use clang::token::{Token, TokenKind};
 use clang::{Entity, EntityKind};
 
 use crate::context::Context;
@@ -22,23 +23,43 @@ pub enum UnexposedAttr {
 }
 
 impl UnexposedAttr {
-    fn from_name(s: &str) -> Option<Self> {
-        match s {
-            "NS_ENUM" | "CF_ENUM" => Some(Self::Enum),
-            "NS_OPTIONS" | "CF_OPTIONS" => Some(Self::Options),
-            "NS_CLOSED_ENUM" | "CF_CLOSED_ENUM" => Some(Self::ClosedEnum),
-            "NS_ERROR_ENUM" => Some(Self::ErrorEnum),
-            "NS_TYPED_ENUM" | "NS_STRING_ENUM" | "CF_TYPED_ENUM" => Some(Self::TypedEnum),
-            "NS_TYPED_EXTENSIBLE_ENUM" | "CF_TYPED_EXTENSIBLE_ENUM" => {
-                Some(Self::TypedExtensibleEnum)
+    pub(crate) fn from_name<T>(
+        s: &str,
+        get_arguments: impl FnOnce() -> T,
+    ) -> Result<Option<Self>, ()> {
+        Ok(match s {
+            "NS_ENUM" | "CF_ENUM" => {
+                let _ = get_arguments();
+                Some(Self::Enum)
+            }
+            "NS_OPTIONS" | "CF_OPTIONS" => {
+                let _ = get_arguments();
+                Some(Self::Options)
+            }
+            "NS_CLOSED_ENUM" | "CF_CLOSED_ENUM" => {
+                let _ = get_arguments();
+                Some(Self::ClosedEnum)
             }
+            "NS_ERROR_ENUM" => {
+                let _ = get_arguments();
+                Some(Self::ErrorEnum)
+            }
+            "NS_TYPED_ENUM" | "NS_STRING_ENUM" | "CF_TYPED_ENUM" => Some(Self::TypedEnum),
+            "_NS_TYPED_EXTENSIBLE_ENUM"
+            | "NS_TYPED_EXTENSIBLE_ENUM"
+            | "CF_TYPED_EXTENSIBLE_ENUM"
+            | "NS_EXTENSIBLE_STRING_ENUM" => Some(Self::TypedExtensibleEnum),
             "NS_SWIFT_BRIDGED_TYPEDEF" | "CF_SWIFT_BRIDGED_TYPEDEF" => Some(Self::BridgedTypedef),
             "CF_BRIDGED_TYPE" => Some(Self::Bridged),
             "CF_BRIDGED_MUTABLE_TYPE" => Some(Self::BridgedMutable),
             "NS_RETURNS_RETAINED" | "CF_RETURNS_RETAINED" => Some(Self::ReturnsRetained),
             "NS_RETURNS_NOT_RETAINED" | "CF_RETURNS_NOT_RETAINED" => Some(Self::ReturnsNotRetained),
+            "NS_RETURNS_INNER_POINTER" => None,
             // TODO
-            "NS_FORMAT_FUNCTION" | "NS_FORMAT_ARGUMENT" => None,
+            "NS_FORMAT_FUNCTION" | "NS_FORMAT_ARGUMENT" => {
+                let _ = get_arguments();
+                None
+            }
             // TODO
             "NS_NOESCAPE" => None,
             // TODO: We could potentially automatically elide this argument
@@ -50,58 +71,113 @@ impl UnexposedAttr {
             "NS_INLINE" => None,
             // We don't synthethize properties, so irrelevant for us.
             "NS_REQUIRES_PROPERTY_DEFINITIONS" => None,
+            // Weak specifiers - would be interesting if Rust supported weak statics
+            "GK_EXTERN_WEAK" => None,
             // Availability attributes - their data is already exposed.
             "API_AVAILABLE"
             | "API_UNAVAILABLE"
             | "API_DEPRECATED"
             | "API_DEPRECATED_WITH_REPLACEMENT"
+            | "API_UNAVAILABLE_BEGIN"
             | "NS_CLASS_AVAILABLE_MAC"
             | "NS_AVAILABLE"
-            | "NS_UNAVAILABLE"
-            | "NS_AUTOMATED_REFCOUNT_UNAVAILABLE"
-            | "NS_AUTOMATED_REFCOUNT_WEAK_UNAVAILABLE"
             | "NS_OPENGL_DEPRECATED"
             | "NS_OPENGL_CLASS_DEPRECATED"
             | "NS_OPENGL_ENUM_DEPRECATED"
             | "OBJC_AVAILABLE"
             | "OBJC_DEPRECATED"
+            | "NS_DEPRECATED"
+            | "NS_DEPRECATED_IOS"
+            | "NS_DEPRECATED_MAC"
+            | "CG_AVAILABLE_STARTING"
+            | "CG_AVAILABLE_BUT_DEPRECATED"
+            | "NS_SWIFT_UNAVAILABLE"
+            | "CF_SWIFT_UNAVAILABLE"
+            | "OBJC_SWIFT_UNAVAILABLE"
+            | "FPUI_AVAILABLE"
+            | "WEBKIT_AVAILABLE_MAC"
+            | "WEBKIT_DEPRECATED_MAC"
+            | "WEBKIT_CLASS_DEPRECATED_MAC"
+            | "WEBKIT_ENUM_DEPRECATED_MAC"
+            | "NS_AVAILABLE_MAC"
+            | "NS_CLASS_AVAILABLE"
+            | "NS_CLASS_DEPRECATED"
+            | "NS_CLASS_DEPRECATED_IOS"
+            | "NS_ENUM_AVAILABLE"
+            | "NS_ENUM_DEPRECATED"
+            | "NS_ENUM_DEPRECATED_IOS"
+            | "__OSX_AVAILABLE_STARTING"
+            | "__TVOS_AVAILABLE"
+            | "__WATCHOS_AVAILABLE"
+            | "__IOS_AVAILABLE" => {
+                let _ = get_arguments();
+                None
+            }
+            "NS_UNAVAILABLE"
             | "UNAVAILABLE_ATTRIBUTE"
+            | "NS_AUTOMATED_REFCOUNT_UNAVAILABLE"
+            | "NS_AUTOMATED_REFCOUNT_WEAK_UNAVAILABLE"
             | "APPKIT_API_UNAVAILABLE_BEGIN_MACCATALYST"
-            | "CG_AVAILABLE_STARTING"
-            | "CG_AVAILABLE_BUT_DEPRECATED" => None,
+            | "__OSX_UNAVAILABLE"
+            | "__OSX_AVAILABLE_BUT_DEPRECATED"
+            | "__TVOS_UNAVAILABLE"
+            | "__TVOS_PROHIBITED"
+            | "__WATCHOS_UNAVAILABLE"
+            | "__WATCHOS_PROHIBITED"
+            | "__IOS_UNAVAILABLE"
+            | "UT_AVAILABLE_BEGIN"
+            | "__IOS_PROHIBITED" => None,
+            s if s.starts_with("AVAILABLE_MAC_OS_X_VERSION_") => None,
+            s if s.starts_with("DEPRECATED_IN_MAC_OS_X_VERSION_") => None,
+            s if s.starts_with("FILEPROVIDER_API_AVAILABILITY_") => None,
             // Might be interesting in the future
             "NS_SWIFT_NAME"
             | "CF_SWIFT_NAME"
             | "NS_SWIFT_ASYNC_NAME"
-            | "NS_SWIFT_NOTHROW"
+            | "WK_SWIFT_ASYNC_NAME"
             | "NS_SWIFT_ASYNC_THROWS_ON_FALSE"
             | "NS_SWIFT_UNAVAILABLE_FROM_ASYNC"
-            | "NS_SWIFT_DISABLE_ASYNC"
-            | "NS_SWIFT_UI_ACTOR"
-            | "NS_REFINED_FOR_SWIFT"
-            | "CF_REFINED_FOR_SWIFT"
-            | "NS_SWIFT_UNAVAILABLE"
-            | "CF_SWIFT_UNAVAILABLE"
-            | "OBJC_SWIFT_UNAVAILABLE" => None,
-            name => {
-                error!(name, "unknown unexposed macro");
+            | "NS_HEADER_AUDIT_BEGIN"
+            | "NS_REFINED_FOR_SWIFT_ASYNC"
+            | "NS_SWIFT_ASYNC"
+            | "WK_SWIFT_ASYNC" => {
+                let _ = get_arguments();
                 None
             }
-        }
+            "NS_SWIFT_NOTHROW"
+            | "NS_REFINED_FOR_SWIFT"
+            | "CF_REFINED_FOR_SWIFT"
+            | "NS_SWIFT_DISABLE_ASYNC"
+            | "NS_SWIFT_UI_ACTOR"
+            | "NS_SWIFT_SENDABLE"
+            | "NS_SWIFT_NONSENDABLE"
+            | "NS_SWIFT_NONISOLATED" => None,
+            _ => return Err(()),
+        })
     }
 
     pub fn parse(entity: &Entity<'_>, context: &Context<'_>) -> Option<Self> {
         if let Some(location) = entity.get_location() {
-            if let Some(macro_name) = context
+            if let Some(entity) = context
                 .macro_invocations
                 .get(&location.get_spelling_location())
             {
-                return Self::from_name(macro_name);
+                let name = entity.get_name().expect("macro name");
+                return Self::from_name(&name, || get_argument_tokens(entity)).unwrap_or_else(
+                    |()| {
+                        error!(
+                            name,
+                            fnlike = entity.is_function_like_macro(),
+                            "unknown unexposed attribute"
+                        );
+                        None
+                    },
+                );
             }
 
             Self::parse_location(location)
         } else {
-            error!("unexposed attr location");
+            warn!(?entity, "no unexposed attr location");
             None
         }
     }
@@ -111,7 +187,12 @@ impl UnexposedAttr {
             match parsed.get_kind() {
                 EntityKind::MacroExpansion => {
                     let macro_name = parsed.get_name().expect("macro name");
-                    Self::from_name(&macro_name)
+                    Self::from_name(&macro_name, || get_argument_tokens(&parsed)).unwrap_or_else(
+                        |()| {
+                            error!(macro_name, "unknown unexposed attribute");
+                            None
+                        },
+                    )
                 }
                 // Some macros can't be found using this method,
                 // for example NS_NOESCAPE.
@@ -122,3 +203,26 @@ impl UnexposedAttr {
         }
     }
 }
+
+fn get_argument_tokens<'a>(entity: &Entity<'a>) -> Vec<Token<'a>> {
+    if !entity.is_function_like_macro() {
+        error!(?entity, "tried to get tokens from non-function-like macro");
+        return vec![];
+    }
+    // Remove the macro name from the full macro tokens
+    let name_ranges = entity.get_name_ranges();
+    assert_eq!(name_ranges.len(), 1, "macro name ranges");
+    let name_range = name_ranges.get(0).unwrap();
+    let range = entity.get_range().expect("macro range");
+
+    let mut tokens = SourceRange::new(name_range.get_end(), range.get_end()).tokenize();
+
+    let start = tokens.remove(0);
+    assert_eq!(start.get_kind(), TokenKind::Punctuation);
+    assert_eq!(start.get_spelling(), "(");
+    let end = tokens.pop().expect("tokens to have parentheses");
+    assert_eq!(end.get_kind(), TokenKind::Punctuation);
+    assert_eq!(end.get_spelling(), ")");
+
+    tokens
+}