opal/memory: add query function to opal/memory structure

This commit makes it possible to set relative priorities for
components. Before the addition of the patched component there was
only one component that would run on any system but that is no longer
the case. When determining which component to open each component's
query function is called and the one that returns the highest priority
is opened. The default priority of the patcher component is set
slightly higher than the old ptmalloc2/ummunotify component.

Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>

Author: hjelmn

opal/mca/memory/base/memory_base_open.c

@@ -13,6 +13,8 @@
  * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -43,20 +45,22 @@ static int empty_process(void)
     return OPAL_SUCCESS;
 }
 
+static int empty_query (int *priority)
+{
+    *priority = 0;
+    return OPAL_SUCCESS;
+}
 
 /*
  * Local variables
  */
 static opal_memory_base_component_2_0_0_t empty_component = {
-    /* Don't care about the version info */
-    { 0, },
-    /* Don't care about the data */
-    { 0, },
     /* Empty / safe functions to call if no memory componet is selected */
-    empty_process,
-    opal_memory_base_component_register_empty,
-    opal_memory_base_component_deregister_empty,
-    opal_memory_base_component_set_alignment_empty,
+    .memoryc_query = empty_query,
+    .memoryc_process = empty_process,
+    .memoryc_register = opal_memory_base_component_register_empty,
+    .memoryc_deregister = opal_memory_base_component_deregister_empty,
+    .memoryc_set_alignment = opal_memory_base_component_set_alignment_empty,
 };
 
 
@@ -73,23 +77,36 @@ opal_memory_base_component_2_0_0_t *opal_memory = &empty_component;
  */
 static int opal_memory_base_open(mca_base_open_flag_t flags)
 {
+    mca_base_component_list_item_t *item, *next;
+    opal_memory_base_component_2_0_0_t *tmp;
+    int priority, highest_priority = 0;
     int ret;
 
-    /* Open up all available components */
+    /* can only be zero or one */
+    OPAL_LIST_FOREACH(item, &opal_memory_base_framework.framework_components, mca_base_component_list_item_t) {
+        tmp = (opal_memory_base_component_2_0_0_t *) item->cli_component;
+        ret = tmp->memoryc_query (&priority);
+        if (OPAL_SUCCESS != ret || priority < highest_priority) {
+            continue;
+        }
+
+        highest_priority = priority;
+        opal_memory = tmp;
+    }
+
+    OPAL_LIST_FOREACH_SAFE(item, next, &opal_memory_base_framework.framework_components, mca_base_component_list_item_t) {
+        if ((void *) opal_memory != (void *) item->cli_component) {
+            mca_base_component_unload (item->cli_component, opal_memory_base_framework.framework_output);
+            opal_list_remove_item (&opal_memory_base_framework.framework_components, &item->super);
+        }
+    }
+
+    /* open remaining component */
     ret = mca_base_framework_components_open (&opal_memory_base_framework, flags);
     if (ret != OPAL_SUCCESS) {
         return ret;
     }
 
-    /* can only be zero or one */
-    if (opal_list_get_size(&opal_memory_base_framework.framework_components) == 1) {
-        mca_base_component_list_item_t *item;
-        item = (mca_base_component_list_item_t*)
-            opal_list_get_first(&opal_memory_base_framework.framework_components);
-        opal_memory = (opal_memory_base_component_2_0_0_t*)
-            item->cli_component;
-    }
-
     /* All done */
     return OPAL_SUCCESS;
 }

opal/mca/memory/linux/hooks.c

@@ -744,6 +744,10 @@ void opal_memory_linux_malloc_init_hook(void)
     check_result_t r1, lp, lpp;
     bool want_rcache = false, found_driver = false;
 
+    if (!opal_memory_linux_opened) {
+        return;
+    }
+
     /* First, check for a FAKEROOT environment.  If we're in a
        fakeroot, then access() (and likely others) have been replaced
        and are not safe to call here in this pre-main environment.  So

opal/mca/memory/linux/memory_linux.h

@@ -83,6 +83,8 @@ OPAL_DECLSPEC void opal_memory_linux_malloc_init_hook(void);
 OPAL_DECLSPEC void opal_memory_linux_malloc_set_alignment(int use_memalign, size_t memalign_threshold);
 #endif /* MEMORY_LINUX_MALLOC_ALIGN_ENABLED */
 
+extern bool opal_memory_linux_opened;
+
 END_C_DECLS
 
 #endif

opal/mca/memory/linux/memory_linux_component.c

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -60,6 +60,7 @@
 static int linux_open(void);
 static int linux_close(void);
 static int linux_register(void);
+static int linux_query(int *);
 
 #if MEMORY_LINUX_UMMUNOTIFY
 static bool ummunotify_opened = false;
@@ -69,6 +70,9 @@ static bool ptmalloc2_opened = false;
 #endif
 
 bool opal_memory_linux_disable = false;
+static int mca_memory_linux_priority;
+
+bool opal_memory_linux_opened = false;
 
 opal_memory_linux_component_t mca_memory_linux_component = {
     /* First, the opal_memory_base_component_2_0_0_t */
@@ -96,6 +100,7 @@ opal_memory_linux_component_t mca_memory_linux_component = {
         /* Memory framework functions.  These function pointer values
            are replaced by memory_linux_ummunotify.c at run time if we
            end up using ummunotify support. */
+        .memoryc_query = linux_query,
         .memoryc_register = opal_memory_base_component_register_empty,
         .memoryc_deregister = opal_memory_base_component_deregister_empty,
 #if MEMORY_LINUX_MALLOC_ALIGN_ENABLED
@@ -243,11 +248,25 @@ static int linux_register(void)
     if (0 > ret) {
         return ret;
     }
+
+    mca_memory_linux_priority = 50;
+    ret = mca_base_component_var_register (&mca_memory_linux_component.super.memoryc_version,
+                                           "priority", "Priority of the linux memory hook component",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_CONSTANT, &mca_memory_linux_priority);
+    if (0 > ret) {
+        return ret;
+    }
 #endif /* MEMORY_LINUX_MALLOC_ALIGN_ENABLED */
 
     return (0 > ret) ? ret : OPAL_SUCCESS;
 }
 
+static int linux_query (int *priority)
+{
+    *priority = mca_memory_linux_priority;
+    return OPAL_SUCCESS;
+}
 
 static int linux_open(void)
 {
@@ -318,6 +337,7 @@ static int linux_open(void)
         __malloc_hook = _opal_memory_linux_malloc_align_hook;
     }
 #endif /* MEMORY_LINUX_MALLOC_ALIGN_ENABLED */
+    opal_memory_linux_opened = true;
 
     return OPAL_SUCCESS;
 }

opal/mca/memory/malloc_solaris/memory_malloc_solaris_component.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2007-2011 Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2009-2011 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -46,6 +46,7 @@ int  __munmap(caddr_t addr, size_t len);
 #endif
 
 static int opal_memory_malloc_open(void);
+static int opal_memory_malloc_query(int *);
 
 const opal_memory_base_component_2_0_0_t mca_memory_malloc_solaris_component = {
     /* First, the mca_component_t struct containing meta information
@@ -68,6 +69,7 @@ const opal_memory_base_component_2_0_0_t mca_memory_malloc_solaris_component = {
 
     /* This component doesn't need these functions, but need to
        provide safe/empty register/deregister functions to call */
+    .memoryc_query = opal_memory_malloc_query,
     .memoryc_register = opal_memory_base_component_register_empty,
     .memoryc_deregister = opal_memory_base_component_deregister_empty,
     .memoryc_set_alignment = opal_memory_base_component_set_alignment_empty,
@@ -93,6 +95,11 @@ opal_memory_malloc_open(void)
     return OPAL_SUCCESS;
 }
 
+static int opal_memory_malloc_query (int *priority)
+{
+    *priority = 79;
+    return OPAL_SUCCESS;
+}
 
 /*
  * Three ways to call munmap.  Prefered is to call __munmap, which

opal/mca/memory/memory.h

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -78,6 +78,12 @@ BEGIN_C_DECLS
  */
 typedef int (*opal_memory_base_component_process_fn_t)(void);
 
+/**
+ * Prototype for a function that is invoked when the memory base is
+ * trying to select a component. This funtionality is required.
+ */
+typedef int (*opal_memory_base_component_query_fn_t)(int *priority);
+
 /**
  * Prototype for a function that is invoked when Open MPI starts to
  * "care" about a specific memory region.  That is, Open MPI declares
@@ -128,6 +134,8 @@ typedef struct opal_memory_base_component_2_0_0_t {
     /** MCA base data */
     mca_base_component_data_t memoryc_data;
 
+    opal_memory_base_component_query_fn_t memoryc_query;
+
     /** Function to call when something has changed, as indicated by
         opal_memory_changed().  Will be ignored if the component does
         not provide an opal_memory_changed() macro that returns

opal/mca/memory/patcher/memory_patcher_component.c

@@ -47,6 +47,9 @@
 static int patcher_open(void);
 static int patcher_close(void);
 static int patcher_register(void);
+static int patcher_query (int *);
+
+static int mca_memory_patcher_priority;
 
 opal_memory_patcher_component_t mca_memory_patcher_component = {
     .super = {
@@ -69,6 +72,7 @@ opal_memory_patcher_component_t mca_memory_patcher_component = {
         },
 
         /* Memory framework functions. */
+        .memoryc_query = patcher_query,
         .memoryc_register = opal_memory_base_component_register_empty,
         .memoryc_deregister = opal_memory_base_component_deregister_empty,
         .memoryc_set_alignment = opal_memory_base_component_set_alignment_empty,
@@ -226,6 +230,18 @@ static int intercept_brk (void *addr)
 
 static int patcher_register (void)
 {
+    mca_memory_patcher_priority = 80;
+    mca_base_component_var_register (&mca_memory_patcher_component.super.memoryc_version,
+                                     "priority", "Priority of the patcher memory hook component",
+                                     MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                     MCA_BASE_VAR_SCOPE_CONSTANT, &mca_memory_patcher_priority);
+
+    return OPAL_SUCCESS;
+}
+
+static int patcher_query (int *priority)
+{
+    *priority = mca_memory_patcher_priority;
     return OPAL_SUCCESS;
 }
 
@@ -240,6 +256,9 @@ static int patcher_open (void)
 
     was_executed_already = 1;
 
+    /* set memory hooks support level */
+    opal_mem_hooks_set_support (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT);
+
     rc = opal_patch_symbol ("mmap", (uintptr_t) intercept_mmap);
     if (OPAL_SUCCESS != rc) {
         return rc;