Add lookup plugins for DL and DH

plugins/lookup/cdp_service.py

@@ -12,25 +12,45 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import re
+
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
 from ansible.module_utils.common.text.converters import to_text, to_native
 from ansible.utils.display import Display
 
+from cdpy.cdpy import Cdpy
+
 display = Display()
+SEMVER = re.compile("(\d+\.[.\d]*\d+)")
+
+def parse_services(terms:list, name:str, entity:dict, service:str, knox:bool, default:any):
+    lookup = 'knoxService' if knox else 'serviceName'
+    results = []
+    try: 
+        for term in LookupBase._flatten(terms):
+            display.vvv("%s_service lookup connecting to '%s[%s]'" % (service, name, term))
+            services = [s for s in entity['endpoints']['endpoints'] if s[lookup] == term and 'serviceUrl' in s]
+            if services:
+                results.append([to_text(s['serviceUrl']) for s in services])
+            else:
+                results.append(default)
+        return results
+    except KeyError as e:
+        raise AnsibleError("Error parsing result for '%s':'" % name, to_native(e))
+
+def parse_environment(environment:str):
+    env = Cdpy().datalake.describe_all_datalakes(environment)                
+
+    if not env:
+        raise AnsibleError("No Datalake found for Environment '%s'" % environment)
+    elif len(env) > 1:
+        raise AnsibleError("Multiple Datalakes found for Enviroment '%s'" % environment)
+
+    raw_version = env[0]['productVersions'][0]['version']
+
+    match = SEMVER.match(raw_version)
+    if not match:
+        raise AnsibleError("Unable to parse runtime version for Environment '%s': %s" % (environment, raw_version))
 
-class CdpServiceLookupModule(LookupBase):
-    def parse_services(self, terms:list, name:str, entity:dict, service:str):
-        lookup = 'knoxService' if self.get_option('knox_service') else 'serviceName'
-        results = []
-        try: 
-            for term in LookupBase._flatten(terms):
-                display.vvv("%s_service lookup connecting to '%s[%s]'" % (service, name, term))
-                services = [s for s in entity['endpoints']['endpoints'] if s[lookup] == term and 'serviceUrl' in s]
-                if services:
-                    results.append([to_text(s['serviceUrl']) for s in services])
-                else:
-                    results.append(self.get_option('default'))
-            return results
-        except KeyError as e:
-            raise AnsibleError("Error parsing result for '%s':'" % (name, to_native(e)))
+    return env[0]['cloudPlatform'], raw_version, match.group(0)

plugins/lookup/datahub_definition.py

@@ -0,0 +1,125 @@
+# Copyright 2023 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+    lookup: datahub_definition
+    author: Webster Mudge (@wmudge) <wmudge@cloudera.com>
+    short_description: Get a Datahub definition for a CDP Public Cloud Environment
+    description:
+        - Allows you to retrieve the Datahub definition matching the Datalake CDH cloud platform and Runtime for one or more CDP Public Cloud Environments.
+        - If an Environment is not found or is ambigious, the lookup will return an error.
+    options:
+        _terms:
+            description:
+                - A CDP Public Cloud Environment name
+            type: string
+            required: True
+        definition:
+            description:
+                - Name (substring) of the Datahub definition to filter the resulting matches
+            required: False
+            type: string
+        detailed:
+            description:
+                - Whether to return the full entry for the matching Datahub definitions
+            required: False
+            type: boolean
+            default: False
+        type:
+            description:
+                - Category of the Datahub definition to filter
+            choices:
+                - OPERATIONALDATABASE
+                - FLOW_MANAGEMENT
+                - DATAMART
+                - DISCOVERY_DATA_AND_EXPLORATION
+                - DATAENGINEERING
+                - STREAMING
+                - OTHER
+            required: False
+            type: string
+    notes:
+        - Requires C(cdpy).
+        - If you encounter I(worker found in a dead state) and are running OSX, set the environment variable, C(OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES).
+    seealso:
+        - module: cloudera.cloud.datahub_definition_info
+          description: Cloudera CDP Public Cloud Datahub definition module
+'''
+
+EXAMPLES = '''
+- name: Retrieve the Datahub definition for a single CDP Public Cloud Environment
+  ansible.builtin.debug:
+    msg: "{{ query('cloudera.cloud.datahub_definition', 'example-env') }}"
+
+- name: Retrieve the Datahub definition that match the given substring
+  ansible.builtin.debug:
+    msg: "{{ query('cloudera.cloud.datahub_definition', 'example-env', definition='Flow Management Light Duty') }}"
+
+- name: Retrieve the only streaming Datahub definition
+  ansible.builtin.debug:
+    msg: "{{ query('cloudera.cloud.datahub_definition', 'example-env', type='STREAMING') }}"
+
+- name: Retrieve the full details for the Datahub definition
+  ansible.builtin.debug:
+    msg: "{{ query('cloudera.cloud.datahub_definition', 'example-env', detailed=True) }}"
+
+- name: Retrieve the Datahub definition details for multiple CDP Public Cloud Environments
+  ansible.builtin.debug:
+    msg: "{{ lookup('cloudera.cloud.datahub_definition', ['example-env', 'another-env'], wantlist=True) }}"
+'''
+
+RETURN = '''
+  _list:
+    description: List of lists of Datahub definition
+    type: list
+    elements: complex
+'''
+
+from ansible.errors import AnsibleError
+from ansible.plugins.lookup import LookupBase
+from ansible.module_utils.common.text.converters import to_native
+from ansible.utils.display import Display
+
+from cdpy.cdpy import Cdpy
+from cdpy.common import CdpError
+
+from ansible_collections.cloudera.cloud.plugins.lookup.cdp_service import parse_environment
+
+display = Display()
+
+class LookupModule(LookupBase):
+    def run(self, terms, variables=None, **kwargs):
+        self.set_options(var_options=variables, direct=kwargs)
+
+        try:
+            all_definitions = Cdpy().datahub.list_cluster_definitions()
+            results = []
+            for term in terms:
+                cloud_platform, raw_version, semantic_version = parse_environment(term)
+                display.vvv("Filtering definitions for %s[%s][%s]" % (term, cloud_platform, semantic_version))
+
+                for d in all_definitions:
+                    if d['cloudPlatform'] == cloud_platform and d['productVersion'] == 'CDH %s' % semantic_version:
+                        if (self.get_option('type') is not None and self.get_option('type') != d['type']) or \
+                           (self.get_option('definition') is not None and self.get_option('definition') not in d['clusterDefinitionName']):
+                            continue
+                        results.append([d if self.get_option('detailed') else d['clusterDefinitionName']])
+            return results
+        except KeyError as e:
+            raise AnsibleError("Error parsing result: %s" % to_native(e))
+        except CdpError as e:
+            raise AnsibleError("Error connecting to CDP: %s" % to_native(e))

plugins/lookup/datahub_instance.py

@@ -0,0 +1,112 @@
+# Copyright 2023 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+    lookup: datahub_instance
+    author: Webster Mudge (@wmudge) <wmudge@cloudera.com>
+    short_description: Get the instances for a CDP Public Cloud Datahub
+    description:
+        - Allows you to retrieve the instances by one or more instance groups for a CDP Public Cloud Datahub.
+        - If the Datahub is not found or is ambigious, the lookup will return an error.
+        - If the instance group is not found, the lookup will return the C(default) value.
+    options:
+        _terms:
+            description:
+                - Instance group name
+            type: string
+            required: True
+        datahub:
+            description:
+                - Name of the Datahub
+            required: True
+            type: string
+        detailed:
+            description:
+                - Whether to return the full entry for the matching Datahub instance group
+            required: False
+            type: boolean
+            default: False
+        default:
+            description: What return when the instance group is not found on the Datahub
+            type: any
+            default: []
+    notes:
+        - Requires C(cdpy).
+    seealso:
+        - module: cloudera.cloud.datahub_cluster_info
+          description: Cloudera CDP Public Cloud Datahub cluster module
+'''
+
+EXAMPLES = '''
+- name: Retrieve the instances for the NiFi instance group for a CDP Public Cloud Flow Management datahub
+  ansible.builtin.debug:
+    msg: "{{ query('cloudera.cloud.datahub_instance', 'nifi', datahub='example-flow-dh') }}"
+
+- name: Retrieve the full details for the instance
+  ansible.builtin.debug:
+    msg: "{{ query('cloudera.cloud.datahub_instance', 'nifi', datahub='example-flow-dh', detailed=True) }}"
+
+- name: Retrieve the instance details for multiple instance groups
+  ansible.builtin.debug:
+    msg: "{{ lookup('cloudera.cloud.datahub_instance', ['nifi', 'management'], datahub='example-flow-dh', wantlist=True) }}"
+'''
+
+RETURN = '''
+  _list:
+    description: List of lists of instances
+    type: list
+    elements: complex
+'''
+
+from ansible.errors import AnsibleError
+from ansible.plugins.lookup import LookupBase
+from ansible.module_utils.common.text.converters import to_native
+from ansible.utils.display import Display
+
+from cdpy.cdpy import Cdpy
+from cdpy.common import CdpError
+
+from ansible_collections.cloudera.cloud.plugins.lookup.cdp_service import parse_environment
+
+display = Display()
+
+class LookupModule(LookupBase):
+    def run(self, terms, variables=None, **kwargs):
+        self.set_options(var_options=variables, direct=kwargs)
+
+        try:
+            datahub = Cdpy().datahub.describe_cluster(self.get_option('datahub'))
+            if datahub is None:
+                raise AnsibleError("No Datahub found for '%s'" % self.get_option('datahub'))
+
+            all_instance_groups = {ig['name']:ig for ig in datahub['instanceGroups']}
+            results = []
+
+            for term in LookupBase._flatten(terms):
+                display.vvv("Filtering instance groups for %s[%s]" % (self.get_option('datahub'), term))
+                if term in all_instance_groups:
+                    if self.get_option('detailed'):
+                        results.append(all_instance_groups[term]['instances'])
+                    else:
+                        results.append([i['fqdn'] for i in all_instance_groups[term]['instances']])
+                else:
+                    results.append(self.get_option('default'))
+            return results
+        except KeyError as e:
+            raise AnsibleError("Error parsing result: %s" % to_native(e))
+        except CdpError as e:
+            raise AnsibleError("Error connecting to CDP: %s" % to_native(e))

plugins/lookup/datahub_service.py

@@ -85,24 +85,22 @@
 '''
 
 from ansible.errors import AnsibleError
+from ansible.plugins.lookup import LookupBase
 from ansible.module_utils.common.text.converters import to_native
 
 from cdpy.cdpy import Cdpy
 from cdpy.common import CdpError
 
-from ansible_collections.cloudera.cloud.plugins.lookup.cdp_service import CdpServiceLookupModule
+from ansible_collections.cloudera.cloud.plugins.lookup.cdp_service import parse_services
 
 
-class LookupModule(CdpServiceLookupModule):
+class LookupModule(LookupBase):
     def run(self, terms, variables=None, **kwargs):
         self.set_options(var_options=variables, direct=kwargs)
-
         try:       
             datahub = Cdpy().datahub.describe_cluster(self.get_option('datahub'))
-
             if datahub is None:
                 raise AnsibleError("No Datahub found for '%s'" % self.get_option('datahub'))
-
-            return self.parse_services(terms, self.get_option('datahub'), datahub, 'datahub')
+            return parse_services(terms, self.get_option('datahub'), datahub, 'datahub', self.get_option('knox_service'), self.get_option('default'))
         except CdpError as e:
             raise AnsibleError("Error connecting to service '%s': %s" % (self.get_option('datahub'), to_native(e)))