Merge pull request #68 from networktocode/mzb-netbox-keeper-disaggregation

Onboarding process refactor

Author: dgarros

development/Dockerfile

@@ -13,8 +13,11 @@ RUN pip install --upgrade pip\
 # -------------------------------------------------------------------------------------
 # Install NetBox
 # -------------------------------------------------------------------------------------
+# Remove redis==3.4.1 from the requirements.txt file as a workaround to #4910
+# https://github.com/netbox-community/netbox/issues/4910, required for version 2.8.8 and earlier
 RUN git clone --single-branch --branch ${netbox_ver} https://github.com/netbox-community/netbox.git /opt/netbox/ && \
     cd /opt/netbox/ && \
+    sed -i '/^redis\=\=/d' /opt/netbox/requirements.txt && \
     pip install -r /opt/netbox/requirements.txt
 
 # Make the django-debug-toolbar always visible when DEBUG is enabled,

docs/examples/example_ios_set_device_role.py

@@ -0,0 +1,96 @@
+"""Example of custom onboarding class.
+
+(c) 2020 Network To Code
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+  http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from netbox_onboarding.netbox_keeper import NetboxKeeper
+from netbox_onboarding.onboarding.onboarding import Onboarding
+
+
+class MyOnboardingClass(Onboarding):
+    """Custom onboarding class example.
+
+    Main purpose of this class is to access and modify the onboarding_kwargs.
+    By accessing the onboarding kwargs, user gains ability to modify
+    onboarding parameters before the objects are created in NetBox.
+
+    This class adds the get_device_role method that does the static
+     string comparison and returns the device role.
+    """
+
+    def run(self, onboarding_kwargs):
+        """Ensures network device."""
+        # Access hostname from onboarding_kwargs and get device role automatically
+        device_new_role = self.get_device_role(hostname=onboarding_kwargs["netdev_hostname"])
+
+        # Update the device role in onboarding kwargs dictionary
+        onboarding_kwargs["netdev_nb_role_slug"] = device_new_role
+
+        nb_k = NetboxKeeper(**onboarding_kwargs)
+        nb_k.ensure_device()
+
+        self.created_device = nb_k.device
+
+    @staticmethod
+    def get_device_role(hostname):
+        """Returns the device role based on hostname data.
+
+        This is a static analysis of hostname string content only
+        """
+        hostname_lower = hostname.lower()
+        if ("rtr" in hostname_lower) or ("router" in hostname_lower):
+            role = "router"
+        elif ("sw" in hostname_lower) or ("switch" in hostname_lower):
+            role = "switch"
+        elif ("fw" in hostname_lower) or ("firewall" in hostname_lower):
+            role = "firewall"
+        elif "dc" in hostname_lower:
+            role = "datacenter"
+        else:
+            role = "generic"
+
+        return role
+
+
+class OnboardingDriverExtensions:
+    """This is an example of a custom onboarding driver extension.
+
+    This extension sets the onboarding_class to MyOnboardingClass,
+     which is an example class of how to access and modify the device
+     role automatically through the onboarding process.
+    """
+
+    def __init__(self, napalm_device):
+        """Inits the class."""
+        self.napalm_device = napalm_device
+        self.onboarding_class = MyOnboardingClass
+        self.ext_result = None
+
+    def get_onboarding_class(self):
+        """Return onboarding class for IOS driver.
+
+        Currently supported is Standalone Onboarding Process
+
+        Result of this method is used by the OnboardingManager to
+        initiate the instance of the onboarding class.
+        """
+        return self.onboarding_class
+
+    def get_ext_result(self):
+        """This method is used to store any object as a return value.
+
+        Result of this method is passed to the onboarding class as
+        driver_addon_result argument.
+
+        :return: Any()
+        """
+        return self.ext_result

netbox_onboarding/__init__.py

@@ -40,6 +40,7 @@ class OnboardingConfig(PluginConfig):
         "default_device_status": "active",
         "create_management_interface_if_missing": True,
         "platform_map": {},
+        "onboarding_extensions_map": {"ios": "netbox_onboarding.onboarding_extensions.ios",},
     }
     caching_config = {}
 

netbox_onboarding/choices.py

@@ -22,12 +22,14 @@ class OnboardingStatusChoices(ChoiceSet):
     STATUS_PENDING = "pending"
     STATUS_RUNNING = "running"
     STATUS_SUCCEEDED = "succeeded"
+    STATUS_SKIPPED = "skipped"
 
     CHOICES = (
         (STATUS_FAILED, "failed"),
         (STATUS_PENDING, "pending"),
         (STATUS_RUNNING, "running"),
         (STATUS_SUCCEEDED, "succeeded"),
+        (STATUS_SKIPPED, "skipped"),
     )
 
 

netbox_onboarding/constants.py

@@ -1,5 +1,6 @@
 """Constants for netbox_onboarding plugin."""
-NETMIKO_TO_NAPALM = {
+
+NETMIKO_TO_NAPALM_STATIC = {
     "cisco_ios": "ios",
     "cisco_nxos": "nxos_ssh",
     "arista_eos": "eos",

netbox_onboarding/exceptions.py

@@ -0,0 +1,39 @@
+"""Exceptions.
+
+(c) 2020 Network To Code
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+  http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+
+class OnboardException(Exception):
+    """A failure occurred during the onboarding process.
+
+    The exception includes a reason "slug" as defined below as well as a humanized message.
+    """
+
+    REASONS = (
+        "fail-config",  # config provided is not valid
+        "fail-connect",  # device is unreachable at IP:PORT
+        "fail-execute",  # unable to execute device/API command
+        "fail-login",  # bad username/password
+        "fail-dns",  # failed to get IP address from name resolution
+        "fail-general",  # other error
+    )
+
+    def __init__(self, reason, message, **kwargs):
+        """Exception Init."""
+        super(OnboardException, self).__init__(kwargs)
+        self.reason = reason
+        self.message = message
+
+    def __str__(self):
+        """Exception __str__."""
+        return f"{self.__class__.__name__}: {self.reason}: {self.message}"

netbox_onboarding/filters.py

@@ -26,8 +26,6 @@ class OnboardingTaskFilter(NameSlugSearchFilterSet):
 
     q = django_filters.CharFilter(method="search", label="Search",)
 
-    site_id = django_filters.ModelMultipleChoiceFilter(queryset=Site.objects.all(), label="Site (ID)",)
-
     site = django_filters.ModelMultipleChoiceFilter(
         field_name="site__slug", queryset=Site.objects.all(), to_field_name="slug", label="Site (slug)",
     )
@@ -46,7 +44,7 @@ class Meta:  # noqa: D106 "Missing docstring in public nested class"
         model = OnboardingTask
         fields = ["id", "site", "site_id", "platform", "role", "status", "failed_reason"]
 
-    def search(self, queryset, name, value):
+    def search(self, queryset, name, value):  # pylint: disable=unused-argument, no-self-use
         """Perform the filtered search."""
         if not value.strip():
             return queryset
@@ -55,7 +53,7 @@ def search(self, queryset, name, value):
             | Q(ip_address__icontains=value)
             | Q(site__name__icontains=value)
             | Q(platform__name__icontains=value)
-            | Q(device__icontains=value)
+            | Q(created_device__name__icontains=value)
             | Q(status__icontains=value)
             | Q(failed_reason__icontains=value)
             | Q(message__icontains=value)

netbox_onboarding/forms.py

@@ -13,11 +13,11 @@
 """
 
 from django import forms
+from django.db import transaction
 from django_rq import get_queue
 
-from utilities.forms import BootstrapMixin
+from utilities.forms import BootstrapMixin, CSVModelForm
 from dcim.models import Site, Platform, DeviceRole, DeviceType
-from extras.forms import CustomFieldModelCSVForm
 
 from .models import OnboardingTask
 from .choices import OnboardingStatusChoices, OnboardingFailChoices
@@ -33,7 +33,7 @@ class OnboardingTaskForm(BootstrapMixin, forms.ModelForm):
         required=True, label="IP address", help_text="IP Address/DNS Name of the device to onboard"
     )
 
-    site = forms.ModelChoiceField(required=True, queryset=Site.objects.all(), to_field_name="slug")
+    site = forms.ModelChoiceField(required=True, queryset=Site.objects.all())
 
     username = forms.CharField(required=False, help_text="Device username (will not be stored in database)")
     password = forms.CharField(
@@ -106,7 +106,7 @@ class Meta:  # noqa: D106 "Missing docstring in public nested class"
         fields = ["q", "site", "platform", "status", "failed_reason"]
 
 
-class OnboardingTaskFeedCSVForm(CustomFieldModelCSVForm):
+class OnboardingTaskFeedCSVForm(CSVModelForm):
     """Form for entering CSV to bulk-import OnboardingTask entries."""
 
     site = forms.ModelChoiceField(
@@ -149,12 +149,21 @@ class OnboardingTaskFeedCSVForm(CustomFieldModelCSVForm):
 
     class Meta:  # noqa: D106 "Missing docstring in public nested class"
         model = OnboardingTask
-        fields = OnboardingTask.csv_headers
+        fields = [
+            "site",
+            "ip_address",
+            "port",
+            "timeout",
+            "platform",
+            "role",
+        ]
 
     def save(self, commit=True, **kwargs):
         """Save the model, and add it and the associated credentials to the onboarding worker queue."""
         model = super().save(commit=commit, **kwargs)
         if commit:
             credentials = Credentials(self.data.get("username"), self.data.get("password"), self.data.get("secret"))
-            get_queue("default").enqueue("netbox_onboarding.worker.onboard_device", model.pk, credentials)
+            transaction.on_commit(
+                lambda: get_queue("default").enqueue("netbox_onboarding.worker.onboard_device", model.pk, credentials)
+            )
         return model

netbox_onboarding/migrations/0002_onboardingdevice.py

@@ -0,0 +1,22 @@
+# Generated by Django 2.2.10 on 2020-08-21 11:05
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("netbox_onboarding", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="OnboardingDevice",
+            fields=[
+                ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False)),
+                ("enabled", models.BooleanField(default=True)),
+                ("device", models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to="dcim.Device")),
+            ],
+        ),
+    ]

netbox_onboarding/models.py

@@ -11,8 +11,13 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+from django.db.models.signals import post_save
+from django.dispatch import receiver
 from django.db import models
+from django.urls import reverse
+from dcim.models import Device
 from .choices import OnboardingStatusChoices, OnboardingFailChoices
+from .release import NETBOX_RELEASE_CURRENT, NETBOX_RELEASE_29
 
 
 class OnboardingTask(models.Model):
@@ -47,18 +52,73 @@ class OnboardingTask(models.Model):
 
     created_on = models.DateTimeField(auto_now_add=True)
 
-    csv_headers = [
-        "site",
-        "ip_address",
-        "port",
-        "timeout",
-        "platform",
-        "role",
-    ]
-
     class Meta:  # noqa: D106 "missing docstring in public nested class"
         ordering = ["created_on"]
 
     def __str__(self):
         """String representation of an OnboardingTask."""
         return f"{self.site} : {self.ip_address}"
+
+    def get_absolute_url(self):
+        """Provide absolute URL to an OnboardingTask."""
+        return reverse("plugins:netbox_onboarding:onboardingtask", kwargs={"pk": self.pk})
+
+    if NETBOX_RELEASE_CURRENT >= NETBOX_RELEASE_29:
+        from utilities.querysets import RestrictedQuerySet  # pylint: disable=no-name-in-module, import-outside-toplevel
+
+        objects = RestrictedQuerySet.as_manager()
+
+
+class OnboardingDevice(models.Model):
+    """The status of each Onboarded Device is tracked in the OnboardingDevice table."""
+
+    device = models.OneToOneField(to="dcim.Device", on_delete=models.CASCADE)
+    enabled = models.BooleanField(default=True, help_text="Whether (re)onboarding of this device is permitted")
+
+    @property
+    def last_check_attempt_date(self):
+        """Date of last onboarding attempt for a device."""
+        try:
+            return OnboardingTask.objects.filter(created_device=self.device).latest("created_on").created_on
+        except ValueError:
+            return "unknown"
+
+    @property
+    def last_check_successful_date(self):
+        """Date of last successful onboarding for a device."""
+        try:
+            return (
+                OnboardingTask.objects.filter(
+                    created_device=self.device, status=OnboardingStatusChoices.STATUS_SUCCEEDED
+                )
+                .latest("created_on")
+                .created_on
+            )
+        except ValueError:
+            return "unknown"
+
+    @property
+    def status(self):
+        """Last onboarding status."""
+        try:
+            return OnboardingTask.objects.filter(created_device=self.device).latest("created_on").status
+        except ValueError:
+            return "unknown"
+
+    @property
+    def last_ot(self):
+        """Last onboarding task."""
+        try:
+            return OnboardingTask.objects.filter(created_device=self.device).latest("created_on")
+        except ValueError:
+            return None
+
+
+@receiver(post_save, sender=Device)
+def init_onboarding_for_new_device(sender, instance, created, **kwargs):  # pylint: disable=unused-argument
+    """Register to create a OnboardingDevice object for each new Device Object using Django Signal.
+
+    https://docs.djangoproject.com/en/3.0/ref/signals/#post-save
+    """
+    if created:
+        OnboardingDevice.objects.create(device=instance)