Merge branch 'feature/INFRA-10377-improve-full-sync' into 'main'

Full-sync optimized to check for existing groups and hosts before creating or...

Closes INFRA-10377

See merge request !16

netbox_awx_plugin/models.py

@@ -57,6 +57,32 @@ class AWXInventory(NetBoxModel):
         else:
             return None
 
+    def get_all_hosts(self):
+        """
+        Retrieves all hosts from the AWX inventory.
+        """
+        hosts = {}
+        page = 1
+        while True:
+            params = {"page": page}
+            r = requests.get(
+                url=urljoin(
+                    self.awx.url,
+                    f"/api/v2/inventories/{self.inventory_id}/hosts/",
+                ),
+                params=params,
+                headers=self.awx.get_headers(),
+                verify=False,
+            )
+            r.raise_for_status()
+            data = r.json()
+            for host in data["results"]:
+                hosts[host["name"]] = host
+            if not data.get("next"):
+                break
+            page += 1
+        return hosts
+
     def create_host(self, data):
         logger.info("Creating host with: {}".format(data))
         r = requests.post(
@@ -65,7 +91,14 @@ class AWXInventory(NetBoxModel):
             json=data,
             verify=False,
         )
-        r.raise_for_status()
+        if r.status_code == 201:
+            return r.json()  # Return the created host object
+        else:
+            logger.error(
+                f"Failed to create host {data['name']}, Response: "
+                f"{r.text if r else 'No response'}"
+            )
+            return None
 
     def update_host(self, host_id, data):
         logger.info("Updating host with: {}".format(data))
@@ -79,12 +112,17 @@ class AWXInventory(NetBoxModel):
 
     def delete_host(self, hostname):
         host = self.get_host(hostname)
-        r = requests.delete(
-            url=urljoin(self.awx.url, "/api/v2/hosts/{}/".format(host["id"])),
-            headers=self.awx.get_headers(),
-            verify=False,
-        )
-        r.raise_for_status()
+        if host:
+            r = requests.delete(
+                url=urljoin(
+                    self.awx.url, "/api/v2/hosts/{}/".format(host["id"])
+                ),
+                headers=self.awx.get_headers(),
+                verify=False,
+            )
+            r.raise_for_status()
+        else:
+            logger.warning(f"Host {hostname} not found for deletion.")
 
     def get_group(self, name):
         params = {"name": name}
@@ -97,6 +135,32 @@ class AWXInventory(NetBoxModel):
         r.raise_for_status()
         return r.json()["results"][0] if len(r.json()["results"]) > 0 else None
 
+    def get_all_groups(self):
+        """
+        Retrieves all groups from the AWX inventory.
+        """
+        groups = {}
+        page = 1
+        while True:
+            params = {"page": page}
+            r = requests.get(
+                url=urljoin(
+                    self.awx.url,
+                    f"/api/v2/inventories/{self.inventory_id}/groups/",
+                ),
+                params=params,
+                headers=self.awx.get_headers(),
+                verify=False,
+            )
+            r.raise_for_status()
+            data = r.json()
+            for group in data["results"]:
+                groups[group["name"]] = group
+            if not data.get("next"):
+                break
+            page += 1
+        return groups
+
     def update_group(self, name, data):
         logger.info("Updating group {} with: {}".format(name, data))
         group = self.get_group(name)
@@ -116,18 +180,27 @@ class AWXInventory(NetBoxModel):
             json=data,
             verify=False,
         )
-        r.raise_for_status()
+        if r.status_code == 201:
+            return r.json()  # Return the created group object
+        else:
+            logger.error(
+                f"Failed to create group {data['name']}, Response: "
+                f"{r.text if r else 'No response'}"
+            )
+            return None
 
     def delete_group(self, name):
         logger.info("Deleting group {}".format(name))
         group = self.get_group(name)
-        if not group is None:
+        if group is not None:
             r = requests.delete(
                 url=urljoin(self.awx.url, "/api/v2/groups/{}/".format(group["id"])),
                 headers=self.awx.get_headers(),
                 verify=False,
             )
             r.raise_for_status()
+        else:
+            logger.warning(f"Group {name} not found for deletion.")
 
     def associate_host_group(self, host_id, group_id):
         logger.info("Associating host {} to group {}".format(host_id, group_id))

netbox_awx_plugin/synchronization.py

@@ -4,7 +4,6 @@ from dcim.models import Device, DeviceRole, DeviceType, Site
 from ipam.models import Prefix
 from virtualization.models import VirtualMachine
 from extras.models import Tag
-from core.models import Job
 from .models import AWXInventory
 from .serializers import (
     serializers_dict,
@@ -14,8 +13,6 @@ from .serializers import (
 from django_rq import get_queue
 from rq import get_current_job, Queue
 from requests.exceptions import RequestException, HTTPError, ConnectionError, Timeout
-from urllib3.exceptions import InsecureRequestWarning
-import urllib3
 
 # Set up logging
 logger = logging.getLogger(__name__)
@@ -195,24 +192,155 @@ def delete_group(inventory, sender, instance):
 
 def sync_all(job):
     """
-    Performs a full synchronization of the AWX inventory.
+    Performs a full synchronization of the AWX inventory, including host-group associations.
+    Optimized to check for existing groups and hosts before creating or updating them.
     """
     inventory = job.object
-    logger.info(f"Performing full inventory sync for inventory {inventory.inventory_id}")
+    logger.info(
+        f"Performing full inventory sync for inventory {inventory.inventory_id}"
+    )
     job.start()
-    for site in Site.objects.all():
-        sync_group(inventory, Site, site)
-    for device_role in DeviceRole.objects.all():
-        sync_group(inventory, DeviceRole, device_role)
-    for device_type in DeviceType.objects.all():
-        sync_group(inventory, DeviceType, device_type)
-    for tag in Tag.objects.all():
-        sync_group(inventory, Tag, tag)
-
-    for device in Device.objects.all():
-        if device.primary_ip4 and device.primary_ip4.dns_name:
-            sync_host(inventory, Device, device)
-    for vm in VirtualMachine.objects.all():
-        if vm.primary_ip4 and vm.primary_ip4.dns_name:
-            sync_host(inventory, VirtualMachine, vm)
+
+    # Collect all AWX groups and hosts
+    awx_groups = inventory.get_all_groups()
+    awx_hosts = inventory.get_all_hosts()
+
+    # Synchronize groups
+    group_models = [Site, DeviceRole, DeviceType, Prefix, Tag]
+    netbox_group_names = set()
+    for model in group_models:
+        instances = model.objects.all()
+        for instance in instances:
+            serializer = serializers_dict[model](instance)
+            group_name = serializer.data["name"]
+            netbox_group_names.add(group_name)
+
+            if group_name in awx_groups:
+                # Group exists in AWX, check if it needs updating
+                awx_group = awx_groups[group_name]
+                awx_group_serializer = AWXGroupSerializer(data=awx_group)
+                if awx_group_serializer.is_valid():
+                    awx_group_data = awx_group_serializer.validated_data
+                    if awx_group_data != serializer.data:
+                        inventory.update_group(awx_group["id"], serializer.data)
+                        logger.info(f"Updated group {group_name} in AWX.")
+                else:
+                    logger.error(f"Invalid data for awx_group_serializer: {awx_group_serializer.errors}")
+            else:
+                # Group does not exist in AWX, create it
+                awx_group = inventory.create_group(serializer.data)
+                if awx_group:
+                    logger.info(f"Created group {group_name} in AWX.")
+                    # Update local cache
+                    awx_groups[group_name] = awx_group
+                else:
+                    logger.error(
+                        f"Failed to create group {group_name} in AWX."
+                    )
+
+    awx_group_names = set(awx_groups.keys())
+    # Delete groups in AWX that are not in NetBox
+    groups_to_delete = awx_group_names - netbox_group_names
+    for group_name in groups_to_delete:
+        inventory.delete_group(group_name)
+        logger.info(f"Deleted group {group_name} from AWX as it no longer exists in NetBox.")
+
+    # Synchronize hosts
+    host_models = [Device, VirtualMachine]
+    netbox_host_names = set()
+    for model in host_models:
+        instances = model.objects.select_related(
+            "primary_ip4"
+        ).prefetch_related("interfaces__ip_addresses")
+        for instance in instances:
+            if not (instance.primary_ip4 and instance.primary_ip4.dns_name):
+                continue
+            serializer = serializers_dict[model](instance)
+            host_name = serializer.data["name"]
+            netbox_host_names.add(host_name)
+
+            if host_name in awx_hosts:
+                # Host exists in AWX, check if it needs updating
+                awx_host = awx_hosts[host_name]
+                awx_host_serializer = AWXHostSerializer(data=awx_host)
+                if awx_host_serializer.is_valid():
+                    awx_host_data = awx_host_serializer.validated_data
+                    if awx_host_data != serializer.data:
+                        inventory.update_host(awx_host["id"], serializer.data)
+                        logger.info(f"Updated host {host_name} in AWX.")
+                    host_id = awx_host["id"]
+                else:
+                    logger.error(f"Invalid data for awx_host_serializer: {awx_host_serializer.errors}")
+                    continue  # Skip this host or handle the error appropriately
+            else:
+                # Host does not exist in AWX, create it
+                awx_host = inventory.create_host(serializer.data)
+                if awx_host:
+                    logger.info(f"Created host {host_name} in AWX.")
+                    # Update local cache
+                    awx_hosts[host_name] = awx_host
+                    host_id = awx_host["id"]
+                else:
+                    logger.error(
+                        f"Failed to create host {host_name} in AWX."
+                    )
+                    continue  # Skip to the next host or handle the error as appropriate
+
+            # Synchronize host-group associations
+            if awx_host["summary_fields"]["groups"]["count"] > len(
+                awx_host["summary_fields"]["groups"]["results"]
+            ):
+                current_groups = inventory.get_host_groups(host_id)
+            else:
+                current_groups = awx_host["summary_fields"]["groups"]["results"]
+            current_group_names = set(group["name"] for group in current_groups)
+
+            valid_group_names = set()
+            # Collect valid group names for this host
+            if hasattr(instance, 'site') and instance.site:
+                group_name = f"{group_prefixes[Site]}{instance.site.slug.replace('-', '_')}"
+                valid_group_names.add(group_name)
+            if hasattr(instance, 'role') and instance.role:
+                group_name = f"{group_prefixes[DeviceRole]}{instance.role.slug.replace('-', '_')}"
+                valid_group_names.add(group_name)
+            if isinstance(instance, Device) and instance.device_type:
+                group_name = f"{group_prefixes[DeviceType]}{instance.device_type.slug.replace('-', '_')}"
+                valid_group_names.add(group_name)
+            if hasattr(instance, 'tags'):
+                tags = instance.tags.all()
+                for tag in tags:
+                    group_name = f"{group_prefixes[Tag]}{tag.slug.replace('-', '_')}"
+                    valid_group_names.add(group_name)
+
+            # Associate host with missing groups
+            groups_to_associate = valid_group_names - current_group_names
+            for group_name in groups_to_associate:
+                group = awx_groups.get(group_name)
+                if group:
+                    inventory.associate_host_group(host_id, group["id"])
+                    logger.info(
+                        f"Associated host {host_name} with group {group_name}."
+                    )
+                else:
+                    logger.error(
+                        f"Group {group_name} not found in AWX when trying to associate with host {host_name}."
+                    )
+
+            # Disassociate host from groups that are no longer valid
+            groups_to_disassociate = current_group_names - valid_group_names
+            for group_name in groups_to_disassociate:
+                group = awx_groups.get(group_name)
+                if group:
+                    inventory.disassociate_host_group(host_id, group["id"])
+                    logger.info(
+                        f"Disassociated host {host_name} from group {group_name}."
+                    )
+
+    # Delete hosts in AWX that are not in NetBox
+    awx_host_names =set(awx_hosts.keys())
+    hosts_to_delete = awx_host_names - netbox_host_names
+    for host_name in hosts_to_delete:
+        inventory.delete_host(host_name)
+        logger.info(f"Deleted host {host_name} from AWX as it no longer exists in NetBox.")
+
     job.terminate()

netbox_awx_plugin/tests/test_synchronization.py

@@ -259,15 +259,60 @@ class SynchronizationTestCase(TestCase):
         # Ensure that associate_host_group was called
         mock_associate_host_group.assert_called_with(1, 2)
 
-    @patch('netbox_awx_plugin.synchronization.sync_host')
-    @patch('netbox_awx_plugin.synchronization.sync_group')
-    def test_sync_all(self, mock_sync_group, mock_sync_host):
+    @patch('netbox_awx_plugin.models.AWXInventory.disassociate_host_group')
+    @patch('netbox_awx_plugin.models.AWXInventory.associate_host_group')
+    @patch('netbox_awx_plugin.models.AWXInventory.get_host_groups')
+    @patch('netbox_awx_plugin.models.AWXInventory.get_group')
+    @patch('netbox_awx_plugin.models.AWXInventory.get_host')
+    @patch('netbox_awx_plugin.models.AWXInventory.get_all_groups')
+    @patch('netbox_awx_plugin.models.AWXInventory.create_group')
+    @patch('netbox_awx_plugin.models.AWXInventory.update_group')
+    @patch('netbox_awx_plugin.models.AWXInventory.get_all_hosts')
+    @patch('netbox_awx_plugin.models.AWXInventory.create_host')
+    @patch('netbox_awx_plugin.models.AWXInventory.update_host')
+    def test_sync_all(self, mock_update_host, mock_create_host, mock_get_all_hosts,
+                      mock_update_group, mock_create_group, mock_get_all_groups,
+                      mock_get_host, mock_get_group, mock_get_host_groups,
+                      mock_associate_host_group, mock_disassociate_host_group):
+        # Mock return values
+        mock_get_all_groups.return_value = {}
+        mock_get_all_hosts.return_value = {}
+        mock_create_group.return_value = {
+            'id': 1,
+            'name': 'site_test_site',
+            'description': '',
+            'variables': ''
+        }
+        mock_create_host.return_value = {
+            'id': 1,
+            'name': 'test-device.example.com',
+            'summary_fields': {
+                'groups': {
+                    'count': 0,
+                    'results': []
+                }
+            },
+            'variables': '',
+            'description': '',
+            'enabled': True
+        }
+
+        mock_get_host.return_value = mock_create_host.return_value
+        mock_get_group.return_value = mock_create_group.return_value
+        mock_get_host_groups.return_value = []
+
         job = Mock()
         job.object = self.awx_inventory
         sync_all(job)
-        # Ensure that sync_host and sync_group are called appropriately
-        self.assertTrue(mock_sync_group.called)
-        self.assertTrue(mock_sync_host.called)
+
+        # Ensure that create_group and create_host are called
+        mock_create_group.assert_called()
+        mock_create_host.assert_called()
+        # Ensure that associate_host_group is called
+        mock_associate_host_group.assert_called_with(1, 1)
+
+
+
 
     # Additional tests for helper functions
     def test_sync_host_group_association(self):