diff --git a/netbox_awx_plugin/models.py b/netbox_awx_plugin/models.py
index 82567ab52adb6e0dfb4d78dfc45275fd1f26228b..59ea0b65bac3e5d10b2d299b84b4f7889e774da4 100644
--- a/netbox_awx_plugin/models.py
+++ b/netbox_awx_plugin/models.py
@@ -57,6 +57,32 @@ class AWXInventory(NetBoxModel):
else:
return None
+ def get_all_hosts(self):
+ """
+ Retrieves all hosts from the AWX inventory.
+ """
+ hosts = {}
+ page = 1
+ while True:
+ params = {"page": page}
+ r = requests.get(
+ url=urljoin(
+ self.awx.url,
+ f"/api/v2/inventories/{self.inventory_id}/hosts/",
+ ),
+ params=params,
+ headers=self.awx.get_headers(),
+ verify=False,
+ )
+ r.raise_for_status()
+ data = r.json()
+ for host in data["results"]:
+ hosts[host["name"]] = host
+ if not data.get("next"):
+ break
+ page += 1
+ return hosts
+
def create_host(self, data):
logger.info("Creating host with: {}".format(data))
r = requests.post(
@@ -65,7 +91,14 @@ class AWXInventory(NetBoxModel):
json=data,
verify=False,
)
- r.raise_for_status()
+ if r.status_code == 201:
+ return r.json() # Return the created host object
+ else:
+ logger.error(
+ f"Failed to create host {data['name']}, Response: "
+ f"{r.text if r else 'No response'}"
+ )
+ return None
def update_host(self, host_id, data):
logger.info("Updating host with: {}".format(data))
@@ -79,12 +112,17 @@ class AWXInventory(NetBoxModel):
def delete_host(self, hostname):
host = self.get_host(hostname)
- r = requests.delete(
- url=urljoin(self.awx.url, "/api/v2/hosts/{}/".format(host["id"])),
- headers=self.awx.get_headers(),
- verify=False,
- )
- r.raise_for_status()
+ if host:
+ r = requests.delete(
+ url=urljoin(
+ self.awx.url, "/api/v2/hosts/{}/".format(host["id"])
+ ),
+ headers=self.awx.get_headers(),
+ verify=False,
+ )
+ r.raise_for_status()
+ else:
+ logger.warning(f"Host {hostname} not found for deletion.")
def get_group(self, name):
params = {"name": name}
@@ -97,6 +135,32 @@ class AWXInventory(NetBoxModel):
r.raise_for_status()
return r.json()["results"][0] if len(r.json()["results"]) > 0 else None
+ def get_all_groups(self):
+ """
+ Retrieves all groups from the AWX inventory.
+ """
+ groups = {}
+ page = 1
+ while True:
+ params = {"page": page}
+ r = requests.get(
+ url=urljoin(
+ self.awx.url,
+ f"/api/v2/inventories/{self.inventory_id}/groups/",
+ ),
+ params=params,
+ headers=self.awx.get_headers(),
+ verify=False,
+ )
+ r.raise_for_status()
+ data = r.json()
+ for group in data["results"]:
+ groups[group["name"]] = group
+ if not data.get("next"):
+ break
+ page += 1
+ return groups
+
def update_group(self, name, data):
logger.info("Updating group {} with: {}".format(name, data))
group = self.get_group(name)
@@ -116,18 +180,27 @@ class AWXInventory(NetBoxModel):
json=data,
verify=False,
)
- r.raise_for_status()
+ if r.status_code == 201:
+ return r.json() # Return the created group object
+ else:
+ logger.error(
+ f"Failed to create group {data['name']}, Response: "
+ f"{r.text if r else 'No response'}"
+ )
+ return None
def delete_group(self, name):
logger.info("Deleting group {}".format(name))
group = self.get_group(name)
- if not group is None:
+ if group is not None:
r = requests.delete(
url=urljoin(self.awx.url, "/api/v2/groups/{}/".format(group["id"])),
headers=self.awx.get_headers(),
verify=False,
)
r.raise_for_status()
+ else:
+ logger.warning(f"Group {name} not found for deletion.")
def associate_host_group(self, host_id, group_id):
logger.info("Associating host {} to group {}".format(host_id, group_id))
diff --git a/netbox_awx_plugin/synchronization.py b/netbox_awx_plugin/synchronization.py
index 2e727b1692131b45ffd1da8ef579e28fbc170cec..7cf1ee280abca4918ebd4c0307c3fd911384254b 100644
--- a/netbox_awx_plugin/synchronization.py
+++ b/netbox_awx_plugin/synchronization.py
@@ -4,7 +4,6 @@ from dcim.models import Device, DeviceRole, DeviceType, Site
from ipam.models import Prefix
from virtualization.models import VirtualMachine
from extras.models import Tag
-from core.models import Job
from .models import AWXInventory
from .serializers import (
serializers_dict,
@@ -14,8 +13,6 @@ from .serializers import (
from django_rq import get_queue
from rq import get_current_job, Queue
from requests.exceptions import RequestException, HTTPError, ConnectionError, Timeout
-from urllib3.exceptions import InsecureRequestWarning
-import urllib3
# Set up logging
logger = logging.getLogger(__name__)
@@ -195,24 +192,155 @@ def delete_group(inventory, sender, instance):
def sync_all(job):
"""
- Performs a full synchronization of the AWX inventory.
+ Performs a full synchronization of the AWX inventory, including host-group associations.
+ Optimized to check for existing groups and hosts before creating or updating them.
"""
inventory = job.object
- logger.info(f"Performing full inventory sync for inventory {inventory.inventory_id}")
+ logger.info(
+ f"Performing full inventory sync for inventory {inventory.inventory_id}"
+ )
job.start()
- for site in Site.objects.all():
- sync_group(inventory, Site, site)
- for device_role in DeviceRole.objects.all():
- sync_group(inventory, DeviceRole, device_role)
- for device_type in DeviceType.objects.all():
- sync_group(inventory, DeviceType, device_type)
- for tag in Tag.objects.all():
- sync_group(inventory, Tag, tag)
-
- for device in Device.objects.all():
- if device.primary_ip4 and device.primary_ip4.dns_name:
- sync_host(inventory, Device, device)
- for vm in VirtualMachine.objects.all():
- if vm.primary_ip4 and vm.primary_ip4.dns_name:
- sync_host(inventory, VirtualMachine, vm)
+
+ # Collect all AWX groups and hosts
+ awx_groups = inventory.get_all_groups()
+ awx_hosts = inventory.get_all_hosts()
+
+ # Synchronize groups
+ group_models = [Site, DeviceRole, DeviceType, Prefix, Tag]
+ netbox_group_names = set()
+ for model in group_models:
+ instances = model.objects.all()
+ for instance in instances:
+ serializer = serializers_dict[model](instance)
+ group_name = serializer.data["name"]
+ netbox_group_names.add(group_name)
+
+ if group_name in awx_groups:
+ # Group exists in AWX, check if it needs updating
+ awx_group = awx_groups[group_name]
+ awx_group_serializer = AWXGroupSerializer(data=awx_group)
+ if awx_group_serializer.is_valid():
+ awx_group_data = awx_group_serializer.validated_data
+ if awx_group_data != serializer.data:
+ inventory.update_group(awx_group["id"], serializer.data)
+ logger.info(f"Updated group {group_name} in AWX.")
+ else:
+ logger.error(f"Invalid data for awx_group_serializer: {awx_group_serializer.errors}")
+ else:
+ # Group does not exist in AWX, create it
+ awx_group = inventory.create_group(serializer.data)
+ if awx_group:
+ logger.info(f"Created group {group_name} in AWX.")
+ # Update local cache
+ awx_groups[group_name] = awx_group
+ else:
+ logger.error(
+ f"Failed to create group {group_name} in AWX."
+ )
+
+ awx_group_names = set(awx_groups.keys())
+ # Delete groups in AWX that are not in NetBox
+ groups_to_delete = awx_group_names - netbox_group_names
+ for group_name in groups_to_delete:
+ inventory.delete_group(group_name)
+ logger.info(f"Deleted group {group_name} from AWX as it no longer exists in NetBox.")
+
+ # Synchronize hosts
+ host_models = [Device, VirtualMachine]
+ netbox_host_names = set()
+ for model in host_models:
+ instances = model.objects.select_related(
+ "primary_ip4"
+ ).prefetch_related("interfaces__ip_addresses")
+ for instance in instances:
+ if not (instance.primary_ip4 and instance.primary_ip4.dns_name):
+ continue
+ serializer = serializers_dict[model](instance)
+ host_name = serializer.data["name"]
+ netbox_host_names.add(host_name)
+
+ if host_name in awx_hosts:
+ # Host exists in AWX, check if it needs updating
+ awx_host = awx_hosts[host_name]
+ awx_host_serializer = AWXHostSerializer(data=awx_host)
+ if awx_host_serializer.is_valid():
+ awx_host_data = awx_host_serializer.validated_data
+ if awx_host_data != serializer.data:
+ inventory.update_host(awx_host["id"], serializer.data)
+ logger.info(f"Updated host {host_name} in AWX.")
+ host_id = awx_host["id"]
+ else:
+ logger.error(f"Invalid data for awx_host_serializer: {awx_host_serializer.errors}")
+ continue # Skip this host or handle the error appropriately
+ else:
+ # Host does not exist in AWX, create it
+ awx_host = inventory.create_host(serializer.data)
+ if awx_host:
+ logger.info(f"Created host {host_name} in AWX.")
+ # Update local cache
+ awx_hosts[host_name] = awx_host
+ host_id = awx_host["id"]
+ else:
+ logger.error(
+ f"Failed to create host {host_name} in AWX."
+ )
+ continue # Skip to the next host or handle the error as appropriate
+
+ # Synchronize host-group associations
+ if awx_host["summary_fields"]["groups"]["count"] > len(
+ awx_host["summary_fields"]["groups"]["results"]
+ ):
+ current_groups = inventory.get_host_groups(host_id)
+ else:
+ current_groups = awx_host["summary_fields"]["groups"]["results"]
+ current_group_names = set(group["name"] for group in current_groups)
+
+ valid_group_names = set()
+ # Collect valid group names for this host
+ if hasattr(instance, 'site') and instance.site:
+ group_name = f"{group_prefixes[Site]}{instance.site.slug.replace('-', '_')}"
+ valid_group_names.add(group_name)
+ if hasattr(instance, 'role') and instance.role:
+ group_name = f"{group_prefixes[DeviceRole]}{instance.role.slug.replace('-', '_')}"
+ valid_group_names.add(group_name)
+ if isinstance(instance, Device) and instance.device_type:
+ group_name = f"{group_prefixes[DeviceType]}{instance.device_type.slug.replace('-', '_')}"
+ valid_group_names.add(group_name)
+ if hasattr(instance, 'tags'):
+ tags = instance.tags.all()
+ for tag in tags:
+ group_name = f"{group_prefixes[Tag]}{tag.slug.replace('-', '_')}"
+ valid_group_names.add(group_name)
+
+ # Associate host with missing groups
+ groups_to_associate = valid_group_names - current_group_names
+ for group_name in groups_to_associate:
+ group = awx_groups.get(group_name)
+ if group:
+ inventory.associate_host_group(host_id, group["id"])
+ logger.info(
+ f"Associated host {host_name} with group {group_name}."
+ )
+ else:
+ logger.error(
+ f"Group {group_name} not found in AWX when trying to associate with host {host_name}."
+ )
+
+ # Disassociate host from groups that are no longer valid
+ groups_to_disassociate = current_group_names - valid_group_names
+ for group_name in groups_to_disassociate:
+ group = awx_groups.get(group_name)
+ if group:
+ inventory.disassociate_host_group(host_id, group["id"])
+ logger.info(
+ f"Disassociated host {host_name} from group {group_name}."
+ )
+
+ # Delete hosts in AWX that are not in NetBox
+ awx_host_names =set(awx_hosts.keys())
+ hosts_to_delete = awx_host_names - netbox_host_names
+ for host_name in hosts_to_delete:
+ inventory.delete_host(host_name)
+ logger.info(f"Deleted host {host_name} from AWX as it no longer exists in NetBox.")
+
job.terminate()
diff --git a/netbox_awx_plugin/tests/test_synchronization.py b/netbox_awx_plugin/tests/test_synchronization.py
index 7e7a0df422748c2bb97f08653363f912264ab312..8ef3b18326a875bc84dd80ed1da7162bc6ee26e8 100644
--- a/netbox_awx_plugin/tests/test_synchronization.py
+++ b/netbox_awx_plugin/tests/test_synchronization.py
@@ -259,15 +259,60 @@ class SynchronizationTestCase(TestCase):
# Ensure that associate_host_group was called
mock_associate_host_group.assert_called_with(1, 2)
- @patch('netbox_awx_plugin.synchronization.sync_host')
- @patch('netbox_awx_plugin.synchronization.sync_group')
- def test_sync_all(self, mock_sync_group, mock_sync_host):
+ @patch('netbox_awx_plugin.models.AWXInventory.disassociate_host_group')
+ @patch('netbox_awx_plugin.models.AWXInventory.associate_host_group')
+ @patch('netbox_awx_plugin.models.AWXInventory.get_host_groups')
+ @patch('netbox_awx_plugin.models.AWXInventory.get_group')
+ @patch('netbox_awx_plugin.models.AWXInventory.get_host')
+ @patch('netbox_awx_plugin.models.AWXInventory.get_all_groups')
+ @patch('netbox_awx_plugin.models.AWXInventory.create_group')
+ @patch('netbox_awx_plugin.models.AWXInventory.update_group')
+ @patch('netbox_awx_plugin.models.AWXInventory.get_all_hosts')
+ @patch('netbox_awx_plugin.models.AWXInventory.create_host')
+ @patch('netbox_awx_plugin.models.AWXInventory.update_host')
+ def test_sync_all(self, mock_update_host, mock_create_host, mock_get_all_hosts,
+ mock_update_group, mock_create_group, mock_get_all_groups,
+ mock_get_host, mock_get_group, mock_get_host_groups,
+ mock_associate_host_group, mock_disassociate_host_group):
+ # Mock return values
+ mock_get_all_groups.return_value = {}
+ mock_get_all_hosts.return_value = {}
+ mock_create_group.return_value = {
+ 'id': 1,
+ 'name': 'site_test_site',
+ 'description': '',
+ 'variables': ''
+ }
+ mock_create_host.return_value = {
+ 'id': 1,
+ 'name': 'test-device.example.com',
+ 'summary_fields': {
+ 'groups': {
+ 'count': 0,
+ 'results': []
+ }
+ },
+ 'variables': '',
+ 'description': '',
+ 'enabled': True
+ }
+
+ mock_get_host.return_value = mock_create_host.return_value
+ mock_get_group.return_value = mock_create_group.return_value
+ mock_get_host_groups.return_value = []
+
job = Mock()
job.object = self.awx_inventory
sync_all(job)
- # Ensure that sync_host and sync_group are called appropriately
- self.assertTrue(mock_sync_group.called)
- self.assertTrue(mock_sync_host.called)
+
+ # Ensure that create_group and create_host are called
+ mock_create_group.assert_called()
+ mock_create_host.assert_called()
+ # Ensure that associate_host_group is called
+ mock_associate_host_group.assert_called_with(1, 1)
+
+
+
# Additional tests for helper functions
def test_sync_host_group_association(self):