Skip to content
Snippets Groups Projects
Commit 86eb971d authored by Anders Harrisson's avatar Anders Harrisson
Browse files

Move full sync to a JobRunner subclass

parent fe5e3317
No related branches found
No related tags found
1 merge request!21Move to NetBox 4.x
Hide whitespace changes
Inline Side-by-side
netbox_awx_plugin/synchronization.py
+ 153
− 150
View file @ 86eb971d
......@@ -4,6 +4,7 @@ from dcim.models import Device, DeviceRole, DeviceType, Site
from ipam.models import Prefix
from virtualization.models import VirtualMachine
from extras.models import Tag
from netbox.jobs import JobRunner
from .models import AWXInventory
from .serializers import (
serializers_dict,
......@@ -190,157 +191,159 @@ def delete_group(inventory, sender, instance):
serializer = serializer_class(instance)
inventory.delete_group(serializer.data["name"])
def sync_all(job):
"""
Performs a full synchronization of the AWX inventory, including host-group associations.
Optimized to check for existing groups and hosts before creating or updating them.
"""
inventory = job.object
logger.info(
f"Performing full inventory sync for inventory {inventory.inventory_id}"
)
job.start()
# Collect all AWX groups and hosts
awx_groups = inventory.get_all_groups()
awx_hosts = inventory.get_all_hosts()
# Synchronize groups
group_models = [Site, DeviceRole, DeviceType, Prefix, Tag]
netbox_group_names = set()
for model in group_models:
instances = model.objects.all()
for instance in instances:
serializer = serializers_dict[model](instance)
group_name = serializer.data["name"]
netbox_group_names.add(group_name)
if group_name in awx_groups:
# Group exists in AWX, check if it needs updating
awx_group = awx_groups[group_name]
awx_group_serializer = AWXGroupSerializer(data=awx_group)
if awx_group_serializer.is_valid():
awx_group_data = awx_group_serializer.validated_data
if awx_group_data != serializer.data:
inventory.update_group(awx_group["id"], serializer.data)
logger.info(f"Updated group {group_name} in AWX.")
else:
logger.error(f"Invalid data for awx_group_serializer: {awx_group_serializer.errors}")
else:
# Group does not exist in AWX, create it
awx_group = inventory.create_group(serializer.data)
if awx_group:
logger.info(f"Created group {group_name} in AWX.")
# Update local cache
awx_groups[group_name] = awx_group
class FullInventorySyncJob(JobRunner):
class Meta:
name = "Full AWX Inventory sync"
def run(self, *args, **kwargs):
"""
Performs a full synchronization of the AWX inventory, including host-group associations.
Optimized to check for existing groups and hosts before creating or updating them.
"""
inventory = kwargs['inventory']
logger.info(
f"Performing full inventory sync for inventory {inventory.inventory_id}"
)
# Collect all AWX groups and hosts
awx_groups = inventory.get_all_groups()
awx_hosts = inventory.get_all_hosts()
# Synchronize groups
group_models = [Site, DeviceRole, DeviceType, Prefix, Tag]
netbox_group_names = set()
for model in group_models:
instances = model.objects.all()
for instance in instances:
serializer = serializers_dict[model](instance)
group_name = serializer.data["name"]
netbox_group_names.add(group_name)
if group_name in awx_groups:
# Group exists in AWX, check if it needs updating
awx_group = awx_groups[group_name]
awx_group_serializer = AWXGroupSerializer(data=awx_group)
if awx_group_serializer.is_valid():
awx_group_data = awx_group_serializer.validated_data
if awx_group_data != serializer.data:
inventory.update_group(awx_group["id"], serializer.data)
logger.info(f"Updated group {group_name} in AWX.")
else:
logger.error(f"Invalid data for awx_group_serializer: {awx_group_serializer.errors}")
else:
logger.error(
f"Failed to create group {group_name} in AWX."
)
awx_group_names = set(awx_groups.keys())
# Delete groups in AWX that are not in NetBox
groups_to_delete = awx_group_names - netbox_group_names
for group_name in groups_to_delete:
inventory.delete_group(group_name)
logger.info(f"Deleted group {group_name} from AWX as it no longer exists in NetBox.")
# Synchronize hosts
host_models = [Device, VirtualMachine]
netbox_host_names = set()
for model in host_models:
instances = model.objects.select_related(
"primary_ip4"
).prefetch_related("interfaces__ip_addresses")
for instance in instances:
if not (instance.primary_ip4 and instance.primary_ip4.dns_name):
continue
serializer = serializers_dict[model](instance)
host_name = serializer.data["name"]
netbox_host_names.add(host_name)
if host_name in awx_hosts:
# Host exists in AWX, check if it needs updating
awx_host = awx_hosts[host_name]
awx_host_serializer = AWXHostSerializer(data=awx_host)
if awx_host_serializer.is_valid():
awx_host_data = awx_host_serializer.validated_data
if awx_host_data != serializer.data:
inventory.update_host(awx_host["id"], serializer.data)
logger.info(f"Updated host {host_name} in AWX.")
host_id = awx_host["id"]
# Group does not exist in AWX, create it
awx_group = inventory.create_group(serializer.data)
if awx_group:
logger.info(f"Created group {group_name} in AWX.")
# Update local cache
awx_groups[group_name] = awx_group
else:
logger.error(
f"Failed to create group {group_name} in AWX."
)
awx_group_names = set(awx_groups.keys())
# Delete groups in AWX that are not in NetBox
groups_to_delete = awx_group_names - netbox_group_names
for group_name in groups_to_delete:
inventory.delete_group(group_name)
logger.info(f"Deleted group {group_name} from AWX as it no longer exists in NetBox.")
# Synchronize hosts
host_models = [Device, VirtualMachine]
netbox_host_names = set()
for model in host_models:
instances = model.objects.select_related(
"primary_ip4"
).prefetch_related("interfaces__ip_addresses")
for instance in instances:
if not (instance.primary_ip4 and instance.primary_ip4.dns_name):
continue
serializer = serializers_dict[model](instance)
host_name = serializer.data["name"]
netbox_host_names.add(host_name)
if host_name in awx_hosts:
# Host exists in AWX, check if it needs updating
awx_host = awx_hosts[host_name]
awx_host_serializer = AWXHostSerializer(data=awx_host)
if awx_host_serializer.is_valid():
awx_host_data = awx_host_serializer.validated_data
if awx_host_data != serializer.data:
inventory.update_host(awx_host["id"], serializer.data)
logger.info(f"Updated host {host_name} in AWX.")
host_id = awx_host["id"]
else:
logger.error(f"Invalid data for awx_host_serializer: {awx_host_serializer.errors}")
continue # Skip this host or handle the error appropriately
else:
logger.error(f"Invalid data for awx_host_serializer: {awx_host_serializer.errors}")
continue # Skip this host or handle the error appropriately
else:
# Host does not exist in AWX, create it
awx_host = inventory.create_host(serializer.data)
if awx_host:
logger.info(f"Created host {host_name} in AWX.")
# Update local cache
awx_hosts[host_name] = awx_host
host_id = awx_host["id"]
# Host does not exist in AWX, create it
awx_host = inventory.create_host(serializer.data)
if awx_host:
logger.info(f"Created host {host_name} in AWX.")
# Update local cache
awx_hosts[host_name] = awx_host
host_id = awx_host["id"]
else:
logger.error(
f"Failed to create host {host_name} in AWX."
)
continue # Skip to the next host or handle the error as appropriate
# Synchronize host-group associations
if awx_host["summary_fields"]["groups"]["count"] > len(
awx_host["summary_fields"]["groups"]["results"]
):
current_groups = inventory.get_host_groups(host_id)
else:
logger.error(
f"Failed to create host {host_name} in AWX."
)
continue # Skip to the next host or handle the error as appropriate
# Synchronize host-group associations
if awx_host["summary_fields"]["groups"]["count"] > len(
awx_host["summary_fields"]["groups"]["results"]
):
current_groups = inventory.get_host_groups(host_id)
else:
current_groups = awx_host["summary_fields"]["groups"]["results"]
current_group_names = set(group["name"] for group in current_groups)
valid_group_names = set()
# Collect valid group names for this host
if hasattr(instance, 'site') and instance.site:
group_name = f"{group_prefixes[Site]}{instance.site.slug.replace('-', '_')}"
valid_group_names.add(group_name)
if hasattr(instance, 'role') and instance.role:
group_name = f"{group_prefixes[DeviceRole]}{instance.role.slug.replace('-', '_')}"
valid_group_names.add(group_name)
if isinstance(instance, Device) and instance.device_type:
group_name = f"{group_prefixes[DeviceType]}{instance.device_type.slug.replace('-', '_')}"
valid_group_names.add(group_name)
if hasattr(instance, 'tags'):
tags = instance.tags.all()
for tag in tags:
group_name = f"{group_prefixes[Tag]}{tag.slug.replace('-', '_')}"
valid_group_names.add(group_name)
current_groups = awx_host["summary_fields"]["groups"]["results"]
current_group_names = set(group["name"] for group in current_groups)
# Associate host with missing groups
groups_to_associate = valid_group_names - current_group_names
for group_name in groups_to_associate:
group = awx_groups.get(group_name)
if group:
inventory.associate_host_group(host_id, group["id"])
logger.info(
f"Associated host {host_name} with group {group_name}."
)
else:
logger.error(
f"Group {group_name} not found in AWX when trying to associate with host {host_name}."
)
# Disassociate host from groups that are no longer valid
groups_to_disassociate = current_group_names - valid_group_names
for group_name in groups_to_disassociate:
group = awx_groups.get(group_name)
if group:
inventory.disassociate_host_group(host_id, group["id"])
logger.info(
f"Disassociated host {host_name} from group {group_name}."
)
# Delete hosts in AWX that are not in NetBox
awx_host_names =set(awx_hosts.keys())
hosts_to_delete = awx_host_names - netbox_host_names
for host_name in hosts_to_delete:
inventory.delete_host(host_name)
logger.info(f"Deleted host {host_name} from AWX as it no longer exists in NetBox.")
job.terminate()
valid_group_names = set()
# Collect valid group names for this host
if hasattr(instance, 'site') and instance.site:
group_name = f"{group_prefixes[Site]}{instance.site.slug.replace('-', '_')}"
valid_group_names.add(group_name)
if hasattr(instance, 'role') and instance.role:
group_name = f"{group_prefixes[DeviceRole]}{instance.role.slug.replace('-', '_')}"
valid_group_names.add(group_name)
if isinstance(instance, Device) and instance.device_type:
group_name = f"{group_prefixes[DeviceType]}{instance.device_type.slug.replace('-', '_')}"
valid_group_names.add(group_name)
if hasattr(instance, 'tags'):
tags = instance.tags.all()
for tag in tags:
group_name = f"{group_prefixes[Tag]}{tag.slug.replace('-', '_')}"
valid_group_names.add(group_name)
# Associate host with missing groups
groups_to_associate = valid_group_names - current_group_names
for group_name in groups_to_associate:
group = awx_groups.get(group_name)
if group:
inventory.associate_host_group(host_id, group["id"])
logger.info(
f"Associated host {host_name} with group {group_name}."
)
else:
logger.error(
f"Group {group_name} not found in AWX when trying to associate with host {host_name}."
)
# Disassociate host from groups that are no longer valid
groups_to_disassociate = current_group_names - valid_group_names
for group_name in groups_to_disassociate:
group = awx_groups.get(group_name)
if group:
inventory.disassociate_host_group(host_id, group["id"])
logger.info(
f"Disassociated host {host_name} from group {group_name}."
)
# Delete hosts in AWX that are not in NetBox
awx_host_names =set(awx_hosts.keys())
hosts_to_delete = awx_host_names - netbox_host_names
for host_name in hosts_to_delete:
inventory.delete_host(host_name)
logger.info(f"Deleted host {host_name} from AWX as it no longer exists in NetBox.")
netbox_awx_plugin/views.py
+ 2
− 2
View file @ 86eb971d
......@@ -6,7 +6,7 @@ from .api.serializers import AWXSerializer, AWXInventorySerializer
from django.shortcuts import redirect, render
from core.models import Job
from .models import AWXInventory
from .synchronization import sync_all
from .synchronization import FullInventorySyncJob
import logging
from django.contrib import messages
......@@ -58,7 +58,7 @@ class AWXInventorySyncView(generic.ObjectView):
def post(self, request, pk, *args, **kwargs):
logger.info("Sync inventory")
inventory = AWXInventory.objects.get(id=pk)
job = Job.enqueue(sync_all, instance=inventory, name="Full sync of inventory")
job = FullInventorySyncJob.enqueue(inventory=inventory)
messages.success(request, f"Queued job #{job.pk} to sync inventory {inventory}")
return render(
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment