Rework service handling

Allow running playbooks without NetBox access. Mainly to bootstrap
NetBox itself.

Would prefer not to access network from filter plugins, so maybe do
that at some point also.

filter_plugins/netbox.py

@@ -7,7 +7,8 @@ class FilterModule(object):
     '''Various utilities for manipulating NetBox data'''
 
     def __init__(self):
-        self.nb = pynetbox.api(os.getenv('NETBOX_API'), os.getenv('NETBOX_TOKEN'))
+        if 'NETBOX_API' in os.environ and 'NETBOX_TOKEN' in os.environ:
+            self.nb = pynetbox.api(os.getenv('NETBOX_API'), os.getenv('NETBOX_TOKEN'))
 
     def filters(self):
         return {
@@ -37,11 +38,12 @@ class FilterModule(object):
 
     def allowed_prefixes(self, service):
         '''Return a list of allowed IP prefixes for the given service'''
-        service_data = self.nb.ipam.services.get(service['id']).custom_fields
+        if 'custom_fields' in service:
-        if service_data['allowed_prefixes']:
+            service = service['custom_fields']
-            yield from self.nb.ipam.prefixes.filter(id=[prefix['id'] for prefix in service_data['allowed_prefixes']])
+        if prefixes := service.get('allowed_prefixes'):
-        if service_data['allowed_vlans']:
+            yield from self.nb.ipam.prefixes.filter(id=[prefix['id'] for prefix in prefixes])
-            yield from self.nb.ipam.prefixes.filter(vlan_id=[vlan['id'] for vlan in service_data['allowed_vlans']])
+        if vlans := service.get('allowed_vlans'):
-        if service_data['allowed_clusters']:
+            yield from self.nb.ipam.prefixes.filter(vlan_id=[vlan['id'] for vlan in vlans])
-            for device in self.nb.dcim.devices.filter(cluster_id=[cluster['id'] for cluster in service_data['allowed_clusters']]):
+        if clusters := service.get('allowed_clusters'):
+            for device in self.nb.dcim.devices.filter(cluster_id=[cluster['id'] for cluster in clusters]):
                 yield from self.nb.ipam.ip_addresses.filter(role='loopback', device_id=device.id)

roles/alpine/templates/local.nft.j2

@@ -7,7 +7,9 @@ table inet filter {
 {% set prefixes4 = prefixes | selectattr('family.value', '==', 4) | map('string') %}
 {% set prefixes6 = prefixes | selectattr('family.value', '==', 6) | map('string') %}
 {% set ports = service.ports | compact_numlist %}
+{% if 'name' in service %}
         # service {{ service.name }}
+{% endif %}
 {% if prefixes4 or prefixes6 %}
 {% if prefixes4 %}
         ip saddr { {{ prefixes4 | join(', ') }} } tcp dport { {{ ports }} } accept

roles/ceph/templates/nftables.conf.j2

@@ -54,7 +54,7 @@ table inet filter {
         ip saddr @allowed accept # TODO remove exceptions
         ip6 saddr @allowed/6 accept # TODO remove exceptions
 
-{% for service in cluster.custom_fields.services %}
+{% for service in cluster_services %}
 {% set prefixes = service | allowed_prefixes %}
 {% set prefixes4 = prefixes | selectattr('family.value', '==', 4) | map('string') %}
 {% set prefixes6 = prefixes | selectattr('family.value', '==', 6) | map('string') %}

roles/facts/tasks/main.yml

@@ -1,18 +1,21 @@
 # Make expensive lookups to NetBox once for later reference by any host.
-- name: Lookup networks and prefixes
+- when: lookup("env", "NETBOX_API") != ""
-  set_fact:
+  block:
-    vlans: '{{ query("netbox.netbox.nb_lookup", "vlans", api_filter="group=new-net", raw_data=true)
+    - name: Lookup networks and prefixes
-        | sort(attribute="vid") }}'
+      set_fact:
-    prefixes: '{{ query("netbox.netbox.nb_lookup", "prefixes", raw_data=true)
+        vlans: '{{ query("netbox.netbox.nb_lookup", "vlans", api_filter="group=new-net", raw_data=true)
-        | sort(attribute="prefix") | sort(attribute="family.value") }}'
+            | sort(attribute="vid") }}'
+        prefixes: '{{ query("netbox.netbox.nb_lookup", "prefixes", raw_data=true)
+            | sort(attribute="prefix") | sort(attribute="family.value") }}'
 
-- name: Get my cluster and all nodes in it
+    - when: 'cluster is defined'
-  set_fact:
+      block:
-    cluster: '{{ query("netbox.netbox.nb_lookup", "clusters", raw_data=true, api_filter="name="+cluster) | first }}'
+        - name: Get my cluster and all nodes in it
-    nodes: '{{ groups["cluster_"+cluster] | map("extract", hostvars) | rejectattr("is_virtual") }}'
+          set_fact:
-  when: cluster
+            cluster: '{{ query("netbox.netbox.nb_lookup", "clusters", raw_data=true, api_filter="name="+cluster) | first }}'
+            nodes: '{{ groups["cluster_"+cluster] | map("extract", hostvars) | rejectattr("is_virtual") }}'
 
-- name: Get my domain names if any
+        - name: Get cluster services
-  set_fact:
+          set_fact:
-    fqdns: '{{ interfaces | map(attribute="ip_addresses") | flatten
+            cluster_services: '{{ (cluster_services|default([])) + query("netbox.netbox.nb_lookup", "services", raw_data=true, api_filter="id="+item) }}'
-        | map(attribute="dns_name") | reject("==", "") | sort | unique }}'
+          loop: '{{ cluster.custom_fields.services | map(attribute="id") | map("string") }}'

roles/proxmox/templates/cluster.fw.j2

@@ -8,16 +8,16 @@ IN Ping(ACCEPT) -log nolog # don’t be rude
 IN SSH(ACCEPT) -i mgmt # for ansible etc.
 IN ACCEPT -source {{ nodes | map('device_address') | flatten | selectattr('family.value', '==', 4) | map(attribute='address') | join(',') }} # my cluster
 IN ACCEPT -source {{ nodes | map('device_address') | flatten | selectattr('family.value', '==', 6) | map(attribute='address') | join(',') }} # my cluster
-{% for service in cluster.custom_fields.services %}
+{% for service in cluster_services %}
 {% set prefixes = service | allowed_prefixes %}
 {% set prefixes4 = prefixes | selectattr('family.value', '==', 4) | map('string') %}
 {% set prefixes6 = prefixes | selectattr('family.value', '==', 6) | map('string') %}
 {% set ports = service.ports | compact_numlist(range_delimiter=':') %}
 {% if prefixes4 %}
-IN ACCEPT -source {{ prefixes4 | join(',') }} -p {{ service.protocol }} -dport {{ ports }} # {{ service.name }}
+IN ACCEPT -source {{ prefixes4 | join(',') }} -p {{ service.protocol.value }} -dport {{ ports }} # {{ service.name }}
 {% endif %}
 {% if prefixes6 %}
-IN ACCEPT -source {{ prefixes6 | join(',') }} -p {{ service.protocol }} -dport {{ ports }} # {{ service.name }}
+IN ACCEPT -source {{ prefixes6 | join(',') }} -p {{ service.protocol.value }} -dport {{ ports }} # {{ service.name }}
 {% endif %}
 {% endfor %}