Rework service handling

Allow running playbooks without NetBox access. Mainly to bootstrap
NetBox itself.

Would prefer not to access network from filter plugins, so maybe do
that at some point also.

filter_plugins/netbox.py

@@ -7,7 +7,8 @@ class FilterModule(object):
     '''Various utilities for manipulating NetBox data'''
 
     def __init__(self):
-        self.nb = pynetbox.api(os.getenv('NETBOX_API'), os.getenv('NETBOX_TOKEN'))
+        if 'NETBOX_API' in os.environ and 'NETBOX_TOKEN' in os.environ:
+            self.nb = pynetbox.api(os.getenv('NETBOX_API'), os.getenv('NETBOX_TOKEN'))
 
     def filters(self):
         return {
@@ -37,11 +38,12 @@ class FilterModule(object):
 
     def allowed_prefixes(self, service):
         '''Return a list of allowed IP prefixes for the given service'''
-        service_data = self.nb.ipam.services.get(service['id']).custom_fields
+        if 'custom_fields' in service:
-        if service_data['allowed_prefixes']:
+            service = service['custom_fields']
-            yield from self.nb.ipam.prefixes.filter(id=[prefix['id'] for prefix in service_data['allowed_prefixes']])
+        if prefixes := service.get('allowed_prefixes'):
-        if service_data['allowed_vlans']:
+            yield from self.nb.ipam.prefixes.filter(id=[prefix['id'] for prefix in prefixes])
-            yield from self.nb.ipam.prefixes.filter(vlan_id=[vlan['id'] for vlan in service_data['allowed_vlans']])
+        if vlans := service.get('allowed_vlans'):
-        if service_data['allowed_clusters']:
+            yield from self.nb.ipam.prefixes.filter(vlan_id=[vlan['id'] for vlan in vlans])
-            for device in self.nb.dcim.devices.filter(cluster_id=[cluster['id'] for cluster in service_data['allowed_clusters']]):
+        if clusters := service.get('allowed_clusters'):
+            for device in self.nb.dcim.devices.filter(cluster_id=[cluster['id'] for cluster in clusters]):
                 yield from self.nb.ipam.ip_addresses.filter(role='loopback', device_id=device.id)

roles/alpine/files/unattended-upgrade

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+upgrade() {
+	echo "Starting upgrade on $(date)"
+	apk upgrade --update
+}
+
+upgrade >> /var/log/unattended-upgrade.log

roles/alpine/files/unattended-upgrade.logrotate

@@ -0,0 +1,3 @@
+/var/log/unattended-upgrade.log {
+	missingok
+}

roles/alpine/tasks/main.yml

@@ -22,6 +22,7 @@
     name:
     - git
     - iproute2
+    - logrotate
     - nftables
     - procps
     - rsync
@@ -64,3 +65,15 @@
         name: qemu-guest-agent
         enabled: yes
         state: started
+
+- name: Install automatic upgrade script
+  copy:
+    dest: /etc/periodic/weekly/
+    src: unattended-upgrade
+    mode: 0755
+
+- name: Configure log rotation for automatic upgrades
+  copy:
+    dest: /etc/logrotate.d/unattended-upgrade
+    src: unattended-upgrade.logrotate
+    mode: 0644

roles/alpine/templates/local.nft.j2

@@ -7,7 +7,9 @@ table inet filter {
 {% set prefixes4 = prefixes | selectattr('family.value', '==', 4) | map('string') %}
 {% set prefixes6 = prefixes | selectattr('family.value', '==', 6) | map('string') %}
 {% set ports = service.ports | compact_numlist %}
+{% if 'name' in service %}
         # service {{ service.name }}
+{% endif %}
 {% if prefixes4 or prefixes6 %}
 {% if prefixes4 %}
         ip saddr { {{ prefixes4 | join(', ') }} } tcp dport { {{ ports }} } accept

roles/ceph/templates/nftables.conf.j2

@@ -54,7 +54,7 @@ table inet filter {
         ip saddr @allowed accept # TODO remove exceptions
         ip6 saddr @allowed/6 accept # TODO remove exceptions
 
-{% for service in cluster.custom_fields.services %}
+{% for service in cluster_services %}
 {% set prefixes = service | allowed_prefixes %}
 {% set prefixes4 = prefixes | selectattr('family.value', '==', 4) | map('string') %}
 {% set prefixes6 = prefixes | selectattr('family.value', '==', 6) | map('string') %}

roles/dokuwiki/templates/dokuwiki.conf.j2

@@ -1,11 +1,10 @@
-{% for fqdn in fqdns %}
 server {
 	listen 443 ssl http2;
 	listen [::]:443 ssl http2;
-	server_name {{ fqdn }};
+	server_name {{ dns_name }};
 
-	ssl_certificate /etc/letsencrypt/live/{{ fqdn }}/fullchain.pem;
+	ssl_certificate /etc/letsencrypt/live/{{ dns_name }}/fullchain.pem;
-	ssl_certificate_key /etc/letsencrypt/live/{{ fqdn }}/privkey.pem;
+	ssl_certificate_key /etc/letsencrypt/live/{{ dns_name }}/privkey.pem;
 
 	client_max_body_size 100M;
 
@@ -35,5 +34,3 @@ server {
 		fastcgi_pass unix:/run/php-fpm.socket;
 	}
 }
-
-{% endfor %}

roles/facts/tasks/main.yml

@@ -1,18 +1,21 @@
 # Make expensive lookups to NetBox once for later reference by any host.
-- name: Lookup networks and prefixes
+- when: lookup("env", "NETBOX_API") != ""
-  set_fact:
+  block:
-    vlans: '{{ query("netbox.netbox.nb_lookup", "vlans", api_filter="group=new-net", raw_data=true)
+    - name: Lookup networks and prefixes
-        | sort(attribute="vid") }}'
+      set_fact:
-    prefixes: '{{ query("netbox.netbox.nb_lookup", "prefixes", raw_data=true)
+        vlans: '{{ query("netbox.netbox.nb_lookup", "vlans", api_filter="group=new-net", raw_data=true)
-        | sort(attribute="prefix") | sort(attribute="family.value") }}'
+            | sort(attribute="vid") }}'
+        prefixes: '{{ query("netbox.netbox.nb_lookup", "prefixes", raw_data=true)
+            | sort(attribute="prefix") | sort(attribute="family.value") }}'
 
-- name: Get my cluster and all nodes in it
+    - when: 'cluster is defined'
-  set_fact:
+      block:
-    cluster: '{{ query("netbox.netbox.nb_lookup", "clusters", raw_data=true, api_filter="name="+cluster) | first }}'
+        - name: Get my cluster and all nodes in it
-    nodes: '{{ groups["cluster_"+cluster] | map("extract", hostvars) | rejectattr("is_virtual") }}'
+          set_fact:
-  when: cluster
+            cluster: '{{ query("netbox.netbox.nb_lookup", "clusters", raw_data=true, api_filter="name="+cluster) | first }}'
+            nodes: '{{ groups["cluster_"+cluster] | map("extract", hostvars) | rejectattr("is_virtual") }}'
 
-- name: Get my domain names if any
+        - name: Get cluster services
-  set_fact:
+          set_fact:
-    fqdns: '{{ interfaces | map(attribute="ip_addresses") | flatten
+            cluster_services: '{{ (cluster_services|default([])) + query("netbox.netbox.nb_lookup", "services", raw_data=true, api_filter="id="+item) }}'
-        | map(attribute="dns_name") | reject("==", "") | sort | unique }}'
+          loop: '{{ cluster.custom_fields.services | map(attribute="id") | map("string") }}'

roles/forgejo/templates/forgejo.conf.j2

@@ -1,10 +1,9 @@
-{% for fqdn in fqdns %}
 server {
-    server_name {{ fqdn }};
+    server_name {{ dns_name }};
 
     listen [::]:443 ssl ipv6only=off;
-    ssl_certificate /etc/letsencrypt/live/{{ fqdn }}/fullchain.pem;
+    ssl_certificate /etc/letsencrypt/live/{{ dns_name }}/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/{{ fqdn }}/privkey.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ dns_name }}/privkey.pem;
 
     location / {
         proxy_pass http://unix:/var/lib/forgejo/socket;
@@ -19,5 +18,3 @@ server {
         client_max_body_size 512M;
     }
 }
-
-{% endfor %}

roles/netbox/tasks/app.yml

@@ -49,7 +49,7 @@
     line: '{{ item.line }}'
   loop:
     - key: '^ALLOWED_HOSTS = '
-      line: "ALLOWED_HOSTS = [{{ fqdns | map('regex_replace', '^(.*)$', '\"\\1\"') | join(', ') }}]"
+      line: "ALLOWED_HOSTS = ['{{ dns_name }}']"
     - key: 'USER.*PostgreSQL username'
       line: "    'USER': '{{ user }}', # PostgreSQL username"
     # XXX unnecessary?

roles/netbox/templates/netbox.conf.j2

@@ -1,10 +1,9 @@
-{% for fqdn in fqdns %}
 server {
-    server_name {{ fqdn }};
+    server_name {{ dns_name }};
 
     listen [::]:443 ssl ipv6only=off;
-    ssl_certificate /etc/letsencrypt/live/{{ fqdn }}/fullchain.pem;
+    ssl_certificate /etc/letsencrypt/live/{{ dns_name }}/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/{{ fqdn }}/privkey.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ dns_name }}/privkey.pem;
 
     client_max_body_size 100m;
 
@@ -19,5 +18,3 @@ server {
         proxy_set_header X-Forwarded-Proto $scheme;
     }
 }
-
-{% endfor %}

roles/nginx/tasks/main.yml

@@ -26,9 +26,8 @@
 
 - name: Get LE certificate
   command:
-    cmd: certbot certonly --non-interactive --agree-tos --register-unsafely-without-email --webroot --webroot-path /srv/http -d {{ item }}
+    cmd: certbot certonly --non-interactive --agree-tos --register-unsafely-without-email --webroot --webroot-path /srv/http -d {{ dns_name }}
-    creates: '/etc/letsencrypt/renewal/{{ item }}.conf'
+    creates: '/etc/letsencrypt/renewal/{{ dns_name }}.conf'
-  loop: '{{ fqdns }}'
 
 - name: Enable certbot renewal
   cron:

roles/proxmox/templates/cluster.fw.j2

@@ -8,16 +8,16 @@ IN Ping(ACCEPT) -log nolog # don’t be rude
 IN SSH(ACCEPT) -i mgmt # for ansible etc.
 IN ACCEPT -source {{ nodes | map('device_address') | flatten | selectattr('family.value', '==', 4) | map(attribute='address') | join(',') }} # my cluster
 IN ACCEPT -source {{ nodes | map('device_address') | flatten | selectattr('family.value', '==', 6) | map(attribute='address') | join(',') }} # my cluster
-{% for service in cluster.custom_fields.services %}
+{% for service in cluster_services %}
 {% set prefixes = service | allowed_prefixes %}
 {% set prefixes4 = prefixes | selectattr('family.value', '==', 4) | map('string') %}
 {% set prefixes6 = prefixes | selectattr('family.value', '==', 6) | map('string') %}
 {% set ports = service.ports | compact_numlist(range_delimiter=':') %}
 {% if prefixes4 %}
-IN ACCEPT -source {{ prefixes4 | join(',') }} -p {{ service.protocol }} -dport {{ ports }} # {{ service.name }}
+IN ACCEPT -source {{ prefixes4 | join(',') }} -p {{ service.protocol.value }} -dport {{ ports }} # {{ service.name }}
 {% endif %}
 {% if prefixes6 %}
-IN ACCEPT -source {{ prefixes6 | join(',') }} -p {{ service.protocol }} -dport {{ ports }} # {{ service.name }}
+IN ACCEPT -source {{ prefixes6 | join(',') }} -p {{ service.protocol.value }} -dport {{ ports }} # {{ service.name }}
 {% endif %}
 {% endfor %}