Factor frr role from debian, ceph and proxmox

Consolidate base system and networking setup into debian role and BGP
configuration into frr role. Add facts role to collect data from NetBox
once to avoid many slow lookups. Also many other tweaks and cleanups.

roles/ceph/handlers/main.yml

@@ -1,7 +1,5 @@
-- name: reboot
-  reboot:
-
 - name: reload nftables
   service:
     name: nftables
     state: reloaded
+  when: "'handler' not in ansible_skip_tags"

roles/ceph/tasks/firewall.yml

@@ -1,7 +1,3 @@
-- name: Retrieve service list
-  set_fact:
-    services: '{{ query("netbox.netbox.nb_lookup", "clusters", raw_data=true, api_filter="name="+cluster) | map(attribute="custom_fields.services") | flatten }}'
-
 - name: Install nftables
   package:
     name: nftables

roles/ceph/tasks/main.yml

@@ -1,7 +1,3 @@
-- name: Get all nodes in my cluster
-  set_fact:
-    nodes: "{{ groups['cluster_'+cluster] | map('extract', hostvars) }}"
-
 - name: Configure /etc/hosts
   template:
     dest: /etc/hosts

roles/ceph/templates/nftables.conf.j2

@@ -46,10 +46,15 @@ table inet filter {
         iif lan0 ip6 saddr fe80::/64 accept
         iif lan1 ip6 saddr fe80::/64 accept
 
+        iifname mgmt accept comment "management access"
+
         ip saddr @cluster accept comment "accept connections from other nodes"
         ip6 saddr @cluster/6 accept comment "accept connections from other nodes"
 
-{% for service in services %}
+        ip saddr @allowed accept # TODO remove exceptions
+        ip6 saddr @allowed/6 accept # TODO remove exceptions
+
+{% for service in cluster.custom_fields.services %}
 {% set prefixes = service | allowed_prefixes %}
 {% set prefixes4 = prefixes | selectattr('family.value', '==', 4) | map('string') %}
 {% set prefixes6 = prefixes | selectattr('family.value', '==', 6) | map('string') %}
@@ -63,11 +68,6 @@ table inet filter {
 {% endif %}
 
 {% endfor %}
-
-        iifname mgmt accept comment "management access"
-
-        ip saddr @allowed accept
-        ip6 saddr @allowed/6 accept
     }
 
     chain forward {

roles/debian/files/sshd_config.mgmt

@@ -1,5 +1,4 @@
 # This is for sshd in management VRF, for ansible and other not-really-OOB stuff.
-
 PidFile none
 UsePAM no
 Subsystem sftp /usr/lib/openssh/sftp-server

roles/debian/handlers/main.yml

@@ -1,14 +1,7 @@
 - name: reboot
   reboot:
+  when: "'handler' not in ansible_skip_tags"
 
 - name: reload interfaces
   command: ifreload -a
-
-- name: reload frr
-  command: /usr/lib/frr/frr-reload.py --reload /etc/frr/frr.conf
-
-- name: restart frr
-  service: name=frr state=restarted
-
-- name: update package cache
-  apt: update_cache=true
+  when: "'handler' not in ansible_skip_tags"

roles/debian/tasks/main.yml

@@ -1,19 +1,3 @@
-- include_tasks: mgmt.yml
-
-- name: Set up loopback interface
-  template:
-    dest: /etc/network/interfaces.d/loopback.intf
-    src: loopback.intf.j2
-    mode: 0644
-  notify: reload interfaces
-
-- name: Set up fabric interfaces
-  template:
-    dest: /etc/network/interfaces.d/fabric.intf
-    src: fabric.intf.j2
-    mode: 0644
-  notify: reload interfaces
-
 - name: Set hostname
   hostname:
     name: '{{ inventory_hostname }}'
@@ -32,33 +16,65 @@
   notify: update package cache
   when: debian_release is defined
 
-- meta: flush_handlers
-
 - name: Install essential packages
-  package: name=git,rsync,vim,tmux
+  package:
+    name:
+      - git
+      - ifupdown2
+      - rsync
+      - vim
+      - tmux
 
-- name: Install FRR
-  package: name=frr,frr-pythontools
-
-- name: Enable BGP and BFD
-  lineinfile:
-    path: /etc/frr/daemons
-    regexp: "^{{ item }}="
-    line: "{{ item }}=yes"
-  loop:
-    - bfdd
-    - bgpd
-  notify: restart frr
-
-- name: Copy FRR config
+- name: Add rules to rename network interfaces
   template:
-    dest: /etc/frr/frr.conf
-    src: frr.conf.j2
+    dest: /etc/udev/rules.d/10-network.rules
+    src: 10-network.rules.j2
     mode: 0644
-  notify: reload frr
+  notify: reboot
 
-- name: Enable FRR service
+# we don’t want to template this file because it gets overwritten by proxmox
+# so just try removing anything that messes with our definitions in interfaces.d
+- name: Remove interface definitions added by installer
+  lineinfile:
+    path: /etc/network/interfaces
+    regexp: '^iface [^ ]* inet'
+    state: absent
+  notify: reload interfaces
+
+- name: Include interfaces.d definitions
+  lineinfile:
+    path: /etc/network/interfaces
+    line: 'source /etc/network/interfaces.d/*'
+  notify: reload interfaces
+
+- name: Set up interfaces
+  template:
+    dest: /etc/network/interfaces.d/ansible.intf
+    src: ansible.intf.j2
+    mode: 0644
+  notify: reload interfaces
+
+- name: Run SSH instance in management VRF
+  when: interfaces | selectattr('vrf') | selectattr('vrf.name', '==', 'mgmt')
+  block:
+    - name: Configure SSH instance in management VRF
+      copy:
+        dest: /etc/ssh/
+        src: sshd_config.mgmt
+        mode: 0644
+      notify: reboot
+
+    - name: Set up a SSH instance in management VRF
+      copy:
+        dest: /etc/systemd/system/
+        src: sshd@mgmt.service
+        mode: 0644
+      notify: reboot
+
+    - name: Enable management SSH
       service:
-    name: frr
+        name: sshd@mgmt
         enabled: yes
-  notify: restart frr
+      notify: reboot
+
+- meta: flush_handlers

roles/debian/tasks/mgmt.yml

@@ -1,39 +0,0 @@
-- name: Add rules to rename network interfaces
-  template:
-    dest: /etc/udev/rules.d/10-network.rules
-    src: 10-network.rules.j2
-    mode: 0644
-  notify: reboot
-
-- name: Set up management interface
-  template:
-    dest: /etc/network/interfaces
-    src: interfaces.j2
-    mode: 0644
-  notify: reboot
-
-- name: Install ifupdown2
-  package: name=ifupdown2
-  notify: reboot
-
-- name: Configure SSH instance in management VRF
-  copy:
-    dest: /etc/ssh/sshd_config.mgmt
-    src: sshd_config.mgmt
-    mode: 0644
-  notify: reboot
-
-- name: Set up a SSH instance in management VRF
-  copy:
-    dest: /etc/systemd/system/
-    src: sshd@mgmt.service
-    mode: 0644
-  notify: reboot
-
-- name: Enable management SSH
-  service:
-    name: sshd@mgmt
-    enabled: yes
-  notify: reboot
-
-- meta: flush_handlers

roles/debian/templates/ansible.intf.j2

@@ -0,0 +1,31 @@
+{% for vrf in interfaces | selectattr('vrf') | map(attribute='vrf.name') | sort | unique %}
+auto {{ vrf }}
+iface {{ vrf }}
+    vrf-table auto
+    address 127.0.0.1/8
+    address ::1/128
+
+{%+ endfor %}
+
+{%- for iface in interfaces | selectattr('enabled') %}
+{% if iface.mgmt_only is not defined or not iface.mgmt_only %}
+auto {{ iface.name }}
+iface {{ iface.name }}{% if iface.name == 'lo' %} inet loopback{% endif +%}
+{% if iface.mtu %}
+    mtu {{ iface.mtu }}
+{% endif %}
+{% if iface.vrf %}
+    vrf {{ iface.vrf.name }}
+{% endif %}
+{% for ip in iface.ip_addresses %}
+    address {{ ip.address }}
+{% set subnet = ip.address | ipaddr('subnet') %}
+{% set prefix = prefixes | selectattr('prefix', '==', subnet) | first %}
+{% set gateway = prefix.custom_fields.gateway.address %}
+{% if gateway is defined %}
+    gateway {{ gateway | ipaddr('address') }}
+{% endif %}
+{% endfor %}
+
+{% endif %}
+{% endfor %}

roles/debian/templates/frr.conf.j2

@@ -1,35 +0,0 @@
-frr defaults datacenter
-service integrated-vtysh-config
-log syslog
-
-router bgp {{ asn.asn }}
-  bgp bestpath as-path multipath-relax
-
-  neighbor fabric peer-group
-  neighbor fabric remote-as external
-  neighbor fabric capability extended-nexthop
-
-{% for iface in interfaces | selectattr('name', 'match', '^lan') %}
-  neighbor {{ iface.name }} interface peer-group fabric
-  neighbor {{ iface.name }} bfd
-{% endfor %}
-
-  address-family ipv4 unicast
-    redistribute connected route-map loopback
-    neighbor fabric activate
-    neighbor fabric route-map local out
-  exit-address-family
-
-  address-family ipv6 unicast
-    redistribute connected route-map loopback
-    neighbor fabric activate
-    neighbor fabric route-map local out
-  exit-address-family
-
-bgp as-path access-list local permit ^$
-
-route-map loopback permit 1
-  match interface lo
-
-route-map local permit 1
-  match as-path local

roles/facts/tasks/main.yml

@@ -0,0 +1,13 @@
+# Make expensive lookups to NetBox once for later reference by any host.
+- name: Lookup networks and prefixes
+  set_fact:
+    vlans: '{{ query("netbox.netbox.nb_lookup", "vlans", api_filter="group=new-net", raw_data=true)
+        | sort(attribute="vid") }}'
+    prefixes: '{{ query("netbox.netbox.nb_lookup", "prefixes", raw_data=true)
+        | sort(attribute="prefix") | sort(attribute="family.value") }}'
+
+- name: Get my cluster and all nodes in it
+  set_fact:
+    cluster: '{{ query("netbox.netbox.nb_lookup", "clusters", raw_data=true, api_filter="name="+cluster) | first }}'
+    nodes: '{{ groups["cluster_"+cluster] | map("extract", hostvars) | rejectattr("is_virtual") }}'
+  when: cluster

roles/frr/handlers/main.yml

@@ -0,0 +1,7 @@
+- name: reload frr
+  command: /usr/lib/frr/frr-reload.py --reload /etc/frr/frr.conf
+  when: "'handler' not in ansible_skip_tags"
+
+- name: restart frr
+  service: name=frr state=restarted
+  when: "'handler' not in ansible_skip_tags"

roles/frr/tasks/main.yml

@@ -1,8 +1,7 @@
 - name: Install FRR
-  package:
-    name: frr
+  package: name=frr,frr-pythontools
 
-- name: Configure FRR
+- name: Copy FRR config
   template:
     dest: /etc/frr/frr.conf
     src: frr.conf.j2
@@ -15,7 +14,17 @@
     line: 'frr_profile="datacenter"'
     regexp: '#?frr_profile="datacenter"'
 
-- name: Enable FRR
+- name: Enable FRR daemons
+  lineinfile:
+    path: /etc/frr/daemons
+    regexp: "^{{ item }}="
+    line: "{{ item }}=yes"
+  loop:
+    - bfdd
+    - bgpd
+  notify: restart frr
+
+- name: Enable FRR service
   service:
     name: frr
     enabled: yes

roles/frr/templates/frr.conf.j2

@@ -2,7 +2,7 @@ frr defaults datacenter
 service integrated-vtysh-config
 log syslog
 
-# We only have the default route, so allow talking to BGP peers over it.
+# we only have the default route, so allow talking to BGP peers over it
 ip nht resolve-via-default
 
 router bgp {{ asn.asn }}
@@ -12,7 +12,7 @@ router bgp {{ asn.asn }}
   neighbor fabric remote-as external
   neighbor fabric capability extended-nexthop
 
-{% for iface in interfaces | rejectattr('name', '==', 'lo') | rejectattr('mgmt_only') | rejectattr('vrf') %}
+{% for iface in interfaces | selectattr('name', 'match', '^lan') %}
   neighbor {{ iface.name }} interface peer-group fabric
   neighbor {{ iface.name }} bfd
 {% endfor %}
@@ -29,11 +29,13 @@ router bgp {{ asn.asn }}
     neighbor fabric route-map local out
   exit-address-family
 
+{% if cluster.custom_fields and cluster.custom_fields.vlans %}
   address-family l2vpn evpn
     neighbor fabric activate
     neighbor fabric route-map local out
     advertise-all-vni
   exit-address-family
+{% endif %}
 
 bgp as-path access-list local permit ^$
 

roles/proxmox/files/sshd@mgmt.service

@@ -1,16 +0,0 @@
-[Unit]
-Description=OpenBSD Secure Shell server (management VRF)
-After=network.target auditd.service
-
-[Service]
-ExecStartPre=/usr/sbin/sshd -t
-ExecStart=ip vrf exec mgmt /usr/sbin/sshd -f /etc/ssh/sshd_config.mgmt
-ExecReload=/usr/sbin/sshd -t
-ExecReload=/bin/kill -HUP $MAINPID
-KillMode=process
-Restart=on-failure
-RestartPreventExitStatus=255
-Type=notify
-
-[Install]
-WantedBy=multi-user.target

roles/proxmox/files/sshd_config.mgmt

@@ -1,15 +0,0 @@
-# This is for sshd in management VRF, for ansible and other not-really-OOB stuff.
-
-PidFile none
-UsePAM no
-Subsystem sftp /usr/lib/openssh/sftp-server
-
-# Only allow pubkey auth.
-KbdInteractiveAuthentication no
-PasswordAuthentication no
-PermitRootLogin prohibit-password
-
-# Disable what we can.
-AllowTcpForwarding no
-GatewayPorts no
-X11Forwarding no

roles/proxmox/handlers/main.yml

@@ -1,8 +1,3 @@
-- name: reboot
-  reboot:
-
 - name: reload interfaces
   command: ifreload -a
-
-- name: reload frr
-  command: /usr/lib/frr/frr-reload.py --reload /etc/frr/frr.conf
+  when: "'handler' not in ansible_skip_tags"

roles/proxmox/tasks/firewall.yml

@@ -5,4 +5,4 @@
     mode: 0640
     owner: root
     group: www-data
-  when: inventory_hostname == primary
+  when: is_primary

roles/proxmox/tasks/main.yml

@@ -1,11 +1,7 @@
-- name: Get all nodes in my cluster
-  set_fact:
-    nodes: "{{ groups['cluster_'+cluster] | map('extract', hostvars) }}"
-
 # choose a node for tasks that should only run on (any) one node, e.g. when writing to /etc/pve
-- name: Select primary node
+- name: Select the primary node
   set_fact:
-    primary: '{{ nodes | map(attribute="inventory_hostname") | sort | first }}'
+    is_primary: '{{ inventory_hostname == (nodes | map(attribute="inventory_hostname") | sort | first) }}'
 
 - name: Disable enterprise repositories
   apt_repository:
@@ -22,27 +18,15 @@
   apt_repository:
     repo: 'deb http://download.proxmox.com/debian/pve {{ ansible_distribution_release }} pve-no-subscription'
 
-- name: Set hostname
-  hostname:
-    name: '{{ inventory_hostname }}'
-
-- name: Add rules to rename network interfaces
-  template:
-    dest: /etc/udev/rules.d/10-network.rules
-    src: 10-network.rules.j2
-    mode: 0644
-  notify: reboot
-
 - name: Set up sysctls
   copy:
     dest: /etc/sysctl.d/local.conf
     src: sysctl.conf
 
-- name: Set up interfaces
+- name: Set VXLAN local tunnel IP
   template:
-    dest: /etc/network/interfaces.d/real.intf
-    src: real.intf.j2
-    mode: 0644
+    dest: /etc/network/interfaces.d/loopback.intf
+    src: loopback.intf.j2
   notify: reload interfaces
 
 - name: Set up bridges
@@ -52,10 +36,6 @@
     mode: 0644
   notify: reload interfaces
 
-- include_tasks: mgmt.yml
-
 - include_tasks: firewall.yml
 
-- include_tasks: frr.yml
-
 - include_tasks: user.yml

roles/proxmox/tasks/mgmt.yml

@@ -1,24 +0,0 @@
-# We could probably avoid rebooting in some cases, but those should never happen
-# in normal operation anyway. This way all setup is done before rebooting once.
-
-- name: Configure SSH instance in management VRF
-  copy:
-    dest: /etc/ssh/
-    src: sshd_config.mgmt
-    mode: 0644
-  notify: reboot
-
-- name: Set up a SSH instance in management VRF
-  copy:
-    dest: /etc/systemd/system/
-    src: sshd@mgmt.service
-    mode: 0644
-  notify: reboot
-
-- name: Enable management SSH
-  service:
-    name: sshd@mgmt
-    enabled: yes
-  notify: reboot
-
-- meta: flush_handlers

roles/proxmox/tasks/user.yml

@@ -1,17 +1,19 @@
 # synchronize user and group data from LDAP when sync-ldap context key is set to a realm
-- block:
+- name: Set up LDAP user synchronization
+  when: '"sync-ldap" in hostvars[inventory_hostname]'
+  block:
     - name: Install LDAP sync script
       template:
         dest: /usr/local/bin/sync-ldap.py
         src: sync-ldap.py.j2
         mode: 0700
-      when: primary == inventory_hostname
+      when: is_primary
 
     - name: Remove LDAP sync script
       file:
         path: /usr/local/bin/sync-ldap.py
         state: absent
-      when: primary != inventory_hostname
+      when: not is_primary
 
     - name: Configure cronjob
       cron:
@@ -21,5 +23,4 @@
         cron_file: sync-ldap
         hour: "2"
         minute: "51"
-        state: '{{ "present" if inventory_hostname == primary else "absent" }}'
-  when: '"sync-ldap" in hostvars[inventory_hostname]'
+        state: '{{ "present" if is_primary else "absent" }}'

roles/proxmox/templates/cluster.fw.j2

@@ -1,5 +1,3 @@
-{% set services = query('netbox.netbox.nb_lookup', 'clusters', raw_data=true, api_filter='name='+cluster)
-        | map(attribute='custom_fields.services') | flatten -%}
 [OPTIONS]
 
 enable: 1
@@ -10,7 +8,7 @@ IN Ping(ACCEPT) -log nolog # don’t be rude
 IN SSH(ACCEPT) -i mgmt # for ansible etc.
 IN ACCEPT -source {{ nodes | map('device_address') | flatten | selectattr('family.value', '==', 4) | map(attribute='address') | join(',') }} # my cluster
 IN ACCEPT -source {{ nodes | map('device_address') | flatten | selectattr('family.value', '==', 6) | map(attribute='address') | join(',') }} # my cluster
-{% for service in services %}
+{% for service in cluster.custom_fields.services %}
 {% set prefixes = service | allowed_prefixes %}
 {% set prefixes4 = prefixes | selectattr('family.value', '==', 4) | map('string') %}
 {% set prefixes6 = prefixes | selectattr('family.value', '==', 6) | map('string') %}

roles/proxmox/templates/interfaces.j2

@@ -1,8 +1,6 @@
-{% set my_cluster = query('netbox.netbox.nb_lookup', 'clusters', raw_data=true,
-	api_filter='name='~cluster) | first -%}
-
-# bridges must be named vmbrN for proxmox to care
-{% for vlan in my_cluster.custom_fields.vlans | default([], true) | sort(attribute='vid') %}
+{# bridges must be named vmbrN for proxmox to care #}
+{% for vlan in cluster.custom_fields.vlans | default([], true) | sort(attribute='vid') %}
+# bridge and VNI for {{ vlan.name }}
 auto vmbr{{ vlan.vid }}
 iface vmbr{{ vlan.vid }}
     # {{ vlan.name }}

roles/proxmox/templates/loopback.intf.j2

@@ -0,0 +1,7 @@
+{% set lo = hostvars[inventory_hostname].interfaces | selectattr('name', '==', 'lo') | first %}
+{% set vxlan_ip = lo.ip_addresses | selectattr('family.value', '==', 4)
+    | map(attribute='address') | first
+    | ipaddr('address') %}
+iface lo
+    # without this the local tunnel IP is sometimes advertised as 0.0.0.0 on boot
+    vxlan-local-tunnelip {{ vxlan_ip }}

roles/proxmox/templates/real.intf.j2

@@ -1,27 +0,0 @@
-# Management VRF.
-auto mgmt
-iface mgmt
-    address 127.0.0.1/8
-    address ::1/128
-    vrf-table auto
-
-{% for iface in interfaces | rejectattr('mgmt_only') %}
-auto {{ iface.name }}
-iface {{ iface.name }} inet {% if iface.name == 'lo' %}loopback{% else %}static{% endif +%}
-{% if iface.mtu %}
-    mtu {{ iface.mtu }}
-{% endif %}
-{% if iface.vrf %}
-    vrf {{ iface.vrf.name }}
-{% endif %}
-{% for ip in iface.ip_addresses %}
-    address {{ ip.address }}
-{% endfor %}
-{% if iface.custom_fields.gateway %}
-    gateway {{ iface.custom_fields.gateway.address | ipaddr('address') }}
-{% endif %}
-{% if iface.name == 'lo' %}
-    vxlan-local-tunnelip {{ iface.ip_addresses | selectattr('family.value', '==', 4) | map(attribute='address') | sort | first | ipaddr('address') }}
-{% endif %}
-
-{% endfor %}

roles/proxmox/templates/sync-ldap.py.j2

@@ -6,7 +6,7 @@ import re
 
 import ldap3
 
-{% set password = lookup('passwordstore', "cluster/"+cluster, returnall=true) | from_yaml %}
+{% set password = lookup('passwordstore', "cluster/"+cluster.name, returnall=true) | from_yaml %}
 realm = '{{ hostvars[inventory_hostname]["sync-ldap"] }}'
 ldap_host = '{{ domain }}'
 ldap_user = '{{ password.ldap_user }}'

setup.yml

@@ -1,10 +1,18 @@
+- hosts: '*'
+  roles:
+    - facts
+  gather_facts: false
+
 - hosts: ceph-*
   roles:
     - debian
+    - frr
     - ceph
 
 - hosts: pve-rc-*, pve-lab-*
   roles:
+    - debian
+    - frr
     - proxmox
 
 - hosts: doku

templates/fabric.intf.j2

@@ -1,6 +0,0 @@
-{% for iface in interfaces | selectattr('name', 'match', '^lan') %}
-auto {{ iface.name }}
-iface {{ iface.name }}
-    mtu {{ iface.mtu | default('9216', true) }}
-
-{% endfor %}

templates/interfaces.j2

@@ -1,24 +0,0 @@
-# Management VRF and link.
-auto mgmt
-iface mgmt
-    address 127.0.0.1/8
-    address ::1/128
-    vrf-table auto
-
-{% for iface in hostvars[inventory_hostname].interfaces | selectattr('name', 'match', '^mgmt') | selectattr('ip_addresses') %}
-auto {{ iface.name }}
-iface {{ iface.name }}
-    vrf mgmt
-{% for ip in iface.ip_addresses %}
-    address {{ ip.address }}
-{% set subnet = ip.address | ipaddr('subnet') %}
-{% set prefix = query('netbox.netbox.nb_lookup', 'prefixes', api_filter=('prefix='+subnet))|first %}
-{% set gateway = prefix.value.custom_fields.gateway.address %}
-{% if gateway is defined %}
-    gateway {{ gateway | ipaddr('address') }}
-{% endif %}
-{% endfor %}
-
-{% endfor %}
-
-source /etc/network/interfaces.d/*

templates/loopback.intf.j2

@@ -1,6 +0,0 @@
-{% set iface_lo = hostvars[inventory_hostname].interfaces | selectattr('name', 'equalto', 'lo') | first %}
-auto lo
-iface lo inet loopback
-{% for ip in iface_lo.ip_addresses %}
-    address {{ ip.address }}
-{% endfor %}