From 0c1cc14e01009a79d143eb15843741da39923519 Mon Sep 17 00:00:00 2001
From: Timotej Lazar <timotej.lazar@fri.uni-lj.si>
Date: Wed, 18 Oct 2023 15:01:02 +0200
Subject: [PATCH] proxmox: add initial support for L2 VXLAN

I heard we like L2 so I put some L2 in our L3 so we can L2 as we L3 on L2.
---
 roles/proxmox/handlers/main.yml       |  3 +++
 roles/proxmox/tasks/main.yml          | 14 +++++++++++
 roles/proxmox/tasks/mgmt.yml          | 13 +++-------
 roles/proxmox/tasks/sdn.yml           | 13 ++++++++++
 roles/proxmox/templates/frr.conf.j2   | 36 +++++++++++++++++++++++++++
 roles/proxmox/templates/interfaces.j2 | 30 ++++++++++++++++++++++
 roles/proxmox/templates/mgmt.intf.j2  | 23 +++++++++++++++++
 setup.yml                             |  4 +++
 8 files changed, 126 insertions(+), 10 deletions(-)
 create mode 100644 roles/proxmox/templates/frr.conf.j2
 create mode 100644 roles/proxmox/templates/interfaces.j2
 create mode 100644 roles/proxmox/templates/mgmt.intf.j2

diff --git a/roles/proxmox/handlers/main.yml b/roles/proxmox/handlers/main.yml
index fce8e53..462d86f 100644
--- a/roles/proxmox/handlers/main.yml
+++ b/roles/proxmox/handlers/main.yml
@@ -3,3 +3,6 @@
 
 - name: reload interfaces
   command: ifreload -a
+
+- name: reload frr
+  command: /usr/lib/frr/frr-reload.py --reload /etc/frr/frr.conf
diff --git a/roles/proxmox/tasks/main.yml b/roles/proxmox/tasks/main.yml
index 0aec069..9b14992 100644
--- a/roles/proxmox/tasks/main.yml
+++ b/roles/proxmox/tasks/main.yml
@@ -13,6 +13,13 @@
   apt_repository:
     repo: 'deb http://download.proxmox.com/debian/pve {{ ansible_distribution_release }} pve-no-subscription'
 
+- name: Add rules to rename network interfaces
+  template:
+    dest: /etc/udev/rules.d/10-network.rules
+    src: 10-network.rules.j2
+    mode: 0644
+  notify: reboot
+
 - name: Set up loopback interface
   template:
     dest: /etc/network/interfaces.d/loopback.intf
@@ -27,6 +34,13 @@
     mode: 0644
   notify: reload interfaces
 
+- name: Set up interfaces
+  template:
+    dest: /etc/network/interfaces
+    src: interfaces.j2
+    mode: 0644
+  notify: reload interfaces
+
 - include_tasks: mgmt.yml
 
 - include_tasks: sdn.yml
diff --git a/roles/proxmox/tasks/mgmt.yml b/roles/proxmox/tasks/mgmt.yml
index 10b079c..672b479 100644
--- a/roles/proxmox/tasks/mgmt.yml
+++ b/roles/proxmox/tasks/mgmt.yml
@@ -1,17 +1,10 @@
 # We could probably avoid rebooting in some cases, but those should never happen
 # in normal operation anyway. This way all setup is done before rebooting once.
 
-- name: Add rules to rename network interfaces
+- name: Set up management interfaces
   template:
-    dest: /etc/udev/rules.d/10-network.rules
-    src: 10-network.rules.j2
-    mode: 0644
-  notify: reboot
-
-- name: Set up interfaces
-  template:
-    dest: /etc/network/interfaces
-    src: interfaces.j2
+    dest: /etc/network/interfaces.d/mgmt.intf
+    src: mgmt.intf.j2
     mode: 0644
   notify: reboot
 
diff --git a/roles/proxmox/tasks/sdn.yml b/roles/proxmox/tasks/sdn.yml
index a88c4b2..cd35e30 100644
--- a/roles/proxmox/tasks/sdn.yml
+++ b/roles/proxmox/tasks/sdn.yml
@@ -1,3 +1,16 @@
 - name: Install packages for SDN
   package:
     name: libpve-network-perl
+
+- name: Copy FRR config
+  template:
+    dest: /etc/frr/frr.conf
+    src: frr.conf.j2
+    mode: 0644
+  notify: reload frr
+
+- name: Enable FRR service
+  service:
+    name: frr
+    enabled: yes
+    state: started
diff --git a/roles/proxmox/templates/frr.conf.j2 b/roles/proxmox/templates/frr.conf.j2
new file mode 100644
index 0000000..89ad8c1
--- /dev/null
+++ b/roles/proxmox/templates/frr.conf.j2
@@ -0,0 +1,36 @@
+frr defaults datacenter
+service integrated-vtysh-config
+log syslog
+
+# We only have the default route, so allow talking to BGP peers over it.
+ip nht resolve-via-default
+
+router bgp {{ hostvars[inventory_hostname].custom_fields.asn.asn }}
+  bgp bestpath as-path multipath-relax
+
+  neighbor fabric peer-group
+  neighbor fabric remote-as external
+  neighbor fabric capability extended-nexthop
+
+{% for iface in hostvars[inventory_hostname].interfaces | selectattr('name', 'match', '^lan') %}
+  neighbor {{ iface.name }} interface peer-group fabric
+  neighbor {{ iface.name }} bfd
+{% endfor %}
+
+  address-family ipv4 unicast
+    redistribute connected route-map loopback
+    neighbor fabric activate
+  exit-address-family
+
+  address-family ipv6 unicast
+    redistribute connected route-map loopback
+    neighbor fabric activate
+  exit-address-family
+
+  address-family l2vpn evpn
+    neighbor fabric activate
+    advertise-all-vni
+  exit-address-family
+
+route-map loopback permit 1
+  match interface lo
diff --git a/roles/proxmox/templates/interfaces.j2 b/roles/proxmox/templates/interfaces.j2
new file mode 100644
index 0000000..902e142
--- /dev/null
+++ b/roles/proxmox/templates/interfaces.j2
@@ -0,0 +1,30 @@
+# Keep vmbr0 named as is and in the main interfaces file so Proxmox can find it.
+
+# Bridge for V(X)LANs.
+auto vmbr0
+iface vmbr0 inet manual
+    bridge-vlan-aware yes
+    bridge-ports {{ vlans | map('regex_replace', '^', 'vni') | join(' ') }}
+    bridge-stp off
+    bridge-fd 0
+
+# Interfaces.
+{% for vlan in vlans %}
+auto vni{{ vlan }}
+iface vni{{ vlan }} inet static
+    vxlan-id {{ vlan }}
+    bridge-access {{ vlan }}
+    mstpctl-bpduguard yes
+    mstpctl-portbpdufilter yes
+
+{% endfor %}
+
+# In place of vni* interfaces above this should work also but does not.
+# Might start working after proxmox upgrades their ifupdown2.
+#auto vxlan
+#iface vxlan inet static
+#    bridge-vlan-vni-map {{ vlans | zip(vlans) | map('join', '=') | join(' ') }}
+#    bridge-vids {{ vlans | join(' ') }}
+#    bridge-learning off
+
+source /etc/network/interfaces.d/*
diff --git a/roles/proxmox/templates/mgmt.intf.j2 b/roles/proxmox/templates/mgmt.intf.j2
new file mode 100644
index 0000000..e15567d
--- /dev/null
+++ b/roles/proxmox/templates/mgmt.intf.j2
@@ -0,0 +1,23 @@
+# Management VRF and link.
+auto mgmt
+iface mgmt
+    address 127.0.0.1/8
+    address ::1/128
+    vrf-table auto
+
+{% for iface in hostvars[inventory_hostname].interfaces | selectattr('name', 'match', '^mgmt') | selectattr('ip_addresses') %}
+auto {{ iface.name }}
+iface {{ iface.name }}
+    vrf mgmt
+{% for ip in iface.ip_addresses %}
+    address {{ ip.address }}
+{% set subnet = ip.address | ipaddr('subnet') %}
+{% set prefix = query('netbox.netbox.nb_lookup', 'prefixes', api_filter=('prefix='+subnet))|first %}
+{% set gateway = prefix.value.custom_fields.gateway.address %}
+{% if gateway is defined %}
+    gateway {{ gateway | ipaddr('address') }}
+{% endif %}
+{% endfor %}
+
+{% endfor %}
+
diff --git a/setup.yml b/setup.yml
index 2e64e19..10588c9 100644
--- a/setup.yml
+++ b/setup.yml
@@ -1,3 +1,7 @@
 - hosts: ceph-*
   roles:
     - debian
+
+- hosts: proxmox-rc-next-*
+  roles:
+    - proxmox