Consolidate NAT and VPN settings into IP sets

I have tried every possible permutation and I think this is the one.

NetBox-managed IP prefixes are pushed with ansible to firewall master.
The managed prefixes are added to custom IP sets defined in the app,
but only NAT addresses and VPN groups can be configured for them.

This way all NAT and VPN policy is (again) configured in the app. Also
both NetBox-managed and user-defined networks are treated the same.

Also improve^Wtweak config generation. Also templates.

web/__init__.py

@@ -54,9 +54,6 @@ def create_app(test_config=None):
     from . import ipsets
     app.register_blueprint(ipsets.blueprint, url_prefix='/ipsets')
 
-    from . import nat
-    app.register_blueprint(nat.blueprint, url_prefix='/nat')
-
     from . import rules
     app.register_blueprint(rules.blueprint, url_prefix='/rules')
 

web/ipsets.py

@@ -15,18 +15,27 @@ def index():
         return flask.Response('forbidden', status=403, mimetype='text/plain')
 
     with db.locked():
-        ipsets = db.read('ipsets')
-        networks = db.read('networks')
         if flask.request.method == 'POST':
-            form = flask.request.form
+            # read network data from NetBox, merge in custom definitions and dump the lot
-            ipsets = {}
+            ipsets = db.read('networks')
-            for name, ip, ip6 in zip(form.getlist('name'), form.getlist('ip'), form.getlist('ip6')):
+            formdata = zip(*(flask.request.form.getlist(e) for e in ('name', 'ip', 'ip6', 'nat', 'vpn')))
-                if name and name not in networks:
+            for name, ip, ip6, nat, vpn in formdata:
-                    ipsets[name] = {
+                # drop sets with empty names
-                        'ip': ip.split(),
+                if not name:
-                        'ip6': ip6.split()
+                    continue
-                    }
+                # assign IPs for custom networks only
+                if name not in ipsets:
+                    ipsets[name] = { 'ip': ip.split(), 'ip6': ip6.split() }
+                # assign NAT and VPN for all networks
+                ipsets[name] |= { 'nat': nat, 'vpn': vpn }
             db.write('ipsets', ipsets)
             system.run(system.save_config)
             return flask.redirect(flask.url_for('ipsets.index'))
+
+        # read network data from NetBox and merge in custom definitions
+        ipsets = db.read('networks')
+        for name, data in db.read('ipsets').items():
+            # keep static IPs if there are any, otherwise set custom flag for this set
+            ipsets[name] = data | ipsets.get(name, {'custom': True})
+
         return flask.render_template('ipsets/index.html', ipsets=ipsets)

web/nat.py

@@ -1,26 +0,0 @@
-import flask
-import flask_login
-
-from . import db
-from . import system
-
-blueprint = flask.Blueprint('nat', __name__)
-
-@blueprint.route('/', methods=('GET', 'POST'))
-@flask_login.login_required
-def index():
-    if not flask_login.current_user.is_admin:
-        return flask.Response('forbidden', status=403, mimetype='text/plain')
-
-    with db.locked():
-        nat = { network: "" for network in db.read('networks') }
-        nat |= db.read('nat')
-        if flask.request.method == 'POST':
-            form = flask.request.form
-            for network, address in form.items():
-                if network in nat:
-                    nat[network] = address
-            db.write('nat', nat)
-            system.run(system.save_config)
-            return flask.redirect(flask.url_for('nat.index'))
-        return flask.render_template('nat/index.html', nat=nat)

web/rules.py

@@ -46,8 +46,6 @@ def edit(index):
 
         with db.locked():
             ipsets = db.read('ipsets')
-            for network, data in db.read('networks').items():
-                ipsets[network] = {'ip': data.get('ip', []), 'ip6': data.get('ip6', [])}
         return flask.render_template('rules/edit.html', index=index, rule=db.load('rules')[index], ipsets=ipsets)
     except IndexError as e:
         return flask.Response(f'invalid rule: {index}', status=400, mimetype='text/plain')

web/system.py

@@ -19,6 +19,10 @@ import ldap3
 
 from . import db
 
+def init_app(app):
+    app.cli.add_command(generate)
+    app.cli.add_command(push)
+
 def mail(rcpt, subject, body):
     try:
         msg = email.message.EmailMessage()
@@ -31,10 +35,6 @@ def mail(rcpt, subject, body):
     except Exception as e:
         syslog.syslog(f'error sending mail: {e}')
 
-def init_app(app):
-    app.cli.add_command(generate)
-    app.cli.add_command(push)
-
 def run(fun, args=()):
     def task():
         if os.fork() == 0:
@@ -42,73 +42,52 @@ def run(fun, args=()):
             fun(*args)
     multiprocessing.Process(target=task).start()
 
-def ipset_add(ipsets, name, ip=None, ip6=None):
+# Generate configuration files and create a config tarball.
-    ipsets[name].update(ip or ())
-    ipsets[f'{name}/6'].update(ip6 or ())
-
 def save_config():
-    # Format strings for creating firewall config files.
-    nft_set = 'set {name} {{\n    type ipv{family}_addr; flags interval; {elements}\n}}\n\n'
-    nft_map = 'map {name} {{\n    type ipv4_addr : interval ipv4_addr; flags interval; {elements}\n}}\n\n'
-    nft_forward = '# {index}. {name}\n{text}\n\n'
-    wg_intf = '[Interface]\nListenPort = {port}\nPrivateKey = {key}\n\n'
-    wg_peer = '# {user}\n[Peer]\nPublicKey = {key}\nAllowedIPs = {ips}\n\n'
-
     output = None
     try:
-        # Just load the settings here but keep the database unlocked
+        # Just load required settings here but keep the database unlocked
         # while we load group memberships from LDAP.
         with db.locked():
+            ipsets = db.read('ipsets')
             settings = db.read('settings')
-            groups = db.read('groups')
 
-        # For each user build a list of networks they have access to, based on
+        # Build LDAP query for users and groups.
-        # group membership in AD. Only query groups associated with at least one
+        filters = [
-        # network, and query each group only once.
-        user_networks = collections.defaultdict(set)
-        ldap = ldap3.Connection(ldap3.Server(settings.get('ldap_host'), use_ssl=True),
-                settings.get('ldap_user'), settings.get('ldap_pass'), auto_bind=True)
-
-        # All of these must match to consider an LDAP object.
-        ldap_query = [
             '(objectClass=user)', # only users
             '(objectCategory=person)', # that are people
             '(!(userAccountControl:1.2.840.113556.1.4.803:=2))', # with enabled accounts
         ]
         if group := settings.get('user_group'):
-            ldap_query += [f'(memberOf:1.2.840.113556.1.4.1941:={group})'] # in given group, recursively
+            filters += [f'(memberOf:1.2.840.113556.1.4.1941:={group})'] # in given group, recursively
 
+        # Run query and store group membership data.
+        server = ldap3.Server(settings['ldap_host'], use_ssl=True)
+        ldap = ldap3.Connection(server, settings['ldap_user'], settings['ldap_pass'], auto_bind=True)
         ldap.search(settings.get('ldap_base_dn', ''),
-                    f'(&{"".join(ldap_query)})', # conjuction (&(…)(…)(…)) of queries
+                f'(&{"".join(filters)})', # conjuction (&(…)(…)(…)) of queries
-                    attributes=['userPrincipalName', 'memberOf'])
+                attributes=['userPrincipalName', 'memberOf'])
-        for entry in ldap.entries:
+        user_groups = { e.userPrincipalName.value: set(e.memberOf) for e in ldap.entries }
-            for group in entry.memberOf:
-                if group in groups:
-                    user_networks[entry.userPrincipalName.value].add(groups[group])
 
-        # Now read the settings again and lock the database while generating
+        # Now read the settings again while keeping the database locked until
-        # config files, then increment version before unlocking.
+        # config files are generated, and increment version before unlocking.
         with db.locked():
+            ipsets = db.read('ipsets')
+            wireguard = db.read('wireguard')
             settings = db.read('settings')
             version = settings['version'] = int(settings.get('version') or '0') + 1
 
-            # Populate IP sets.
+            # Update IP sets with VPN addresses based on AD group membership.
-            ipsets = collections.defaultdict(set)
+            vpn_groups = set([e['vpn'] for e in ipsets.values() if e.get('vpn')])
-            # Sets corresponding to VLANs in NetBox. Prefixes for these sets are configured on firewall nodes with ansible.
+            group_networks = {
-            for name, network in db.read('networks').items():
+                group: [name for name, data in ipsets.items() if data['vpn'] == group] for group in vpn_groups
-                ipset_add(ipsets, name)
+            }
-            # Sets defined by user in friwall app.
-            for name, network in db.read('ipsets').items():
-                ipset_add(ipsets, name, network.get('ip'), network.get('ip6'))
-
-            # Add registered VPN addresses for each network based on
-            # LDAP group membership.
-            wireguard = db.read('wireguard')
             for ip, key in wireguard.items():
-                ip4 = [f'{ip}/32']
+                for group in user_groups.get(key.get('user', ''), ()):
-                ip6 = [f'{key["ip6"]}'] if key.get('ip6') else None
+                    for network in group_networks.get(group, ()):
-                for network in user_networks.get(key.get('user', ''), ()):
+                        ipsets[network]['ip'].append(f'{ip}/32')
-                    ipset_add(ipsets, network, ip4, ip6)
+                        if ip6 := key.get('ip6'):
+                            ipsets[network]['ip6'].append(ip6)
 
             # Create config files.
             output = pathlib.Path.home() / 'config' / f'{version}'
@@ -122,51 +101,69 @@ def save_config():
 
             # Print nftables sets.
             with open(output / 'etc/nftables.d/sets.nft', 'w', encoding='utf-8') as f:
-                for name, ips in ipsets.items():
+                nft_set = 'set {name} {{\n    type ipv4_addr; flags interval; {ips}\n}}\n'
-                    f.write(nft_set.format(
+                nft_set6 = 'set {name}/6 {{\n    type ipv6_addr; flags interval; {ips}\n}}\n'
-                        name=name,
+                def make_set(ips):
-                        family='6' if name.endswith('/6') else '4',
+                    # return "elements = { ip1, ip2, … }", prefixed with "# " if no ips
-                        elements=f'{"" if ips else "# "}elements = {{ {", ".join(ips)} }}'))
+                    return f'{"" if ips else "# "}elements = {{ {", ".join(ips)} }}'
+                for name, data in ipsets.items():
+                    f.write(nft_set.format(name=name, ips=make_set(data.get('ip', ()))))
+                    f.write(nft_set6.format(name=name, ips=make_set(data.get('ip6', ()))))
+                    f.write('\n')
 
             # Print static NAT (1:1) rules.
             with open(output / 'etc/nftables.d/netmap.nft', 'w', encoding='utf-8') as f:
-                netmap = db.read('netmap') # { private range: public range… }
+                nft_map = 'map {name} {{\n    type ipv4_addr : interval ipv4_addr; flags interval; elements = {{\n{ips}\n    }}\n}}\n'
-                if netmap:
+                def make_map(ips, reverse=False):
-                    f.write(nft_map.format(
+                    # return "{ from1: to1, from2: to2, … }" with possibly reversed from and to
-                        name='netmap-out',
+                    return ',\n'.join(f"{b if reverse else a}: {a if reverse else b}" for a, b in ips)
-                        elements='elements = {' + ',\n'.join(f'{a}: {b}' for a, b in netmap.items()) + '}'))
+                if netmap := db.read('netmap'): # { private range: public range… }
-                    f.write(nft_map.format(
+                    f.write(nft_map.format(name='netmap-out', ips=make_map(netmap.items())))
-                        name='netmap-in',
+                    f.write('\n')
-                        elements='elements = {' + ',\n'.join(f'{b}: {a}' for a, b in netmap.items()) + '}'))
+                    f.write(nft_map.format(name='netmap-in', ips=make_map(netmap.items(), reverse=True)))
 
             # Print dynamic NAT rules.
             with open(output / 'etc/nftables.d/nat.nft', 'w', encoding='utf-8') as f:
-                nat = db.read('nat') # { network name: public range… }
+                nft_nat = 'iif @inside oif @outside ip saddr @{name} snat to {nat}\n'
-                for network, address in nat.items():
+                for name, data in ipsets.items():
-                    if address:
+                    if nat := data.get('nat'):
-                        f.write(f'iif @inside oif @outside ip saddr @{network} snat to {address}\n')
+                        f.write(nft_nat.format(name=name, nat=nat))
 
             # Print forwarding rules.
             with open(output / 'etc/nftables.d/forward.nft', 'w', encoding='utf-8') as f:
+                # Forwarding rules for VPN users.
+                if vpn_networks := sorted(name for name, data in ipsets.items() if data.get('vpn')):
+                    nft_forward = 'iif @inside oif @inside ip saddr @{name} ip daddr @{name} accept\n'
+                    f.write('# forward from the VPN interface to physical networks and back\n')
+                    for name in vpn_networks:
+                        f.write(nft_forward.format(name=name))
+                    for name in vpn_networks:
+                        f.write(nft_forward.format(name=f'{name}/6'))
+                    f.write('\n')
+
+                # Custom forwarding rules.
+                nft_rule = '# {index}. {name}\n{text}\n\n'
                 for index, rule in enumerate(db.read('rules')):
                     if rule.get('enabled') and rule.get('text'):
-                        f.write(nft_forward.format(index=index, name=rule.get('name', ''), text=rule['text']))
+                        f.write(nft_rule.format(index=index, name=rule.get('name', ''), text=rule['text']))
 
             # Print wireguard config.
             with open(output / 'etc/wireguard/wg.conf', 'w', encoding='utf-8') as f:
-                f.write(wg_intf.format(
+                # Server configuration.
-                    port=settings.get('wg_port') or 51820,
+                wg_intf = '[Interface]\nListenPort = {port}\nPrivateKey = {key}\n\n'
-                    key=settings.get('wg_key')))
+                f.write(wg_intf.format(port=settings.get('wg_port') or 51820, key=settings.get('wg_key')))
+
+                # Client configuration.
+                wg_peer = '# {user}\n[Peer]\nPublicKey = {key}\nAllowedIPs = {ips}\n\n'
                 for ip, data in wireguard.items():
                     f.write(wg_peer.format(
                         user=data.get('user'),
                         key=data.get('key'),
                         ips=', '.join(filter(None, [ip, data.get('ip6')]))))
 
-            # Make a config archive in a temporary place, so we don’t send incomplete tars.
+            # Make a temporary config archive and move it to the final location,
+            # so we avoid sending incomplete tars.
             tar_file = shutil.make_archive(f'{output}-tmp', 'gztar', root_dir=output, owner='root', group='root')
-
-            # Move config archive to the final destination.
             os.rename(tar_file, f'{output}.tar.gz')
 
             # If we get here, write settings with the new version.

web/templates/base.html

@@ -27,6 +27,9 @@ h1 > a {
     color: unset;
     text-decoration: none;
 }
+input:read-only {
+    border-style: dotted;
+}
 pre {
     background-color: #eeeeee;
     border: 1px solid black;
@@ -39,6 +42,9 @@ th {
 th, td {
     padding-right: 1em;
 }
+th {
+    border-bottom: 1px solid black;
+}
 ul.keys {
     margin: 0 0.5em 0.5em;
     padding-left: 1em;
@@ -48,6 +54,8 @@ ul.keys a {
 }
 </style>
 
+{% block header %}{% endblock %}
+
 <title>FRIwall</title>
 
 {% if current_user.is_authenticated %}

web/templates/index.html

@@ -14,18 +14,14 @@
 <dl>
 <dt><a href="{{ url_for('nodes') }}">Status</a>
 <dd>status opek v požarnem zidu
-<dt><a href="{{ url_for('config.index') }}">Nastavitve</a>
+<dt><a href="{{ url_for('ipsets.index') }}">Omrežja</a>
-<dd>nastavitve aplikacije FRIwall
+<dd>območja IP, naslovi NAT in skupine za VPN
-<dt><a href="{{ url_for('ipsets.index') }}">Območja IP</a>
-<dd>definicije območij IP
 <dt><a href="{{ url_for('rules.index') }}">Urejanje pravil</a>
 <dd>pravila za posredovanje prometa
-<dt><a href="{{ url_for('nat.index') }}">NAT</a>
-<dd>javni naslovi za pisarniška omrežja
 <dt><a href="{{ url_for('config.edit', name='netmap') }}">Netmap</a>
 <dd>statične 1:1 preslikave naslovov za strežniška omrežja
-<dt><a href="{{ url_for('config.edit', name='groups') }}">Skupine</a>
+<dt><a href="{{ url_for('config.index') }}">Nastavitve</a>
-<dd>preslikave uporabnikov LDAP v pisarniška omrežja
+<dd>nastavitve aplikacije FRIwall
 </dl>
 </section>
 {% endif %}

web/templates/ipsets/index.html

@@ -1,25 +1,38 @@
 {% extends 'base.html' %}
+{% block header %}
+<style>
+td > input {
+    width: 100%;
+}
+</style>
+{% endblock %}
 
 {% block content %}
 <p>
-Urejate območja IP. Za vsako območje lahko dodate enega ali več obsegov IP in/ali IPv6, ločenih s presledki.
+Urejate naslovna območja. Za statična omrežja lahko določimo naslov NAT in skupino za VPN. Za lastna območja lahko poleg tega definiramo enega ali več obsegov IP, ločenih s presledki.
+
+<p>
+NAT se izvaja na območjih, kjer je nastavljen. Uporabniki VPN imajo glede na skupine v AD enake dostope kot območja, za katera so nastavljene te skupine.
 
 <form id="request" method="POST">
-<table>
+<table style="width: 100%;">
 <thead>
-<th>Ime<th>IP<th>IPv6
+<th>Ime<th>IP<th>IPv6<th>NAT<th>VPN
 <tbody>
-<tbody>
+{% for name, data in ipsets.items() %}
-{% for name, addresses in ipsets.items() %}
 <tr>
-<td><input name="name" value="{{ name }}" />
+<td style="max-width: 4em;"><input name="name" value="{{ name }}" {% if not data.custom %}readonly{% endif %} />
-<td><input name="ip" value="{{ addresses.ip|join(' ') }}" />
+<td style="max-width: 5em;"><input name="ip" value="{{ data.ip|join(' ') }}" {% if not data.custom %}readonly{% endif %} />
-<td><input name="ip6" value="{{ addresses.ip6|join(' ') }}" />
+<td style="max-width: 8em;"><input name="ip6" value="{{ data.ip6|join(' ') }}" {% if not data.custom %}readonly{% endif %} />
+<td style="max-width: 5em;"><input name="nat" value="{{ data.nat }}" />
+<td style=""><input name="vpn" value="{{ data.vpn }}" />
 {% endfor %}
 <tr>
-<td><input name="name" />
+<td style="max-width: 4em;"><input name="name" />
-<td><input name="ip" />
+<td style="max-width: 5em;"><input name="ip" />
-<td><input name="ip6" />
+<td style="max-width: 8em;"><input name="ip6" />
+<td style="max-width: 5em;"><input name="nat" />
+<td><input name="vpn" />
 </table>
 <p><button id="submit" type="submit">Shrani</button>
 </form>

web/templates/nat/index.html

@@ -1,19 +0,0 @@
-{% extends 'base.html' %}
-
-{% block content %}
-<p>
-Urejate naslove NAT za pisarniška omrežja.
-
-<form id="request" method="POST">
-<table>
-<tbody>
-{% for office, address in nat.items() %}
-<tr>
-<td><label for="{{ office }}">{{ office }}</label>
-<td><input id="{{ office }}" name="{{ office }}" value="{{ address }}" />
-{% endfor %}
-</table>
-<p><button id="submit" type="submit">Shrani</button>
-</form>
-
-{% endblock %}

web/templates/rules/edit.html

@@ -1,8 +1,18 @@
 {% extends 'base.html' %}
+{% block header %}
+<style>
+tbody > tr:nth-child(odd) {
+    background-color: #eeeeee;
+}
+td {
+    vertical-align: top;
+}
+</style>
+{% endblock %}
 
 {% block content %}
 <p>
-Urejate pravilo #{{ index }}. V pravilih lahko uporabljate imena območij IP, prikazana spodaj. <a href="{{ url_for('rules.index') }}">Seznam pravil.</a>
+Urejate pravilo #{{ index }}. <a href="{{ url_for('rules.index') }}">Seznam pravil.</a>
 
 <form id="request" method="POST">
 <p>
@@ -18,19 +28,28 @@ Uporabniki, ki lahko o(ne)mogočijo pravilo<br>
 
 <p>
 <label for="text">Pravila nftables</label>
-<textarea id="text" name="text" style="width: 100%; height: 20em;">{{ rule.text }}</textarea>
+<textarea id="text" name="text" style="width: 100%; height: 20em;" placeholder="iif @inside ip saddr @from ip daddr @to accept
iif @inside ip6 saddr @from/6 ip6 daddr @to/6 accept">
+{{- rule.text }}
+</textarea>
 <p><button id="submit" type="submit">Shrani</button>
 </form>
 
-<table>
+<p>
+V pravilih lahko uporabljamo spodnja območja IP, npr. <code>@pr5</code> in <code>@pr5/6</code> za območji IPv4 in IPv6 učilnice 5. Za notranja omrežja uporabimo vmesnik <code>@inside</code>, za zunanja pa vmesnik <code>@outside</code>. Primere z razlago najdemo v <a href="https://wiki.nftables.org">dokumentaciji nftables</a>.
+
+<table style="width: 100%;">
 <thead>
-<th>Območje<th>IP<th>IPv6
+<th>Omrežje
+<th>IP
+<th>IPv6
+<th>VPN
 <tbody>
-{% for network, addresses in ipsets.items() %}
+{% for name, data in ipsets.items() %}
 <tr>
-<td>{{ network }}
+<td>{{ name }}
-<td>{{ addresses.ip|join('<br>')|safe }}
+<td>{{ data.ip|join('<br>')|safe }}
-<td>{{ addresses.ip6|join('<br>')|safe }}
+<td>{{ data.ip6|join('<br>')|safe }}
+<td>{{ data.vpn }}
 {% endfor %}
 </table>