From 048195c45cb509595475853013357bbbef7ce98a Mon Sep 17 00:00:00 2001 From: Timotej Lazar Date: Wed, 14 Aug 2024 11:25:07 +0200 Subject: [PATCH] Always combine IP set data with static network definitions from NetBox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before we relied on the combined data being present in ipsets.json when generating a new config, but ipsets.json is only updated through the form at /ipsets. So submitting any other form after changing NetBox definitions might crash when trying to find an entry from networks.json in ipsets.json. Now we introduce a helper functon to always read both files and combine the prefixes fron networks.json with ipsets.json. This way it is not necessary to save a new ipsets.json before other changes. Also don’t crash when enumerating networks for each VPN group. --- README.md | 10 +++++++++- web/ipsets.py | 31 +++++++++++++++++++------------ web/rules.py | 4 ++-- web/system.py | 18 +++++++++--------- web/vpn.py | 4 ++-- 5 files changed, 41 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 41dad29..622aa13 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,17 @@ The firewall consists of two servers (or “bricks”) in active–backup config On each configuration change, a tarball of relevant files in `/etc` is generated and pushed via SSH to both nodes. This happens for instance each time an IP set or a forwarding rule is modified, or a VPN key is added or removed. Firewall nodes and the current configuration version for each are stored in `nodes.json`. +## Database + +Application data is stored in a number of JSON files in the home directory of the user the application runs as. The `db` module defines utility functions to ensure consistency when manipulating data: + + - `lock(name)` and `unlock(name)` acquire or release the lock for a given file or all files with no argument; + - `read(name)` and `write(name, data)` retrieve or store a dictionary in the given file, which should be locked; + - `load(name)` and `save(name, data)` do the same but lock the file first. + ## IP sets -Names and IP prefixes for physical networks are configured in NetBox and stored in `networks.json`, which is never modified by the application. Custom IP sets used for forwarding rules may be defined at `/ipsets`, as well as NAT addresse and VPN access for all networks. These settings are stored in `ipsets.json` and added to static definitions in `networks.json` when generating firewall configuration. +Names and IP prefixes for physical networks are configured in NetBox and stored in `networks.json`, which is never modified by the application. Custom IP sets used for forwarding rules may be defined at `/ipsets`, as well as NAT addresse and VPN access for all networks. These settings are stored in `ipsets.json`. To read and combine data from both files, the `ipsets.read` utility function should be used. Combined data may then be modified and written back to `ipsets.json`, as with all other files. ## Rules diff --git a/web/ipsets.py b/web/ipsets.py index 203a78e..e641c2f 100644 --- a/web/ipsets.py +++ b/web/ipsets.py @@ -8,6 +8,19 @@ from . import system blueprint = flask.Blueprint('ipsets', __name__) +# return combined data for all IP sets, stored in two files: +# - networks.json: IP prefixes for networks defined in NetBox +# - ipsets.json: custom IP prefixes defined with this app, and NAT and other settings for all sets +# database should be locked when calling this function +def read(): + # first read in static data + all_ipsets = db.read('networks') + # then add the custom definitions, not overriding any IP prefixes from NetBox + for name, data in db.read('ipsets').items(): + # set “custom” flag for this set if it is not present in networks.json + all_ipsets[name] = data | all_ipsets.get(name, {'custom': True}) + return all_ipsets + @blueprint.route('/', methods=('GET', 'POST')) @flask_login.login_required def index(): @@ -17,25 +30,19 @@ def index(): with db.locked(): if flask.request.method == 'POST': # read network data from NetBox, merge in custom definitions and dump the lot - ipsets = db.read('networks') + sets = db.read('networks') formdata = zip(*(flask.request.form.getlist(e) for e in ('name', 'ip', 'ip6', 'nat', 'vpn'))) for name, ip, ip6, nat, vpn in formdata: # drop sets with empty names if not name: continue # assign IPs for custom networks only - if name not in ipsets: - ipsets[name] = { 'ip': ip.split(), 'ip6': ip6.split() } + if name not in sets: + sets[name] = { 'ip': ip.split(), 'ip6': ip6.split() } # assign NAT and VPN for all networks - ipsets[name] |= { 'nat': nat, 'vpn': vpn } - db.write('ipsets', ipsets) + sets[name] |= { 'nat': nat, 'vpn': vpn } + db.write('ipsets', sets) system.run(system.save_config) return flask.redirect(flask.url_for('ipsets.index')) - # read network data from NetBox and merge in custom definitions - ipsets = db.read('networks') - for name, data in db.read('ipsets').items(): - # keep static IPs if there are any, otherwise set custom flag for this set - ipsets[name] = data | ipsets.get(name, {'custom': True}) - - return flask.render_template('ipsets/index.html', ipsets=ipsets) + return flask.render_template('ipsets/index.html', ipsets=read()) diff --git a/web/rules.py b/web/rules.py index 349dd5f..dc26bcc 100644 --- a/web/rules.py +++ b/web/rules.py @@ -2,6 +2,7 @@ import flask import flask_login from . import db +from . import ipsets from . import system blueprint = flask.Blueprint('rules', __name__) @@ -45,8 +46,7 @@ def edit(index): system.run(system.save_config) with db.locked(): - ipsets = db.read('ipsets') - return flask.render_template('rules/edit.html', index=index, rule=db.load('rules')[index], ipsets=ipsets) + return flask.render_template('rules/edit.html', index=index, rule=db.load('rules')[index], ipsets=ipsets.read()) except IndexError as e: return flask.Response(f'invalid rule: {index}', status=400, mimetype='text/plain') diff --git a/web/system.py b/web/system.py index 1bde640..3122967 100644 --- a/web/system.py +++ b/web/system.py @@ -18,6 +18,7 @@ import flask.cli import ldap3 from . import db +from . import ipsets def init_app(app): app.cli.add_command(generate) @@ -49,7 +50,6 @@ def save_config(): # Just load required settings here but keep the database unlocked # while we load group memberships from LDAP. with db.locked(): - ipsets = db.read('ipsets') settings = db.read('settings') # Build LDAP query for users and groups. @@ -72,15 +72,15 @@ def save_config(): # Now read the settings again while keeping the database locked until # config files are generated, and increment version before unlocking. with db.locked(): - ipsets = db.read('ipsets') + sets = ipsets.read() wireguard = db.read('wireguard') settings = db.read('settings') version = settings['version'] = int(settings.get('version') or '0') + 1 # Find networks accessible to VPN users for each AD group. - vpn_groups = {e['vpn'] for e in ipsets.values() if e.get('vpn')} + vpn_groups = {e['vpn'] for e in sets.values() if e.get('vpn')} group_networks = { - group: [name for name, data in ipsets.items() if data['vpn'] == group] for group in vpn_groups + group: [name for name, data in sets.items() if data.get('vpn') == group] for group in vpn_groups } # Add VPN addresses to IP sets. @@ -92,9 +92,9 @@ def save_config(): for group in user_groups.get(key.get('user', ''), ()): key_networks |= set(group_networks.get(group, ())) for network in key_networks: - ipsets[network]['ip'].append(f'{ip}/32') + sets[network]['ip'].append(f'{ip}/32') if ip6 := key.get('ip6'): - ipsets[network]['ip6'].append(ip6) + sets[network]['ip6'].append(ip6) # Create config files. output = pathlib.Path.home() / 'config' / f'{version}' @@ -113,7 +113,7 @@ def save_config(): def make_set(ips): # return "elements = { ip1, ip2, … }", prefixed with "# " if no ips return f'{"" if ips else "# "}elements = {{ {", ".join(ips)} }}' - for name, data in ipsets.items(): + for name, data in sets.items(): f.write(nft_set.format(name=name, ips=make_set(data.get('ip', ())))) f.write(nft_set6.format(name=name, ips=make_set(data.get('ip6', ())))) f.write('\n') @@ -132,14 +132,14 @@ def save_config(): # Print dynamic NAT rules. with open(output / 'etc/nftables.d/nat.nft', 'w', encoding='utf-8') as f: nft_nat = 'iif @inside oif @outside ip saddr @{name} snat to {nat}\n' - for name, data in ipsets.items(): + for name, data in sets.items(): if nat := data.get('nat'): f.write(nft_nat.format(name=name, nat=nat)) # Print forwarding rules. with open(output / 'etc/nftables.d/forward.nft', 'w', encoding='utf-8') as f: # Forwarding rules for VPN users. - if vpn_networks := sorted(name for name, data in ipsets.items() if data.get('vpn')): + if vpn_networks := sorted(name for name, data in sets.items() if data.get('vpn')): nft_forward = 'iif @inside oif @inside ip saddr @{name} ip daddr @{name} accept\n' nft_forward6 = 'iif @inside oif @inside ip6 saddr @{name}/6 ip6 daddr @{name}/6 accept\n' f.write('# forward from the VPN interface to physical networks and back\n') diff --git a/web/vpn.py b/web/vpn.py index 876fc50..0b77e45 100644 --- a/web/vpn.py +++ b/web/vpn.py @@ -8,6 +8,7 @@ import flask import flask_login from . import db +from . import ipsets from . import system blueprint = flask.Blueprint('vpn', __name__) @@ -25,8 +26,7 @@ def custom(): return flask.Response('forbidden', status=403, mimetype='text/plain') with db.locked(): keys = {ip: data for ip, data in db.read('wireguard').items() if data.get('networks') and not data.get('user')} - ipsets = db.read('networks') | db.read('ipsets') - return flask.render_template('vpn/custom.html', keys=keys, ipsets=ipsets.keys()) + return flask.render_template('vpn/custom.html', keys=keys, ipsets=ipsets.read().keys()) @blueprint.route('/list') @flask_login.login_required