From 763b810ca1a9d755205e49b1246025b83abb5132 Mon Sep 17 00:00:00 2001 From: Mike Crute Date: Wed, 27 Jan 2021 04:41:00 +0000 Subject: Add netbox --- netbox/Dockerfile | 73 +++++++++++++++++++++++++++++ netbox/Makefile | 24 ++++++++++ netbox/config-patch1.diff | 26 +++++++++++ netbox/config-patch2.diff | 89 ++++++++++++++++++++++++++++++++++++ netbox/django-driver.py | 69 ++++++++++++++++++++++++++++ netbox/django-vault-client.py | 84 ++++++++++++++++++++++++++++++++++ netbox/entrypoint.sh | 22 +++++++++ netbox/etc/service/netbox-rq/log/run | 3 ++ netbox/etc/service/netbox-rq/run | 23 ++++++++++ netbox/etc/service/uwsgi/log/run | 3 ++ netbox/etc/service/uwsgi/run | 23 ++++++++++ netbox/etc/uwsgi/netbox.ini | 15 ++++++ netbox/settings-patch.diff | 11 +++++ 13 files changed, 465 insertions(+) create mode 100644 netbox/Dockerfile create mode 100644 netbox/Makefile create mode 100644 netbox/config-patch1.diff create mode 100644 netbox/config-patch2.diff create mode 100644 netbox/django-driver.py create mode 100644 netbox/django-vault-client.py create mode 100755 netbox/entrypoint.sh create mode 100755 netbox/etc/service/netbox-rq/log/run create mode 100755 netbox/etc/service/netbox-rq/run create mode 100755 netbox/etc/service/uwsgi/log/run create mode 100755 netbox/etc/service/uwsgi/run create mode 100644 netbox/etc/uwsgi/netbox.ini create mode 100644 netbox/settings-patch.diff diff --git a/netbox/Dockerfile b/netbox/Dockerfile new file mode 100644 index 0000000..022aebf --- /dev/null +++ b/netbox/Dockerfile @@ -0,0 +1,73 @@ +FROM alpine:latest +LABEL maintainer="Mike Crute " + +ARG netbox_version + +ADD config-patch1.diff /config-patch1.diff +ADD config-patch2.diff /config-patch2.diff +ADD settings-patch.diff /settings-patch.diff + +RUN set -euxo pipefail; \ + \ + apk --no-cache add \ + build-base \ + dumb-init \ + jpeg-dev \ + libffi-dev \ + libxml2-dev \ + libxslt-dev \ + openssl-dev \ + postgresql-dev \ + py3-pip \ + python3-dev \ + runit \ + su-exec \ + uwsgi \ + uwsgi-python \ + zlib-dev \ + ; \ + cd /tmp; \ + wget "https://github.com/netbox-community/netbox/archive/v${netbox_version}.tar.gz"; \ + tar -xvf "v${netbox_version}.tar.gz" -C /opt; \ + rm "v${netbox_version}.tar.gz"; \ + mv /opt/netbox-${netbox_version}/ /opt/netbox/; \ + \ + cd /; \ + cp /opt/netbox/netbox/netbox/configuration.example.py /opt/netbox/netbox/netbox/configuration.py; \ + patch -p1 < /config-patch1.diff; \ + rm /config-patch1.diff; \ + \ + addgroup -S netbox; \ + adduser -S -G netbox netbox; \ + chown -R netbox:netbox /opt/netbox/netbox/media; \ + \ + cd /opt/netbox; \ + pip3 install wheel; \ + pip3 install -r requirements.txt; \ + \ + python3 netbox/manage.py collectstatic --no-input; \ + \ + cd /; \ + cp /opt/netbox/netbox/netbox/configuration.example.py /opt/netbox/netbox/netbox/configuration.py; \ + patch -p1 < /config-patch2.diff; \ + rm /config-patch2.diff; \ + \ + patch -p1 < /settings-patch.diff; \ + rm /settings-patch.diff; \ + \ + mkdir -p /usr/lib/python3.8/site-packages/django/db/backends/postgresqlvault; \ + touch /usr/lib/python3.8/site-packages/django/db/backends/postgresqlvault/__init__.py; \ + \ + rm -rf /root/.cache; \ + apk --no-cache del --purge \ + build-base \ + ; + +ADD django-vault-client.py /usr/lib/python3.8/site-packages/django/contrib/vault_client.py +ADD django-driver.py /usr/lib/python3.8/site-packages/django/db/backends/postgresqlvault/base.py +ADD etc/ /etc/ +ADD entrypoint.sh /entrypoint.sh + +STOPSIGNAL SIGHUP +ENTRYPOINT [ "/entrypoint.sh" ] +CMD [ "/usr/bin/dumb-init", "/sbin/runsvdir", "/etc/service" ] diff --git a/netbox/Makefile b/netbox/Makefile new file mode 100644 index 0000000..ffe7e64 --- /dev/null +++ b/netbox/Makefile @@ -0,0 +1,24 @@ +VERSION=2.10.3 +IMAGE=docker.crute.me/netbox:$(VERSION) +LATEST=$(subst :$(VERSION),,$(IMAGE)):latest + +all: + #docker pull alpine:latest + docker build \ + --build-arg=netbox_version=$(VERSION) \ + -t $(IMAGE) . + +all-no-cache: + docker build --no-cache -t $(IMAGE) . + +run: + docker run -d \ + -p 9110:9000 \ + -p 9111:9001 \ + -v /srv/code:/srv/code \ + $(IMAGE) + +publish: + docker push $(IMAGE) + docker tag $(IMAGE) $(LATEST) + docker push $(LATEST) diff --git a/netbox/config-patch1.diff b/netbox/config-patch1.diff new file mode 100644 index 0000000..be5b068 --- /dev/null +++ b/netbox/config-patch1.diff @@ -0,0 +1,26 @@ +--- a/opt/netbox/netbox/netbox/configuration.py ++++ b/opt/netbox/netbox/netbox/configuration.py +@@ -4,11 +4,13 @@ + # # + ######################### + ++import urllib3; urllib3.disable_warnings() ++ + # This is a list of valid fully-qualified domain names (FQDNs) for the NetBox server. NetBox will not permit write + # access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name. + # + # Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local'] +-ALLOWED_HOSTS = [] ++ALLOWED_HOSTS = ['*'] + + # PostgreSQL database configuration. See the Django documentation for a complete list of available parameters: + # https://docs.djangoproject.com/en/stable/ref/settings/#databases +@@ -51,7 +53,7 @@ + # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and + # symbols. NetBox will not run without this defined. For more information, see + # https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY +-SECRET_KEY = '' ++SECRET_KEY = 'dummy setup value will get cleared after setup run' + + + ######################### diff --git a/netbox/config-patch2.diff b/netbox/config-patch2.diff new file mode 100644 index 0000000..5983cc1 --- /dev/null +++ b/netbox/config-patch2.diff @@ -0,0 +1,89 @@ +--- a/opt/netbox/netbox/netbox/configuration.py ++++ b/opt/netbox/netbox/netbox/configuration.py +@@ -4,21 +4,35 @@ + # # + ######################### + ++import os ++from django.contrib.vault_client import SimpleVaultClient ++ ++ ++def _is_affirmative(value): ++ value = "" if not value else value ++ return value.lower() in ["yes", "true", "on", "1"] ++ ++ + # This is a list of valid fully-qualified domain names (FQDNs) for the NetBox server. NetBox will not permit write + # access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name. + # + # Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local'] +-ALLOWED_HOSTS = [] ++ALLOWED_HOSTS = ['*'] + + # PostgreSQL database configuration. See the Django documentation for a complete list of available parameters: + # https://docs.djangoproject.com/en/stable/ref/settings/#databases ++port = os.getenv("NETBOX_DB_PORT") + DATABASE = { +- 'NAME': 'netbox', # Database name +- 'USER': '', # PostgreSQL username +- 'PASSWORD': '', # PostgreSQL password +- 'HOST': 'localhost', # Database server +- 'PORT': '', # Database port (leave blank for default) +- 'CONN_MAX_AGE': 300, # Max database connection age ++ 'NAME': os.getenv("NETBOX_DB_NAME"), ++ 'HOST': os.getenv("NETBOX_DB_HOST"), ++ 'PORT': int(port) if port else "", ++ 'CONN_MAX_AGE': 300, ++ "VAULT_SKIP_VERIFY": os.getenv("VAULT_SKIP_VERIFY"), ++ "VAULT_ADDR": os.getenv("VAULT_ADDR"), ++ "VAULT_TOKEN": os.getenv("VAULT_TOKEN"), ++ "VAULT_DB_ROLE_NAME": os.getenv("VAULT_DB_ROLE_NAME"), ++ "VAULT_ROLE_ID": os.getenv("VAULT_ROLE_ID"), ++ "VAULT_SECRET_ID": os.getenv("VAULT_SECRET_ID"), + } + + # Redis database settings. Redis is used for caching and for queuing background tasks such as webhook events. A separate +@@ -26,23 +40,23 @@ + # to use two separate database IDs. + REDIS = { + 'tasks': { +- 'HOST': 'localhost', ++ 'HOST': os.getenv("NETBOX_REDIS_HOST"), + 'PORT': 6379, + # Comment out `HOST` and `PORT` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', + 'PASSWORD': '', +- 'DATABASE': 0, ++ 'DATABASE': int(os.getenv("NETBOX_REDIS_TASK_DB")), + 'SSL': False, + }, + 'caching': { +- 'HOST': 'localhost', ++ 'HOST': os.getenv("NETBOX_REDIS_HOST"), + 'PORT': 6379, + # Comment out `HOST` and `PORT` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', + 'PASSWORD': '', +- 'DATABASE': 1, ++ 'DATABASE': int(os.getenv("NETBOX_REDIS_CACHE_DB")), + 'SSL': False, + } + } +@@ -51,7 +65,14 @@ + # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and + # symbols. NetBox will not run without this defined. For more information, see + # https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY +-SECRET_KEY = '' ++vc = SimpleVaultClient( ++ os.getenv("VAULT_ADDR"), ++ os.getenv("VAULT_ROLE_ID"), ++ os.getenv("VAULT_SECRET_ID"), ++ ssl_verify=not _is_affirmative(os.getenv("VAULT_SKIP_VERIFY")) ++) ++SECRET_KEY = vc.get_kv_secret(os.getenv("NETBOX_VAULT_SECRET_NAME"), "key") ++del vc + + + ######################### diff --git a/netbox/django-driver.py b/netbox/django-driver.py new file mode 100644 index 0000000..65a9136 --- /dev/null +++ b/netbox/django-driver.py @@ -0,0 +1,69 @@ +import threading +from datetime import datetime, timedelta + +from django.core.exceptions import ImproperlyConfigured +from django.contrib.vault_client import SimpleVaultClient, Credential +from django.db.backends.postgresql.base import DatabaseWrapper as OrigWrapper + + +def _is_affirmative(value): + value = "" if not value else value + return value.lower() in ["yes", "true", "on", "1"] + + +def _must_get(store, key): + value = store.get(key) + + if not value: + raise ImproperlyConfigured( + f"Database parameter {key} is required but not set.") + + return value + + +class DatabaseWrapper(OrigWrapper): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._vault_cache_lock = threading.Lock() + self._vault_cred_cache = Credential.empty() + + def close_if_unusable_or_obsolete(self): + super().close_if_unusable_or_obsolete() + + if self.connection is None: + return + + with self._vault_cache_lock: + if not self._vault_cred_cache.is_valid: + self.close() + + # All of this is done under lock + def _get_vault_cred(self): + print("Getting credentials from vault") + params = self.settings_dict + + verify = not _is_affirmative(params.get("VAULT_SKIP_VERIFY")) + url = _must_get(params, "VAULT_ADDR") + token = params.get("VAULT_TOKEN") + db_role_name = _must_get(params, "VAULT_DB_ROLE_NAME") + role_id = _must_get(params, "VAULT_ROLE_ID") + role_secret = _must_get(params, "VAULT_SECRET_ID") + + client = SimpleVaultClient(url, role_id, role_secret, verify) + + self._vault_cred_cache = client.get_db_credential(db_role_name) + + def get_connection_params(self): + conn_params = super().get_connection_params() + + # Do the fetch under lock to prevent multiple threads from piling onto + # the vault server + with self._vault_cache_lock: + if not self._vault_cred_cache.is_valid: + self._get_vault_cred() + + conn_params["user"] = self._vault_cred_cache.username + conn_params["password"] = self._vault_cred_cache.password + + return conn_params diff --git a/netbox/django-vault-client.py b/netbox/django-vault-client.py new file mode 100644 index 0000000..e699db3 --- /dev/null +++ b/netbox/django-vault-client.py @@ -0,0 +1,84 @@ +import os +import ssl +import json +from urllib import request, parse +from datetime import datetime, timedelta + + +class Credential: + + def __init__(self, data): + self.username = data["data"]["username"] + self.password = data["data"]["password"] + + # Leave 2 minutes in case we encounter a failure + _duration = timedelta(seconds=data["lease_duration"] - 120) + self.expires = datetime.now() + _duration + + @classmethod + def empty(cls): + return cls({ + "lease_duration": -1, + "data": { "username": "", "password": "" } + }) + + @property + def is_valid(self): + return self.expires > datetime.now() + + + +class SimpleVaultClient: + + def __init__(self, base_url, role_id, role_secret, ssl_verify=True): + self.base_url = base_url + self.ssl_verify = ssl_verify + self.role_id = role_id + self.role_secret = role_secret + + env_token = os.getenv("VAULT_TOKEN") + if env_token: + self._token = env_token + self._token_expires = None # Token is assumed to never expire + else: + self._token = None + self._token_expires = datetime.now() + timedelta(seconds=-1) + + def _login_approle(self, role_id, secret): + res = self._make_request("auth/approle/login", auth=False, data={ + "role_id": role_id, + "secret_id": secret, + }) + + self._token = res["auth"]["client_token"] + self._token_expires = datetime.now() + \ + timedelta(seconds=res["auth"]["lease_duration"]) + + @property + def _token_is_expired(self): + return self._token_expires and \ + self._token_expires < (datetime.now() + timedelta(seconds=120)) + + def _auth_as_needed(self): + if self._token_is_expired: + self._login_approle(self.role_id, self.role_secret) + + def _make_request(self, url, data=None, auth=True): + context = ssl._create_unverified_context() \ + if not self.ssl_verify else None + + data = json.dumps(data).encode("utf-8") if data else None + headers = { "X-Vault-Token": self._token } if auth else {} + + url = parse.urljoin(self.base_url, parse.urljoin("/v1/", url)) + req = request.Request(url, headers=headers, data=data) + res = request.urlopen(req, context=context) + return json.load(res) + + def get_kv_secret(self, path, key): + self._auth_as_needed() + return self._make_request(f"kv/data/{path}")["data"]["data"][key] + + def get_db_credential(self, role): + self._auth_as_needed() + return Credential(self._make_request(f"database/creds/{role}")) diff --git a/netbox/entrypoint.sh b/netbox/entrypoint.sh new file mode 100755 index 0000000..a4f844c --- /dev/null +++ b/netbox/entrypoint.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +set -e + +cd "/opt/netbox" + +# Apply any database migrations +/sbin/su-exec netbox python3 netbox/manage.py migrate + +# Trace any missing cable paths (not typically needed) +/sbin/su-exec netbox python3 netbox/manage.py trace_paths --no-input + +# Delete any stale content types +/sbin/su-exec netbox python3 netbox/manage.py remove_stale_contenttypes --no-input + +# Delete any expired user sessions +/sbin/su-exec netbox python3 netbox/manage.py clearsessions + +# Clear all cached data +/sbin/su-exec netbox python3 netbox/manage.py invalidate all + +exec "$@" diff --git a/netbox/etc/service/netbox-rq/log/run b/netbox/etc/service/netbox-rq/log/run new file mode 100755 index 0000000..6193824 --- /dev/null +++ b/netbox/etc/service/netbox-rq/log/run @@ -0,0 +1,3 @@ +#!/bin/sh + +cat - diff --git a/netbox/etc/service/netbox-rq/run b/netbox/etc/service/netbox-rq/run new file mode 100755 index 0000000..aa4b675 --- /dev/null +++ b/netbox/etc/service/netbox-rq/run @@ -0,0 +1,23 @@ +#!/bin/sh + +# runsv sends us a TERM but uwsgi will only shutdown cleanly +# if it receives an INT so we need to translate the signal +# properly for uwsgi +trap 'kill -INT $PID' TERM + +/sbin/su-exec netbox /usr/bin/python3 /opt/netbox/netbox/manage.py rqworker & + +PID=$! + +# wait for uwsgi, will get cancelled when runsv TERMs us and +# the trap will get executed next, unless something goes wrong +# and uwsgi fails then this wait will run +wait $PID + +# if something went wrong then unregister the trap because it +# won't have a target +trap - TERM + +# waiting on a dead process will return the return code of the +# processes original exit +wait $PID diff --git a/netbox/etc/service/uwsgi/log/run b/netbox/etc/service/uwsgi/log/run new file mode 100755 index 0000000..6193824 --- /dev/null +++ b/netbox/etc/service/uwsgi/log/run @@ -0,0 +1,3 @@ +#!/bin/sh + +cat - diff --git a/netbox/etc/service/uwsgi/run b/netbox/etc/service/uwsgi/run new file mode 100755 index 0000000..e24ede7 --- /dev/null +++ b/netbox/etc/service/uwsgi/run @@ -0,0 +1,23 @@ +#!/bin/sh + +# runsv sends us a TERM but uwsgi will only shutdown cleanly +# if it receives an INT so we need to translate the signal +# properly for uwsgi +trap 'kill -INT $PID' TERM + +/usr/sbin/uwsgi --ini /etc/uwsgi/netbox.ini & + +PID=$! + +# wait for uwsgi, will get cancelled when runsv TERMs us and +# the trap will get executed next, unless something goes wrong +# and uwsgi fails then this wait will run +wait $PID + +# if something went wrong then unregister the trap because it +# won't have a target +trap - TERM + +# waiting on a dead process will return the return code of the +# processes original exit +wait $PID diff --git a/netbox/etc/uwsgi/netbox.ini b/netbox/etc/uwsgi/netbox.ini new file mode 100644 index 0000000..8431c6f --- /dev/null +++ b/netbox/etc/uwsgi/netbox.ini @@ -0,0 +1,15 @@ +[uwsgi] +plugin = python +master = true +no-orphans = true +socket = [::]:9000 +uid = netbox +gid = netbox +mime-file = /etc/mime.types +chdir = /opt/netbox/netbox +pythonpath = /opt/netbox/netbox +workers = 2 +wsgi-file = netbox/wsgi.py +harakiri = 300 +offload-threads = 4 +static-map = /static=/opt/netbox/netbox/static diff --git a/netbox/settings-patch.diff b/netbox/settings-patch.diff new file mode 100644 index 0000000..78510ad --- /dev/null +++ b/netbox/settings-patch.diff @@ -0,0 +1,11 @@ +--- /opt/netbox/netbox/netbox/settings.py ++++ /opt/netbox/netbox/netbox/settings.py +@@ -147,7 +147,7 @@ + }) + else: + DATABASE.update({ +- 'ENGINE': 'django.db.backends.postgresql' ++ 'ENGINE': 'django.db.backends.postgresqlvault' + }) + + DATABASES = { -- cgit v1.2.3