bump version: 0.1.28-beta -> 0.1.29-beta

Resolve "Websocket Proxying"

Closes #38

See merge request BeryJu.org/passbook!24

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.27-beta
+current_version = 0.1.29-beta
 tag = True
 commit = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)\-(?P<release>.*)

.gitlab-ci.yml

@@ -40,6 +40,7 @@ pylint:
     stage: test
 coverage:
     script:
+        - python manage.py collectstatic --no-input
         - coverage run manage.py test
         - coverage report
     stage: test
@@ -55,7 +56,7 @@ package-docker:
     before_script:
         - echo "{\"auths\":{\"docker.$NEXUS_URL\":{\"auth\":\"$NEXUS_AUTH\"}}}" > /kaniko/.docker/config.json
     script:
-        - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination docker.pkg.beryju.org/passbook:latest --destination docker.pkg.beryju.org/passbook:0.1.27-beta
+        - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination docker.pkg.beryju.org/passbook:latest --destination docker.pkg.beryju.org/passbook:0.1.29-beta
     stage: build
     only:
         - tags

Dockerfile

@@ -6,7 +6,7 @@ COPY ./requirements.txt /app/
 
 WORKDIR /app/
 
-RUN apt-get update && apt-get install build-essential libssl-dev libffi-dev -y && \
+RUN apt-get update && apt-get install build-essential libssl-dev libffi-dev libpq-dev -y && \
     mkdir /app/static/ && \
     pip install -r requirements.txt && \
     pip install psycopg2 && \
@@ -23,7 +23,7 @@ COPY --from=build /app/static /app/static/
 
 WORKDIR /app/
 
-RUN apt-get update && apt-get install build-essential libssl-dev libffi-dev -y && \
+RUN apt-get update && apt-get install build-essential libssl-dev libffi-dev libpq-dev -y && \
     pip install -r requirements.txt && \
     pip install psycopg2 && \
     adduser --system --home /app/ passbook && \

client-packages/allauth/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup
 
 setup(
     name='django-allauth-passbook',
-    version='0.1.27-beta',
+    version='0.1.29-beta',
     description='passbook support for django-allauth',
     # long_description='\n'.join(read_simple('docs/index.md')[2:]),
     long_description_content_type='text/markdown',

client-packages/sentry-auth-passbook/setup.py

@@ -18,7 +18,7 @@ tests_require = [
 
 setup(
     name='sentry-auth-passbook',
-    version='0.1.27-beta',
+    version='0.1.29-beta',
     author='BeryJu.org',
     author_email='support@beryju.org',
     url='https://passbook.beryju.org',

debian/changelog

@@ -1,3 +1,30 @@
+passbook (0.1.29) stable; urgency=medium
+
+  * bump version: 0.1.27-beta -> 0.1.28-beta
+  * Add libpq-dev dependency so psycopg2 build works
+  * switch to whitenoise for static files
+  * replace cherrypy with daphne
+  * Run collectstatic before coverage, use autoreload on celery worker
+
+ -- Jens Langhammer <jens.langhammer@beryju.org>  Thu, 11 Apr 2019 12:00:27 +0000
+
+passbook (0.1.28) stable; urgency=medium
+
+  * bump version: 0.1.26-beta -> 0.1.27-beta
+  * fix allauth client's formatting
+  * switch from raven to sentry_sdk
+  * add ability to have non-expiring nonces, clean up expired nonces
+  * fully remove raven and switch WSGI and logging to sentry_sdk
+  * fix failing CI
+  * trigger autoreload from config files
+  * Choose upstream more cleverly
+  * Move code from django-revproxy to app_gw to fix cookie bug
+  * Implement websocket proxy
+  * switch kubernetes deployment to daphne server
+  * set default log level to warn, fix clean_nonces not working
+
+ -- Jens Langhammer <jens.langhammer@beryju.org>  Thu, 11 Apr 2019 08:46:44 +0000
+
 passbook (0.1.27) stable; urgency=medium
 
   * bump version: 0.1.25-beta -> 0.1.26-beta

debian/control

@@ -3,7 +3,7 @@ Section: admin
 Priority: optional
 Maintainer: BeryJu.org <support@beryju.org>
 Uploaders: Jens Langhammer <jens@beryju.org>, BeryJu.org <support@beryju.org>
-Build-Depends: debhelper (>= 10), dh-systemd (>= 1.5), dh-exec, wget, dh-exec, python3 (>= 3.5) | python3.6 | python3.7
+Build-Depends: debhelper (>= 10), dh-systemd (>= 1.5), dh-exec, wget, dh-exec, python3 (>= 3.5) | python3.6 | python3.7, libpq-dev
 Standards-Version: 3.9.6
 
 Package: passbook

helm/passbook/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
-appVersion: "0.1.27-beta"
+appVersion: "0.1.29-beta"
 description: A Helm chart for passbook.
 name: passbook
-version: "0.1.27-beta"
+version: "0.1.29-beta"
 icon: https://passbook.beryju.org/images/logo.png

helm/passbook/templates/passbook-configmap.yaml

@@ -15,8 +15,8 @@ data:
         port: ''
     log:
       level:
-        console: DEBUG
-        file: DEBUG
+        console: WARNING
+        file: WARNING
       file: /dev/null
       syslog:
         host: 127.0.0.1

helm/passbook/values.yaml

@@ -5,7 +5,7 @@
 replicaCount: 1
 
 image:
-  tag: 0.1.27-beta
+  tag: 0.1.29-beta
 
 nameOverride: ""
 

passbook/__init__.py

@@ -1,2 +1,2 @@
 """passbook"""
-__version__ = '0.1.27-beta'
+__version__ = '0.1.29-beta'

passbook/admin/__init__.py

@@ -1,2 +1,2 @@
 """passbook admin"""
-__version__ = '0.1.27-beta'
+__version__ = '0.1.29-beta'

passbook/api/__init__.py

@@ -1,2 +1,2 @@
 """passbook api"""
-__version__ = '0.1.27-beta'
+__version__ = '0.1.29-beta'

passbook/app_gw/__init__.py

@@ -1,2 +1,2 @@
 """passbook Application Security Gateway Header"""
-__version__ = '0.1.27-beta'
+__version__ = '0.1.29-beta'

passbook/app_gw/middleware.py

@@ -1,6 +1,7 @@
 """passbook app_gw middleware"""
 import mimetypes
 from logging import getLogger
+from random import SystemRandom
 from urllib.parse import urlparse
 
 import certifi
@@ -9,15 +10,16 @@ from django.core.cache import cache
 from django.utils.http import urlencode
 from django.views.generic import RedirectView
 from revproxy.exceptions import InvalidUpstream
-from revproxy.response import get_django_response
-from revproxy.utils import encode_items, normalize_request_headers
 
 from passbook.app_gw.models import ApplicationGatewayProvider
+from passbook.app_gw.proxy.response import get_django_response
+from passbook.app_gw.proxy.utils import encode_items, normalize_request_headers
 from passbook.app_gw.rewrite import Rewriter
 from passbook.core.models import Application
 from passbook.core.policies import PolicyEngine
 from passbook.lib.config import CONFIG
 
+SESSION_UPSTREAM_KEY = 'passbook_app_gw_upstream'
 IGNORED_HOSTNAMES_KEY = 'passbook_app_gw_ignored'
 LOGGER = getLogger(__name__)
 QUOTE_SAFE = r'<.;>\(}*+|~=-$/_:^@)[{]&\'!,"`'
@@ -68,11 +70,11 @@ class ApplicationGatewayMiddleware:
             return True, None
         # At this point we're certain there's a matching ApplicationGateway
         if len(matches) > 1:
-            # TODO This should never happen
+            # This should never happen
             raise ValueError
         app_gw = matches.first()
         try:
-            # Check if ApplicationGateway is associcaited with application
+            # Check if ApplicationGateway is associated with application
             getattr(app_gw, 'application')
             return False, app_gw
         except Application.DoesNotExist:
@@ -87,10 +89,18 @@ class ApplicationGatewayMiddleware:
         self.request = request
         return self.dispatch(request)
 
+    def _get_upstream(self):
+        """Choose random upstream and save in session"""
+        if SESSION_UPSTREAM_KEY not in self.request.session:
+            self.request.session[SESSION_UPSTREAM_KEY] = {}
+        if self.app_gw.pk not in self.request.session[SESSION_UPSTREAM_KEY]:
+            upstream_index = SystemRandom().randrange(len(self.app_gw.upstream))
+            self.request.session[SESSION_UPSTREAM_KEY][self.app_gw.pk] = upstream_index
+        return self.app_gw.upstream[self.request.session[SESSION_UPSTREAM_KEY][self.app_gw.pk]]
+
     def get_upstream(self):
         """Get upstream as parsed url"""
-        # TODO: How to choose upstream?
-        upstream = self.app_gw.upstream[0]
+        upstream = self._get_upstream()
 
         self._parsed_url = urlparse(upstream)
 

passbook/app_gw/proxy/exceptions.py

@@ -0,0 +1,8 @@
+"""Exception classes"""
+
+class ReverseProxyException(Exception):
+    """Base for revproxy exception"""
+
+
+class InvalidUpstream(ReverseProxyException):
+    """Invalid upstream set"""

passbook/app_gw/proxy/response.py

@@ -0,0 +1,63 @@
+"""response functions from django-revproxy"""
+import logging
+
+from django.http import HttpResponse, StreamingHttpResponse
+
+from passbook.app_gw.proxy.utils import (cookie_from_string,
+                                         set_response_headers, should_stream)
+
+#: Default number of bytes that are going to be read in a file lecture
+DEFAULT_AMT = 2 ** 16
+
+logger = logging.getLogger('revproxy.response')
+
+
+def get_django_response(proxy_response, strict_cookies=False):
+    """This method is used to create an appropriate response based on the
+    Content-Length of the proxy_response. If the content is bigger than
+    MIN_STREAMING_LENGTH, which is found on utils.py,
+    than django.http.StreamingHttpResponse will be created,
+    else a django.http.HTTPResponse will be created instead
+
+    :param proxy_response: An Instance of urllib3.response.HTTPResponse that
+                           will create an appropriate response
+    :param strict_cookies: Whether to only accept RFC-compliant cookies
+    :returns: Returns an appropriate response based on the proxy_response
+              content-length
+    """
+    status = proxy_response.status
+    headers = proxy_response.headers
+
+    logger.debug('Proxy response headers: %s', headers)
+
+    content_type = headers.get('Content-Type')
+
+    logger.debug('Content-Type: %s', content_type)
+
+    if should_stream(proxy_response):
+        logger.info('Content-Length is bigger than %s', DEFAULT_AMT)
+        response = StreamingHttpResponse(proxy_response.stream(DEFAULT_AMT),
+                                         status=status,
+                                         content_type=content_type)
+    else:
+        content = proxy_response.data or b''
+        response = HttpResponse(content, status=status,
+                                content_type=content_type)
+
+    logger.info('Normalizing response headers')
+    set_response_headers(response, headers)
+
+    logger.debug('Response headers: %s', getattr(response, '_headers'))
+
+    cookies = proxy_response.headers.getlist('set-cookie')
+    logger.info('Checking for invalid cookies')
+    for cookie_string in cookies:
+        cookie_dict = cookie_from_string(cookie_string,
+                                         strict_cookies=strict_cookies)
+        # if cookie is invalid cookie_dict will be None
+        if cookie_dict:
+            response.set_cookie(**cookie_dict)
+
+    logger.debug('Response cookies: %s', response.cookies)
+
+    return response

passbook/app_gw/proxy/utils.py

@@ -0,0 +1,227 @@
+"""Utils from django-revproxy, slightly adjusted"""
+import logging
+import re
+from wsgiref.util import is_hop_by_hop
+
+try:
+    from http.cookies import SimpleCookie
+    COOKIE_PREFIX = ''
+except ImportError:
+    from Cookie import SimpleCookie
+    COOKIE_PREFIX = 'Set-Cookie: '
+
+
+#: List containing string constant that are used to represent headers that can
+#: be ignored in the required_header function
+IGNORE_HEADERS = (
+    'HTTP_ACCEPT_ENCODING',  # We want content to be uncompressed so
+                             # we remove the Accept-Encoding from
+                             # original request
+    'HTTP_HOST',
+    'HTTP_REMOTE_USER',
+)
+
+
+# Default from HTTP RFC 2616
+#   See: http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.7.1
+#: Variable that represent the default charset used
+DEFAULT_CHARSET = 'latin-1'
+
+#: List containing string constants that represents possible html content type
+HTML_CONTENT_TYPES = (
+    'text/html',
+    'application/xhtml+xml'
+)
+
+#: Variable used to represent a minimal content size required for response
+#: to be turned into stream
+MIN_STREAMING_LENGTH = 4 * 1024  # 4KB
+
+#: Regex used to find charset in a html content type
+_get_charset_re = re.compile(r';\s*charset=(?P<charset>[^\s;]+)', re.I)
+
+
+def is_html_content_type(content_type):
+    """Function used to verify if the parameter is a proper html content type
+
+    :param content_type: String variable that represent a content-type
+    :returns:  A boolean value stating if the content_type is a valid html
+               content type
+    """
+    for html_content_type in HTML_CONTENT_TYPES:
+        if content_type.startswith(html_content_type):
+            return True
+
+    return False
+
+
+def should_stream(proxy_response):
+    """Function to verify if the proxy_response must be converted into
+    a stream.This will be done by checking the proxy_response content-length
+    and verify if its length is bigger than one stipulated
+    by MIN_STREAMING_LENGTH.
+
+    :param proxy_response: An Instance of urllib3.response.HTTPResponse
+    :returns: A boolean stating if the proxy_response should
+              be treated as a stream
+    """
+    content_type = proxy_response.headers.get('Content-Type')
+
+    if is_html_content_type(content_type):
+        return False
+
+    try:
+        content_length = int(proxy_response.headers.get('Content-Length', 0))
+    except ValueError:
+        content_length = 0
+
+    if not content_length or content_length > MIN_STREAMING_LENGTH:
+        return True
+
+    return False
+
+
+def get_charset(content_type):
+    """Function used to retrieve the charset from a content-type.If there is no
+    charset in the content type then the charset defined on DEFAULT_CHARSET
+    will be returned
+
+    :param  content_type:   A string containing a Content-Type header
+    :returns:               A string containing the charset
+    """
+    if not content_type:
+        return DEFAULT_CHARSET
+
+    matched = _get_charset_re.search(content_type)
+    if matched:
+        # Extract the charset and strip its double quotes
+        return matched.group('charset').replace('"', '')
+    return DEFAULT_CHARSET
+
+
+def required_header(header):
+    """Function that verify if the header parameter is a essential header
+
+    :param header:  A string represented a header
+    :returns:       A boolean value that represent if the header is required
+    """
+    if header in IGNORE_HEADERS:
+        return False
+
+    if header.startswith('HTTP_') or header == 'CONTENT_TYPE':
+        return True
+
+    return False
+
+
+def set_response_headers(response, response_headers):
+    """Set response's header"""
+    for header, value in response_headers.items():
+        if is_hop_by_hop(header) or header.lower() == 'set-cookie':
+            continue
+
+        response[header.title()] = value
+
+    logger.debug('Response headers: %s', getattr(response, '_headers'))
+
+
+def normalize_request_headers(request):
+    """Function used to transform header, replacing 'HTTP\\_' to ''
+    and replace '_' to '-'
+
+    :param request:  A HttpRequest that will be transformed
+    :returns:        A dictionary with the normalized headers
+    """
+    norm_headers = {}
+    for header, value in request.META.items():
+        if required_header(header):
+            norm_header = header.replace('HTTP_', '').title().replace('_', '-')
+            norm_headers[norm_header] = value
+
+    return norm_headers
+
+
+def encode_items(items):
+    """Function that encode all elements in the list of items passed as
+    a parameter
+
+    :param items:  A list of tuple
+    :returns:      A list of tuple with all items encoded in 'utf-8'
+    """
+    encoded = []
+    for key, values in items:
+        for value in values:
+            encoded.append((key.encode('utf-8'), value.encode('utf-8')))
+    return encoded
+
+
+logger = logging.getLogger('revproxy.cookies')
+
+
+def cookie_from_string(cookie_string, strict_cookies=False):
+    """Parser for HTTP header set-cookie
+    The return from this function will be used as parameters for
+    django's response.set_cookie method. Because set_cookie doesn't
+    have parameter comment, this cookie attribute will be ignored.
+
+    :param  cookie_string: A string representing a valid cookie
+    :param  strict_cookies: Whether to only accept RFC-compliant cookies
+    :returns: A dictionary containing the cookie_string attributes
+    """
+
+    if strict_cookies:
+
+        cookies = SimpleCookie(COOKIE_PREFIX + cookie_string)
+        if not cookies.keys():
+            return None
+        cookie_name, = cookies.keys()
+        cookie_dict = {k: v for k, v in cookies[cookie_name].items()
+                       if v and k != 'comment'}
+        cookie_dict['key'] = cookie_name
+        cookie_dict['value'] = cookies[cookie_name].value
+        return cookie_dict
+    valid_attrs = ('path', 'domain', 'comment', 'expires',
+                   'max_age', 'httponly', 'secure')
+
+    cookie_dict = {}
+
+    cookie_parts = cookie_string.split(';')
+    try:
+        cookie_dict['key'], cookie_dict['value'] = \
+            cookie_parts[0].split('=', 1)
+        cookie_dict['value'] = cookie_dict['value'].replace('"', '')
+        # print('aaaaaaaaaaaaaaaaaaaaaaaaaaaa')
+        # print(cookie_parts[0].split('=', 1))
+    except ValueError:
+        logger.warning('Invalid cookie: `%s`', cookie_string)
+        return None
+
+    if cookie_dict['value'].startswith('='):
+        logger.warning('Invalid cookie: `%s`', cookie_string)
+        return None
+
+    for part in cookie_parts[1:]:
+        if '=' in part:
+            attr, value = part.split('=', 1)
+            value = value.strip()
+        else:
+            attr = part
+            value = ''
+
+        attr = attr.strip().lower()
+        if not attr:
+            continue
+
+        if attr in valid_attrs:
+            if attr in ('httponly', 'secure'):
+                cookie_dict[attr] = True
+            elif attr in 'comment':
+                # ignoring comment attr as explained in the
+                # function docstring
+                continue
+            else:
+                cookie_dict[attr] = value
+        else:
+            logger.warning('Unknown cookie attribute %s', attr)
+
+        return cookie_dict

passbook/app_gw/requirements.txt

@@ -1,2 +1,7 @@
 django-revproxy
 urllib3[secure]
+channels
+service_identity
+websocket-client
+daphne<2.3.0
+asgiref~=2.3

passbook/app_gw/settings.py

@@ -1,5 +1,5 @@
 """Application Security Gateway settings"""
-
-# INSTALLED_APPS = [
-#     'revproxy'
-# ]
+INSTALLED_APPS = [
+    'channels'
+]
+ASGI_APPLICATION = "passbook.app_gw.websocket.routing.application"

passbook/app_gw/websocket/consumer.py

@@ -0,0 +1,83 @@
+"""websocket proxy consumer"""
+import threading
+from logging import getLogger
+from ssl import CERT_NONE
+
+import websocket
+from channels.generic.websocket import WebsocketConsumer
+
+from passbook.app_gw.models import ApplicationGatewayProvider
+
+LOGGER = getLogger(__name__)
+
+class ProxyConsumer(WebsocketConsumer):
+    """Proxy websocket connection to upstream"""
+
+    _headers_dict = {}
+    _app_gw = None
+    _client = None
+    _thread = None
+
+    def _fix_headers(self, input_dict):
+        """Fix headers from bytestrings to normal strings"""
+        return {
+            key.decode('utf-8'): value.decode('utf-8')
+            for key, value in dict(input_dict).items()
+        }
+
+    def connect(self):
+        """Extract host header, lookup in database and proxy connection"""
+        self._headers_dict = self._fix_headers(dict(self.scope.get('headers')))
+        host = self._headers_dict.pop('host')
+        query_string = self.scope.get('query_string').decode('utf-8')
+        matches = ApplicationGatewayProvider.objects.filter(
+            server_name__contains=[host],
+            enabled=True)
+        if matches.exists():
+            self._app_gw = matches.first()
+            # TODO: Get upstream that starts with wss or
+            upstream = self._app_gw.upstream[0].replace('http', 'ws') + self.scope.get('path')
+            if query_string:
+                upstream += '?' + query_string
+            sslopt = {}
+            if not self._app_gw.upstream_ssl_verification:
+                sslopt = {"cert_reqs": CERT_NONE}
+            self._client = websocket.WebSocketApp(
+                url=upstream,
+                subprotocols=self.scope.get('subprotocols'),
+                header=self._headers_dict,
+                on_message=self._client_on_message_handler(),
+                on_error=self._client_on_error_handler(),
+                on_close=self._client_on_close_handler(),
+                on_open=self._client_on_open_handler())
+            LOGGER.debug("Accepting connection for %s", host)
+            self._thread = threading.Thread(target=lambda: self._client.run_forever(sslopt=sslopt))
+            self._thread.start()
+
+    def _client_on_open_handler(self):
+        return lambda ws: self.accept(self._client.sock.handshake_response.subprotocol)
+
+    def _client_on_message_handler(self):
+        # pylint: disable=unused-argument,invalid-name
+        def message_handler(ws, message):
+            if isinstance(message, str):
+                self.send(text_data=message)
+            else:
+                self.send(bytes_data=message)
+        return message_handler
+
+    def _client_on_error_handler(self):
+        return lambda ws, error: print(error)
+
+    def _client_on_close_handler(self):
+        return lambda ws: self.disconnect(0)
+
+    def disconnect(self, code):
+        self._client.close()
+
+    def receive(self, text_data=None, bytes_data=None):
+        if text_data:
+            opcode = websocket.ABNF.OPCODE_TEXT
+        if bytes_data:
+            opcode = websocket.ABNF.OPCODE_BINARY
+        self._client.send(text_data or bytes_data, opcode)

passbook/app_gw/websocket/routing.py

@@ -0,0 +1,17 @@
+"""app_gw websocket proxy"""
+from channels.auth import AuthMiddlewareStack
+from channels.routing import ProtocolTypeRouter, URLRouter
+from django.conf.urls import url
+
+from passbook.app_gw.websocket.consumer import ProxyConsumer
+
+websocket_urlpatterns = [
+    url(r'^(.*)$', ProxyConsumer),
+]
+
+application = ProtocolTypeRouter({
+    # (http->django views is added by default)
+    'websocket': AuthMiddlewareStack(
+        URLRouter(websocket_urlpatterns)
+    ),
+})

passbook/audit/__init__.py

@@ -1,2 +1,2 @@
 """passbook audit Header"""
-__version__ = '0.1.27-beta'
+__version__ = '0.1.29-beta'

passbook/captcha_factor/__init__.py

@@ -1,2 +1,2 @@
 """passbook captcha_factor Header"""
-__version__ = '0.1.27-beta'
+__version__ = '0.1.29-beta'

passbook/core/__init__.py

@@ -1,2 +1,2 @@
 """passbook core"""
-__version__ = '0.1.27-beta'
+__version__ = '0.1.29-beta'

passbook/core/asgi.py

@@ -0,0 +1,13 @@
+"""
+ASGI entrypoint. Configures Django and then runs the application
+defined in the ASGI_APPLICATION setting.
+"""
+
+import os
+
+import django
+from channels.routing import get_default_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "passbook.core.settings")
+django.setup()
+application = get_default_application()

passbook/core/celery.py

@@ -3,10 +3,8 @@
 import logging
 import os
 
-import celery
+from celery import Celery, signals
 from django.conf import settings
-from raven import Client
-from raven.contrib.celery import register_logger_signal, register_signal
 
 # set the default Django settings module for the 'celery' program.
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "passbook.core.settings")
@@ -14,31 +12,18 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "passbook.core.settings")
 LOGGER = logging.getLogger(__name__)
 
 
-class Celery(celery.Celery):
-    """Custom Celery class with Raven configured"""
-
-    # pylint: disable=method-hidden
-    def on_configure(self):
-        """Update raven client"""
-        try:
-            client = Client(settings.RAVEN_CONFIG.get('dsn'))
-            # register a custom filter to filter out duplicate logs
-            register_logger_signal(client)
-            # hook into the Celery error handler
-            register_signal(client)
-        except RecursionError:  # This error happens when pdoc is running
-            pass
+CELERY_APP = Celery('passbook')
 
 
 # pylint: disable=unused-argument
-@celery.signals.setup_logging.connect
+@signals.setup_logging.connect
 def config_loggers(*args, **kwags):
     """Apply logging settings from settings.py to celery"""
     logging.config.dictConfig(settings.LOGGING)
 
 
 # pylint: disable=unused-argument
-@celery.signals.after_task_publish.connect
+@signals.after_task_publish.connect
 def after_task_publish(sender=None, headers=None, body=None, **kwargs):
     """Log task_id after it was published"""
     info = headers if 'task' in headers else body
@@ -46,22 +31,20 @@ def after_task_publish(sender=None, headers=None, body=None, **kwargs):
 
 
 # pylint: disable=unused-argument
-@celery.signals.task_prerun.connect
+@signals.task_prerun.connect
 def task_prerun(task_id, task, *args, **kwargs):
     """Log task_id on worker"""
     LOGGER.debug('%-40s started (name=%s)', task_id, task.__name__)
 
 
 # pylint: disable=unused-argument
-@celery.signals.task_postrun.connect
+@signals.task_postrun.connect
 def task_postrun(task_id, task, *args, retval=None, state=None, **kwargs):
     """Log task_id on worker"""
     LOGGER.debug('%-40s finished (name=%s, state=%s)',
                  task_id, task.__name__, state)
 
 
-CELERY_APP = Celery('passbook')
-
 # Using a string here means the worker doesn't have to serialize
 # the configuration object to child processes.
 # - namespace='CELERY' means all celery-related configuration keys

passbook/core/management/commands/web.py

@@ -2,11 +2,11 @@
 
 from logging import getLogger
 
-import cherrypy
-from django.conf import settings
+from daphne.cli import CommandLineInterface
 from django.core.management.base import BaseCommand
+from django.utils import autoreload
 
-from passbook.core.wsgi import application
+from passbook.lib.config import CONFIG
 
 LOGGER = getLogger(__name__)
 
@@ -15,20 +15,16 @@ class Command(BaseCommand):
     """Run CherryPy webserver"""
 
     def handle(self, *args, **options):
-        """passbook cherrypy server"""
-        config = settings.CHERRYPY_SERVER
-        config.update(**options)
-        cherrypy.config.update(config)
-        cherrypy.tree.graft(application, '/')
-        # Mount NullObject to serve static files
-        cherrypy.tree.mount(None, '/static', config={
-            '/': {
-                'tools.staticdir.on': True,
-                'tools.staticdir.dir': settings.STATIC_ROOT,
-                'tools.expires.on': True,
-                'tools.expires.secs': 86400,
-                'tools.gzip.on': True,
-            }
-        })
-        cherrypy.engine.start()
-        cherrypy.engine.block()
+        """passbook daphne server"""
+        autoreload.run_with_reloader(self.daphne_server)
+
+    def daphne_server(self):
+        """Run daphne server within autoreload"""
+        autoreload.raise_last_exception()
+        with CONFIG.cd('web'):
+            CommandLineInterface().run([
+                '-p', str(CONFIG.get('port', 8000)),
+                '-b', CONFIG.get('listen', '0.0.0.0'),  # nosec
+                '--access-log', '/dev/null',
+                'passbook.core.asgi:application'
+            ])

passbook/core/management/commands/worker.py

@@ -3,6 +3,7 @@
 from logging import getLogger
 
 from django.core.management.base import BaseCommand
+from django.utils import autoreload
 
 from passbook.core.celery import CELERY_APP
 
@@ -14,4 +15,9 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         """celery worker"""
-        CELERY_APP.worker_main(['worker', '--autoscale=10,3', '-E'])
+        autoreload.run_with_reloader(self.celery_worker)
+
+    def celery_worker(self):
+        """Run celery worker within autoreload"""
+        autoreload.raise_last_exception()
+        CELERY_APP.worker_main(['worker', '--autoscale=10,3', '-E', '-B'])

passbook/core/migrations/0022_nonce_expiring.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.1.7 on 2019-04-04 19:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('passbook_core', '0021_policy_timeout'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='nonce',
+            name='expiring',
+            field=models.BooleanField(default=True),
+        ),
+    ]

passbook/core/models.py

@@ -437,6 +437,7 @@ class Nonce(UUIDModel):
 
     expires = models.DateTimeField(default=default_nonce_duration)
     user = models.ForeignKey('User', on_delete=models.CASCADE)
+    expiring = models.BooleanField(default=True)
 
     def __str__(self):
         return "Nonce %s (expires=%s)" % (self.uuid.hex, self.expires)

passbook/core/requirements.txt

@@ -1,5 +1,4 @@
 celery
-cherrypy
 colorlog
 django-ipware
 django-model-utils
@@ -10,4 +9,6 @@ idna<2.8,>=2.5
 markdown
 psycopg2
 PyYAML
-raven
+sentry-sdk
+pip
+whitenoise

passbook/core/settings.py

@@ -11,10 +11,16 @@ https://docs.djangoproject.com/en/2.1/ref/settings/
 """
 
 import importlib
+import logging
 import os
 import sys
 
+from celery.schedules import crontab
 from django.contrib import messages
+from sentry_sdk import init as sentry_init
+from sentry_sdk.integrations.celery import CeleryIntegration
+from sentry_sdk.integrations.django import DjangoIntegration
+from sentry_sdk.integrations.logging import LoggingIntegration
 
 from passbook import __version__
 from passbook.lib.config import CONFIG
@@ -34,7 +40,8 @@ SECRET_KEY = CONFIG.get('secret_key')
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = CONFIG.get('debug')
 INTERNAL_IPS = ['127.0.0.1']
-ALLOWED_HOSTS = CONFIG.get('domains', []) + [CONFIG.get('primary_domain')]
+# ALLOWED_HOSTS = CONFIG.get('domains', []) + [CONFIG.get('primary_domain')]
+ALLOWED_HOSTS = ['*']
 SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
 
 LOGIN_URL = 'passbook_core:auth-login'
@@ -66,7 +73,6 @@ INSTALLED_APPS = [
     'django.contrib.postgres',
     'rest_framework',
     'drf_yasg',
-    'raven.contrib.django.raven_compat',
     'passbook.core.apps.PassbookCoreConfig',
     'passbook.admin.apps.PassbookAdminConfig',
     'passbook.api.apps.PassbookAPIConfig',
@@ -115,6 +121,7 @@ CACHES = {
 
 MIDDLEWARE = [
     'django.contrib.sessions.middleware.SessionMiddleware',
+    'whitenoise.middleware.WhiteNoiseMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'passbook.app_gw.middleware.ApplicationGatewayMiddleware',
     'django.middleware.security.SecurityMiddleware',
@@ -122,7 +129,6 @@ MIDDLEWARE = [
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
-    'raven.contrib.django.raven_compat.middleware.SentryResponseErrorIdMiddleware',
 ]
 
 ROOT_URLCONF = 'passbook.core.urls'
@@ -204,32 +210,34 @@ CELERY_BROKER_URL = 'amqp://%s' % CONFIG.get('rabbitmq')
 CELERY_RESULT_BACKEND = 'rpc://'
 CELERY_ACKS_LATE = True
 CELERY_BROKER_HEARTBEAT = 0
-
-# Raven settings
-RAVEN_CONFIG = {
-    'dsn': ('https://55b5dd780bc14f4c96bba69b7a9abbcc:449af483bd0745'
-            '0d83be640d834e5458@sentry.services.beryju.org/8'),
-    'release': VERSION,
-    'environment': 'dev' if DEBUG else 'production',
+CELERY_BEAT_SCHEDULE = {
+    'cleanup-expired-nonces': {
+        'task': 'passbook.core.tasks.clean_nonces',
+        'schedule': crontab(hour=1, minute=1)
+    }
 }
 
-# CherryPY settings
-with CONFIG.cd('web'):
-    CHERRYPY_SERVER = {
-        'server.socket_host': CONFIG.get('listen', '0.0.0.0'),  # nosec
-        'server.socket_port': CONFIG.get('port', 8000),
-        'server.thread_pool': CONFIG.get('threads', 30),
-        'log.screen': False,
-        'log.access_file': '',
-        'log.error_file': '',
-    }
+sentry_init(
+    dsn=("https://55b5dd780bc14f4c96bba69b7a9abbcc:449af483bd0745"
+         "0d83be640d834e5458@sentry.services.beryju.org/8"),
+    integrations=[
+        DjangoIntegration(),
+        CeleryIntegration(),
+        LoggingIntegration(
+            level=logging.INFO,
+            event_level=logging.ERROR
+        )
+    ],
+    send_default_pii=True
+)
 
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/2.1/howto/static-files/
 
 STATIC_URL = '/static/'
+STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'
 
-LOG_HANDLERS = ['console', 'syslog', 'file', 'sentry']
+LOG_HANDLERS = ['console', 'syslog', 'file']
 
 with CONFIG.cd('log'):
     LOGGING = {
@@ -260,10 +268,6 @@ with CONFIG.cd('log'):
                 'class': 'logging.StreamHandler',
                 'formatter': 'color',
             },
-            'sentry': {
-                'level': 'ERROR',
-                'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler',
-            },
             'syslog': {
                 'level': CONFIG.get('level').get('file'),
                 'class': 'logging.handlers.SysLogHandler',
@@ -309,6 +313,11 @@ with CONFIG.cd('log'):
                 'level': 'DEBUG',
                 'propagate': True,
             },
+            'daphne': {
+                'handlers': LOG_HANDLERS,
+                'level': 'DEBUG',
+                'propagate': True,
+            }
         }
     }
 

passbook/core/tasks.py

@@ -1,11 +1,16 @@
 """passbook core tasks"""
+from datetime import datetime
+from logging import getLogger
+
 from django.core.mail import EmailMultiAlternatives
 from django.template.loader import render_to_string
 from django.utils.html import strip_tags
 
 from passbook.core.celery import CELERY_APP
+from passbook.core.models import Nonce
 from passbook.lib.config import CONFIG
 
+LOGGER = getLogger(__name__)
 
 @CELERY_APP.task()
 def send_email(to_address, subject, template, context):
@@ -15,3 +20,9 @@ def send_email(to_address, subject, template, context):
     msg = EmailMultiAlternatives(subject, text_content, CONFIG.y('email.from'), [to_address])
     msg.attach_alternative(html_content, "text/html")
     msg.send()
+
+@CELERY_APP.task()
+def clean_nonces():
+    """Remove expired nonces"""
+    amount, _ = Nonce.objects.filter(expires__lt=datetime.now(), expiring=True).delete()
+    LOGGER.debug("Deleted expired %d nonces", amount)

passbook/core/urls.py

@@ -17,7 +17,7 @@ admin.site.login = RedirectView.as_view(pattern_name='passbook_core:auth-login')
 handler400 = error.BadRequestView.as_view()
 handler403 = error.ForbiddenView.as_view()
 handler404 = error.NotFoundView.as_view()
-handler500 = error.BadRequestView.as_view()
+handler500 = error.ServerErrorView.as_view()
 
 core_urls = [
     # Authentication views

passbook/core/views/error.py

@@ -58,3 +58,8 @@ class ServerErrorView(TemplateView):
     extra_context = {
         'is_login': True
     }
+
+    # pylint: disable=useless-super-delegation
+    def dispatch(self, *args, **kwargs):
+        """Little wrapper so django accepts this function"""
+        return super().dispatch(*args, **kwargs)

passbook/core/wsgi.py

@@ -10,8 +10,8 @@ https://docs.djangoproject.com/en/2.1/howto/deployment/wsgi/
 import os
 
 from django.core.wsgi import get_wsgi_application
-from raven.contrib.django.raven_compat.middleware.wsgi import Sentry
+from sentry_sdk.integrations.wsgi import SentryWsgiMiddleware
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'passbook.settings')
 
-application = Sentry(get_wsgi_application())
+application = SentryWsgiMiddleware(get_wsgi_application())

passbook/hibp_policy/__init__.py

@@ -1,2 +1,2 @@
 """passbook hibp_policy"""
-__version__ = '0.1.27-beta'
+__version__ = '0.1.29-beta'

passbook/ldap/__init__.py

@@ -1,2 +1,2 @@
 """Passbook ldap app Header"""
-__version__ = '0.1.27-beta'
+__version__ = '0.1.29-beta'

passbook/lib/__init__.py

@@ -1,2 +1,2 @@
 """passbook lib"""
-__version__ = '0.1.27-beta'
+__version__ = '0.1.29-beta'

passbook/lib/config.py

@@ -8,6 +8,7 @@ from typing import Any
 
 import yaml
 from django.conf import ImproperlyConfigured
+from django.utils.autoreload import autoreload_started
 
 SEARCH_PATHS = [
     'passbook/lib/default.yml',
@@ -21,6 +22,8 @@ ENVIRONMENT = os.getenv('PASSBOOK_ENV', 'local')
 class ConfigLoader:
     """Search through SEARCH_PATHS and load configuration"""
 
+    loaded_file = []
+
     __config = {}
     __context_default = None
     __sub_dicts = []
@@ -69,6 +72,8 @@ class ConfigLoader:
             with open(path) as file:
                 try:
                     self.update(self.__config, yaml.safe_load(file))
+                    LOGGER.debug("Loaded %s", path)
+                    self.loaded_file.append(path)
                 except yaml.YAMLError as exc:
                     raise ImproperlyConfigured from exc
         except PermissionError as exc:
@@ -126,3 +131,10 @@ class ConfigLoader:
 
 
 CONFIG = ConfigLoader()
+
+# pylint: disable=unused-argument
+def signal_handler(sender, **kwargs):
+    """Add all loaded config files to autoreload watcher"""
+    for path in CONFIG.loaded_file:
+        sender.watch_file(path)
+autoreload_started.connect(signal_handler)

passbook/lib/default.yml

@@ -35,6 +35,8 @@ redis: localhost/0
 error_report_enabled: true
 secret_key: 9$@r!d^1^jrn#fk#1#@ks#9&i$^s#1)_13%$rwjrhd=e8jfi_s
 
+domains:
+    - passbook.local
 primary_domain: 'localhost'
 
 passbook:

passbook/oauth_client/__init__.py

@@ -1,2 +1,2 @@
 """passbook oauth_client Header"""
-__version__ = '0.1.27-beta'
+__version__ = '0.1.29-beta'

passbook/oauth_provider/__init__.py

@@ -1,2 +1,2 @@
 """passbook oauth_provider Header"""
-__version__ = '0.1.27-beta'
+__version__ = '0.1.29-beta'

passbook/otp/__init__.py

@@ -1,2 +1,2 @@
 """passbook otp Header"""
-__version__ = '0.1.27-beta'
+__version__ = '0.1.29-beta'

passbook/password_expiry_policy/__init__.py

@@ -1,2 +1,2 @@
 """passbook password_expiry"""
-__version__ = '0.1.27-beta'
+__version__ = '0.1.29-beta'

passbook/saml_idp/__init__.py

@@ -1,2 +1,2 @@
 """passbook saml_idp Header"""
-__version__ = '0.1.27-beta'
+__version__ = '0.1.29-beta'

passbook/suspicious_policy/__init__.py

@@ -1,2 +1,2 @@
 """passbook suspicious_policy"""
-__version__ = '0.1.27-beta'
+__version__ = '0.1.29-beta'