Update dependencies, moving to ASGI3

ASGI3:
- Update application routing and middleware to new standards.
- asgiref's `sync_to_async` now defaults to `thread_sensitive=True`
  Set `thread_sensitive=False` on blocking calls like mp.Event.wait()
  to prevent deadlocks.

Django 3.2:
- set DEFAULT_AUTO_FIELD to its effective old value. At some point,
  a migration should be written to migrate automatic primary keys from
  AutoField to AutoBigField.

requirements.txt:
- lots of updates
- drop ipython and dependencies; it's useful, but not required.

PyJWT:
- update exception names

subscribers:
- Update _ChannelLayerMixin based on changes in Django Channels.

.pylintrc

@@ -29,6 +29,7 @@ limit-inference-results=100
 # List of plugins (as comma separated values of python module names) to load,
 # usually to register additional checkers.
 load-plugins=pylint_django
+django-settings-module=roost_ng.settings
 
 # Pickle collected data for later comparisons.
 persistent=yes

requirements.txt

@@ -1,69 +1,55 @@
 aioredis==1.3.1
-asgiref==3.2.10
-astroid==2.4.2
+asgiref==3.4.1
+astroid==2.6.2
 async-timeout==3.0.1
-attrs==20.2.0
-autobahn==20.12.3
+attrs==21.2.0
+autobahn==21.3.1
 Automat==20.2.0
-autopep8==1.5.4
-backcall==0.2.0
-cffi==1.14.2
-channels==2.4.0
-channels-redis==3.1.0
+cffi==1.14.6
+channels==3.0.4
+channels-redis==3.3.0
 constantly==15.1.0
-cryptography==3.3.2
-Cython==0.29.21
-daphne==2.5.0
-decorator==4.4.2
-Django==3.1.8
-django-cors-headers==3.5.0
-djangorestframework==3.11.2
+cryptography==3.4.7
+Cython==0.29.24
+daphne==3.0.2
+decorator==5.0.9
+Django==3.2.5
+django-cors-headers==3.7.0
+djangorestframework==3.12.4
 djangorestframework-camel-case==1.2.0
-entrypoints==0.3
-flake8==3.8.3
-gssapi==1.6.9
-hiredis==1.1.0
-hyperlink==20.0.1
-idna==2.10
-importlib-metadata==1.7.0
-incremental==17.5.0
-ipython==7.18.1
-ipython-genutils==0.2.0
-isort==5.5.1
-jedi==0.17.2
-lazy-object-proxy==1.4.3
+flake8==3.9.2
+gssapi==1.6.14
+hiredis==2.0.0
+hyperlink==21.0.0
+idna==3.2
+importlib-metadata==4.6.1
+incremental==21.3.0
+isort==5.9.2
+lazy-object-proxy==1.6.0
 mccabe==0.6.1
-msgpack==1.0.0
-parso==0.7.1
-pexpect==4.8.0
-pickleshare==0.7.5
-prompt-toolkit==3.0.7
-ptyprocess==0.6.0
+msgpack==1.0.2
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
-pycodestyle==2.6.0
+pycodestyle==2.7.0
 pycparser==2.20
-pyflakes==2.2.0
-Pygments==2.7.4
-PyHamcrest==2.0.2
-PyJWT==1.7.1
-PyYAML==5.4
-pylint==2.6.0
-pylint-django==2.3.0
+pyflakes==2.3.1
+PyJWT==2.1.0
+pylint==2.9.3
+pylint-django==2.4.4
 pylint-plugin-utils==0.6
-pyOpenSSL==19.1.0
-pytz==2020.1
-git+git://github.com/asedeno/python-zephyr@roost-fork#PyZephyr
-service-identity==18.1.0
-setproctitle==1.1.10
-six==1.15.0
-sqlparse==0.3.1
-toml==0.10.1
-traitlets==5.0.4
-Twisted==20.3.0
-txaio==20.4.1
-typed-ast==1.4.1
-wcwidth==0.2.5
+pyOpenSSL==20.0.1
+pytz==2021.1
+PyYAML==5.4.1
+PyZephyr @ git+git://github.com/asedeno/python-zephyr@3c4edc9c02bc023c55a625b686af86f3d226a505
+service-identity==21.1.0
+setproctitle==1.2.2
+six==1.16.0
+sqlparse==0.4.1
+toml==0.10.2
+Twisted==21.2.0
+txaio==21.2.1
+typed-ast==1.4.3
+typing-extensions==3.10.0.0
 wrapt==1.12.1
-zipp==3.1.0
-zope.interface==5.1.0
+zipp==3.5.0
+zope.interface==5.4.0

roost_backend/authentication.py

@@ -23,11 +23,11 @@ def authenticate(self, request):
     def _decode_token(token):
         try:
             return jwt.decode(token, secrets.AUTHTOKEN_KEY, algorithms=['HS256'])
-        except jwt.DecodeError:
+        except (jwt.DecodeError, jwt.InvalidSignatureError):
             return None
-        except jwt.ExpiredSignature as exc:
+        except jwt.ExpiredSignatureError as exc:
             raise exceptions.AuthenticationFailed('Expired token') from exc
-        except jwt.InvalidAudience as exc:
+        except jwt.InvalidAudienceError as exc:
             raise exceptions.AuthenticationFailed('Invalid token') from exc
 
     @classmethod

roost_backend/models.py

@@ -31,7 +31,7 @@ def get_auth_token_dict(self, **claims):
             'exp': exp,
         })
         return {
-            'auth_token': jwt.encode(claims, secrets.AUTHTOKEN_KEY, algorithm='HS256').decode('utf-8'),
+            'auth_token': jwt.encode(claims, secrets.AUTHTOKEN_KEY, algorithm='HS256'),
             # Since the JWT token will have expiration to the second, drop fractional parts of the timestamp.
             'expires': int(exp.timestamp()) * 1000,
         }
@@ -167,11 +167,12 @@ def _d(octets: bytes) -> str:
                 return octets.decode('utf-8')
             if notice._charset == b'ISO-8859-1':
                 return octets.decode('latin-1')
-            for enc in ('ascii', 'utf-8', 'latin-1'):
+            for enc in ('ascii', 'utf-8'):
                 try:
                     return octets.decode(enc)
                 except UnicodeDecodeError:
                     pass
+            return octets.decode('latin-1')
 
         msg = cls()
         msg.zclass = _d(notice.cls)

roost_backend/subscribers.py

@@ -9,6 +9,7 @@
 
 from asgiref.sync import sync_to_async, async_to_sync
 import channels.consumer
+from channels import DEFAULT_CHANNEL_LAYER
 from channels.db import database_sync_to_async
 import channels.layers
 import channels.utils
@@ -46,6 +47,8 @@ class _ChannelLayerMixin:
     groups to subscribe to. Then start a task to run the `channel_layer_handler`, cancel it when you
     want to stop. This may be worth extracting to a utility module."""
 
+    channel_layer_alias = DEFAULT_CHANNEL_LAYER
+
     def __init__(self):
         super().__init__()
         self.channel_layer = None
@@ -64,7 +67,7 @@ async def channel_layer_resubscribe(self):
 
     async def channel_layer_handler(self):
         # Initialize channel layer.'
-        self.channel_layer = channels.layers.get_channel_layer()
+        self.channel_layer = channels.layers.get_channel_layer(self.channel_layer_alias)
         self.channel_name = await self.channel_layer.new_channel()
         self.channel_receive = functools.partial(self.channel_layer.receive, self.channel_name)
 
@@ -190,7 +193,7 @@ async def zephyr_handler(self):
             await self.load_user_data()
 
             # No need to start looking for incoming messages until we have initialized zephyr.
-            await sync_to_async(self.z_initialized.wait)()
+            await sync_to_async(self.z_initialized.wait, thread_sensitive=False)()
             zephyr_fd = _zephyr.getFD()
             loop.add_reader(zephyr_fd, receive_event.set)
             _LOGGER.debug('[%s] zephyr handler now receiving...', self.log_prefix)
@@ -459,7 +462,7 @@ async def oversee(self):
         await asyncio.sleep(0)
 
         _LOGGER.debug('[OVERSEER] waiting for stop event...')
-        await sync_to_async(self.stop_event.wait)()
+        await sync_to_async(self.stop_event.wait, thread_sensitive=False)()
         _LOGGER.debug('[OVERSEER] received stop event...')
 
         if self.server_stop_event:
@@ -571,7 +574,7 @@ async def run(self):
         _LOGGER.debug('[%s] announced.', self.log_prefix)
         await asyncio.wait(
             [
-                sync_to_async(self.stop_event.wait)(),
+                sync_to_async(self.stop_event.wait, thread_sensitive=False)(),
                 zephyr_task,
                 channel_task,
             ],
@@ -641,7 +644,7 @@ async def run(self):
         channel_task = asyncio.create_task(self.channel_layer_handler())
         await asyncio.wait(
             [
-                sync_to_async(self.stop_event.wait)(),
+                sync_to_async(self.stop_event.wait, thread_sensitive=False)(),
                 zephyr_task,
                 channel_task,
             ],

roost_ng/asgi.py

@@ -9,21 +9,18 @@
 
 import os
 
-from channels.routing import get_default_application
+from channels.routing import ProtocolTypeRouter, URLRouter
+from django.core.asgi import get_asgi_application
 import django
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'roost_ng.settings')
 django.setup()
-_application = get_default_application()
-
-
-def application(scope):
-    if scope['type'] == 'websocket':
-        # Daphne does not deal with the daphne-root-path header for websockets,
-        # so we will deal with it here.
-        headers = dict(scope['headers'])
-        root_path = headers.get(b'daphne-root-path', b'').decode()
-        path = scope['path']
-        if root_path and path.startswith(root_path):
-            scope['path'] = path[len(root_path):]
-    return _application(scope)
+
+import roost_ng.routing
+from roost_ng.middleware import DaphneRootPathForWebsockets
+
+
+application = ProtocolTypeRouter({
+    "http": get_asgi_application(),
+    "websocket": DaphneRootPathForWebsockets(URLRouter(roost_ng.routing.websocket_urlpatterns)),
+})

roost_ng/middleware.py

@@ -0,0 +1,16 @@
+from channels.middleware import BaseMiddleware
+
+
+class DaphneRootPathForWebsockets(BaseMiddleware):
+    # pylint: disable=too-few-public-methods
+    async def __call__(self, scope, receive, send):
+        # Copy scope to stop changes going upstream
+        scope = dict(scope)
+        # Daphne does not deal with the daphne-root-path header for websockets,
+        # so we will deal with it here.
+        headers = dict(scope['headers'])
+        root_path = headers.get(b'daphne-root-path', b'').decode()
+        path = scope['path']
+        if root_path and path.startswith(root_path):
+            scope['path'] = path[len(root_path):]
+        return await self.inner(scope, receive, send)

roost_ng/routing.py

@@ -1,19 +1,7 @@
 from django.conf.urls import url
-from channels.routing import ProtocolTypeRouter, URLRouter
 
 from roost_backend.consumers import UserSocketConsumer
-# from roost_backend.middleware import JWTAuthTokenMiddleware
 
-# pylint: disable=invalid-name
-application = ProtocolTypeRouter({
-    # Empty for now (http->django views is added by default)
-    # This would be cool if roost did websocket auth by header.
-    # 'websocket': JWTAuthTokenMiddleware(
-    #     URLRouter([
-    #         url(r'^v1/socket/websocket', UserSocketConsumer),
-    #     ])
-    # ),
-    'websocket': URLRouter([
-        url(r'^v1/socket/websocket', UserSocketConsumer),
-    ]),
-})
+websocket_urlpatterns = [
+    url(r'^v1/socket/websocket', UserSocketConsumer.as_asgi()),
+]

roost_ng/settings/django.py

@@ -51,7 +51,7 @@
 ROOT_URLCONF = 'roost_ng.urls'
 
 WSGI_APPLICATION = 'roost_ng.wsgi.application'
-ASGI_APPLICATION = 'roost_ng.routing.application'
+ASGI_APPLICATION = 'roost_ng.asgi.application'
 
 
 # Password validation
@@ -93,3 +93,7 @@
 STATIC_URL = '/static/'
 
 APPEND_SLASH = False
+
+
+# New in Django 3.2; TODO: write migration to AutoBigField
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'