From 8924157c92fff9827c83585600197299712fbf5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20R=2E=20Sede=C3=B1o?= Date: Thu, 15 Jul 2021 14:02:10 -0400 Subject: [PATCH] Update dependencies, moving to ASGI3 ASGI3: - Update application routing and middleware to new standards. - asgiref's `sync_to_async` now defaults to `thread_sensitive=True` Set `thread_sensitive=False` on blocking calls like mp.Event.wait() to prevent deadlocks. Django 3.2: - set DEFAULT_AUTO_FIELD to its effective old value. At some point, a migration should be written to migrate automatic primary keys from AutoField to AutoBigField. requirements.txt: - lots of updates - drop ipython and dependencies; it's useful, but not required. PyJWT: - update exception names subscribers: - Update _ChannelLayerMixin based on changes in Django Channels. --- .pylintrc | 1 + requirements.txt | 102 ++++++++++++++------------------ roost_backend/authentication.py | 6 +- roost_backend/models.py | 5 +- roost_backend/subscribers.py | 13 ++-- roost_ng/asgi.py | 25 ++++---- roost_ng/middleware.py | 16 +++++ roost_ng/routing.py | 18 +----- roost_ng/settings/django.py | 6 +- 9 files changed, 94 insertions(+), 98 deletions(-) create mode 100644 roost_ng/middleware.py diff --git a/.pylintrc b/.pylintrc index b555262..56615d4 100644 --- a/.pylintrc +++ b/.pylintrc @@ -29,6 +29,7 @@ limit-inference-results=100 # List of plugins (as comma separated values of python module names) to load, # usually to register additional checkers. load-plugins=pylint_django +django-settings-module=roost_ng.settings # Pickle collected data for later comparisons. persistent=yes diff --git a/requirements.txt b/requirements.txt index 45e660a..ce5e120 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,69 +1,55 @@ aioredis==1.3.1 -asgiref==3.2.10 -astroid==2.4.2 +asgiref==3.4.1 +astroid==2.6.2 async-timeout==3.0.1 -attrs==20.2.0 -autobahn==20.12.3 +attrs==21.2.0 +autobahn==21.3.1 Automat==20.2.0 -autopep8==1.5.4 -backcall==0.2.0 -cffi==1.14.2 -channels==2.4.0 -channels-redis==3.1.0 +cffi==1.14.6 +channels==3.0.4 +channels-redis==3.3.0 constantly==15.1.0 -cryptography==3.3.2 -Cython==0.29.21 -daphne==2.5.0 -decorator==4.4.2 -Django==3.1.8 -django-cors-headers==3.5.0 -djangorestframework==3.11.2 +cryptography==3.4.7 +Cython==0.29.24 +daphne==3.0.2 +decorator==5.0.9 +Django==3.2.5 +django-cors-headers==3.7.0 +djangorestframework==3.12.4 djangorestframework-camel-case==1.2.0 -entrypoints==0.3 -flake8==3.8.3 -gssapi==1.6.9 -hiredis==1.1.0 -hyperlink==20.0.1 -idna==2.10 -importlib-metadata==1.7.0 -incremental==17.5.0 -ipython==7.18.1 -ipython-genutils==0.2.0 -isort==5.5.1 -jedi==0.17.2 -lazy-object-proxy==1.4.3 +flake8==3.9.2 +gssapi==1.6.14 +hiredis==2.0.0 +hyperlink==21.0.0 +idna==3.2 +importlib-metadata==4.6.1 +incremental==21.3.0 +isort==5.9.2 +lazy-object-proxy==1.6.0 mccabe==0.6.1 -msgpack==1.0.0 -parso==0.7.1 -pexpect==4.8.0 -pickleshare==0.7.5 -prompt-toolkit==3.0.7 -ptyprocess==0.6.0 +msgpack==1.0.2 pyasn1==0.4.8 pyasn1-modules==0.2.8 -pycodestyle==2.6.0 +pycodestyle==2.7.0 pycparser==2.20 -pyflakes==2.2.0 -Pygments==2.7.4 -PyHamcrest==2.0.2 -PyJWT==1.7.1 -PyYAML==5.4 -pylint==2.6.0 -pylint-django==2.3.0 +pyflakes==2.3.1 +PyJWT==2.1.0 +pylint==2.9.3 +pylint-django==2.4.4 pylint-plugin-utils==0.6 -pyOpenSSL==19.1.0 -pytz==2020.1 -git+git://github.com/asedeno/python-zephyr@roost-fork#PyZephyr -service-identity==18.1.0 -setproctitle==1.1.10 -six==1.15.0 -sqlparse==0.3.1 -toml==0.10.1 -traitlets==5.0.4 -Twisted==20.3.0 -txaio==20.4.1 -typed-ast==1.4.1 -wcwidth==0.2.5 +pyOpenSSL==20.0.1 +pytz==2021.1 +PyYAML==5.4.1 +PyZephyr @ git+git://github.com/asedeno/python-zephyr@3c4edc9c02bc023c55a625b686af86f3d226a505 +service-identity==21.1.0 +setproctitle==1.2.2 +six==1.16.0 +sqlparse==0.4.1 +toml==0.10.2 +Twisted==21.2.0 +txaio==21.2.1 +typed-ast==1.4.3 +typing-extensions==3.10.0.0 wrapt==1.12.1 -zipp==3.1.0 -zope.interface==5.1.0 +zipp==3.5.0 +zope.interface==5.4.0 diff --git a/roost_backend/authentication.py b/roost_backend/authentication.py index f260c54..5ca89fc 100644 --- a/roost_backend/authentication.py +++ b/roost_backend/authentication.py @@ -23,11 +23,11 @@ def authenticate(self, request): def _decode_token(token): try: return jwt.decode(token, secrets.AUTHTOKEN_KEY, algorithms=['HS256']) - except jwt.DecodeError: + except (jwt.DecodeError, jwt.InvalidSignatureError): return None - except jwt.ExpiredSignature as exc: + except jwt.ExpiredSignatureError as exc: raise exceptions.AuthenticationFailed('Expired token') from exc - except jwt.InvalidAudience as exc: + except jwt.InvalidAudienceError as exc: raise exceptions.AuthenticationFailed('Invalid token') from exc @classmethod diff --git a/roost_backend/models.py b/roost_backend/models.py index 1be5a04..2506b35 100644 --- a/roost_backend/models.py +++ b/roost_backend/models.py @@ -31,7 +31,7 @@ def get_auth_token_dict(self, **claims): 'exp': exp, }) return { - 'auth_token': jwt.encode(claims, secrets.AUTHTOKEN_KEY, algorithm='HS256').decode('utf-8'), + 'auth_token': jwt.encode(claims, secrets.AUTHTOKEN_KEY, algorithm='HS256'), # Since the JWT token will have expiration to the second, drop fractional parts of the timestamp. 'expires': int(exp.timestamp()) * 1000, } @@ -167,11 +167,12 @@ def _d(octets: bytes) -> str: return octets.decode('utf-8') if notice._charset == b'ISO-8859-1': return octets.decode('latin-1') - for enc in ('ascii', 'utf-8', 'latin-1'): + for enc in ('ascii', 'utf-8'): try: return octets.decode(enc) except UnicodeDecodeError: pass + return octets.decode('latin-1') msg = cls() msg.zclass = _d(notice.cls) diff --git a/roost_backend/subscribers.py b/roost_backend/subscribers.py index 7102047..af9f9a3 100644 --- a/roost_backend/subscribers.py +++ b/roost_backend/subscribers.py @@ -9,6 +9,7 @@ from asgiref.sync import sync_to_async, async_to_sync import channels.consumer +from channels import DEFAULT_CHANNEL_LAYER from channels.db import database_sync_to_async import channels.layers import channels.utils @@ -46,6 +47,8 @@ class _ChannelLayerMixin: groups to subscribe to. Then start a task to run the `channel_layer_handler`, cancel it when you want to stop. This may be worth extracting to a utility module.""" + channel_layer_alias = DEFAULT_CHANNEL_LAYER + def __init__(self): super().__init__() self.channel_layer = None @@ -64,7 +67,7 @@ async def channel_layer_resubscribe(self): async def channel_layer_handler(self): # Initialize channel layer.' - self.channel_layer = channels.layers.get_channel_layer() + self.channel_layer = channels.layers.get_channel_layer(self.channel_layer_alias) self.channel_name = await self.channel_layer.new_channel() self.channel_receive = functools.partial(self.channel_layer.receive, self.channel_name) @@ -190,7 +193,7 @@ async def zephyr_handler(self): await self.load_user_data() # No need to start looking for incoming messages until we have initialized zephyr. - await sync_to_async(self.z_initialized.wait)() + await sync_to_async(self.z_initialized.wait, thread_sensitive=False)() zephyr_fd = _zephyr.getFD() loop.add_reader(zephyr_fd, receive_event.set) _LOGGER.debug('[%s] zephyr handler now receiving...', self.log_prefix) @@ -459,7 +462,7 @@ async def oversee(self): await asyncio.sleep(0) _LOGGER.debug('[OVERSEER] waiting for stop event...') - await sync_to_async(self.stop_event.wait)() + await sync_to_async(self.stop_event.wait, thread_sensitive=False)() _LOGGER.debug('[OVERSEER] received stop event...') if self.server_stop_event: @@ -571,7 +574,7 @@ async def run(self): _LOGGER.debug('[%s] announced.', self.log_prefix) await asyncio.wait( [ - sync_to_async(self.stop_event.wait)(), + sync_to_async(self.stop_event.wait, thread_sensitive=False)(), zephyr_task, channel_task, ], @@ -641,7 +644,7 @@ async def run(self): channel_task = asyncio.create_task(self.channel_layer_handler()) await asyncio.wait( [ - sync_to_async(self.stop_event.wait)(), + sync_to_async(self.stop_event.wait, thread_sensitive=False)(), zephyr_task, channel_task, ], diff --git a/roost_ng/asgi.py b/roost_ng/asgi.py index 2a90358..ba86c6f 100644 --- a/roost_ng/asgi.py +++ b/roost_ng/asgi.py @@ -9,21 +9,18 @@ import os -from channels.routing import get_default_application +from channels.routing import ProtocolTypeRouter, URLRouter +from django.core.asgi import get_asgi_application import django os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'roost_ng.settings') django.setup() -_application = get_default_application() - - -def application(scope): - if scope['type'] == 'websocket': - # Daphne does not deal with the daphne-root-path header for websockets, - # so we will deal with it here. - headers = dict(scope['headers']) - root_path = headers.get(b'daphne-root-path', b'').decode() - path = scope['path'] - if root_path and path.startswith(root_path): - scope['path'] = path[len(root_path):] - return _application(scope) + +import roost_ng.routing +from roost_ng.middleware import DaphneRootPathForWebsockets + + +application = ProtocolTypeRouter({ + "http": get_asgi_application(), + "websocket": DaphneRootPathForWebsockets(URLRouter(roost_ng.routing.websocket_urlpatterns)), +}) diff --git a/roost_ng/middleware.py b/roost_ng/middleware.py new file mode 100644 index 0000000..0474d96 --- /dev/null +++ b/roost_ng/middleware.py @@ -0,0 +1,16 @@ +from channels.middleware import BaseMiddleware + + +class DaphneRootPathForWebsockets(BaseMiddleware): + # pylint: disable=too-few-public-methods + async def __call__(self, scope, receive, send): + # Copy scope to stop changes going upstream + scope = dict(scope) + # Daphne does not deal with the daphne-root-path header for websockets, + # so we will deal with it here. + headers = dict(scope['headers']) + root_path = headers.get(b'daphne-root-path', b'').decode() + path = scope['path'] + if root_path and path.startswith(root_path): + scope['path'] = path[len(root_path):] + return await self.inner(scope, receive, send) diff --git a/roost_ng/routing.py b/roost_ng/routing.py index 5907c75..f0f97ff 100644 --- a/roost_ng/routing.py +++ b/roost_ng/routing.py @@ -1,19 +1,7 @@ from django.conf.urls import url -from channels.routing import ProtocolTypeRouter, URLRouter from roost_backend.consumers import UserSocketConsumer -# from roost_backend.middleware import JWTAuthTokenMiddleware -# pylint: disable=invalid-name -application = ProtocolTypeRouter({ - # Empty for now (http->django views is added by default) - # This would be cool if roost did websocket auth by header. - # 'websocket': JWTAuthTokenMiddleware( - # URLRouter([ - # url(r'^v1/socket/websocket', UserSocketConsumer), - # ]) - # ), - 'websocket': URLRouter([ - url(r'^v1/socket/websocket', UserSocketConsumer), - ]), -}) +websocket_urlpatterns = [ + url(r'^v1/socket/websocket', UserSocketConsumer.as_asgi()), +] diff --git a/roost_ng/settings/django.py b/roost_ng/settings/django.py index d768a75..386c6b5 100644 --- a/roost_ng/settings/django.py +++ b/roost_ng/settings/django.py @@ -51,7 +51,7 @@ ROOT_URLCONF = 'roost_ng.urls' WSGI_APPLICATION = 'roost_ng.wsgi.application' -ASGI_APPLICATION = 'roost_ng.routing.application' +ASGI_APPLICATION = 'roost_ng.asgi.application' # Password validation @@ -93,3 +93,7 @@ STATIC_URL = '/static/' APPEND_SLASH = False + + +# New in Django 3.2; TODO: write migration to AutoBigField +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'