From 3c5ec422ace644d848d2f845b0f3ef8de73462ef Mon Sep 17 00:00:00 2001 From: mar77i Date: Sun, 9 Jun 2024 15:43:04 +0200 Subject: [PATCH] big cleanup and refactoring #1 --- hub/app.py | 74 ++++------- hub/hubapp.py | 232 +++++++++++++++++++++-------------- hub/utils.py | 37 ++++-- hub/websocket.py | 50 ++------ requirements.txt | 25 ++-- setup_venv.sh | 4 +- webroot/common.js | 4 +- webroot/first/index.html.j2 | 2 +- webroot/first/index.js | 9 +- webroot/first/master.html.j2 | 2 +- webroot/first/style.css | 4 + webroot/index.html.j2 | 14 +-- webroot/style.css | 5 + 13 files changed, 244 insertions(+), 218 deletions(-) create mode 100644 webroot/first/style.css diff --git a/hub/app.py b/hub/app.py index 22a7cd4..2f1b849 100644 --- a/hub/app.py +++ b/hub/app.py @@ -1,11 +1,7 @@ import socket import sys from argparse import ArgumentParser -from base64 import urlsafe_b64encode -from hashlib import pbkdf2_hmac -from itertools import chain from pathlib import Path -from secrets import token_urlsafe from subprocess import run from traceback import print_exception from urllib.parse import urlunsplit @@ -15,7 +11,7 @@ from falcon.constants import MEDIA_HTML from jinja2 import Environment, FileSystemLoader, select_autoescape from uvicorn import Config, Server -from .hubapp import HubApp, RootApp +from .hubapp import RootApp, HubApp class App(FalconApp): @@ -28,25 +24,15 @@ class App(FalconApp): autoescape=select_autoescape(), extensions=["hub.utils.StaticTag"], ) - self.secret = secret or token_urlsafe(64) - self.hubapps = {} - RootApp(self, self.base_dir) + self.hubapps = {"root": RootApp(self, self.base_dir, "/derp", secret)} for base_dir in self.base_dir.iterdir(): - if not base_dir.is_dir() or HubApp.is_ignored_filename(base_dir): + if not base_dir.is_dir() or RootApp.is_ignored_filename(base_dir): continue - HubApp(self, base_dir) + self.hubapps[base_dir.name] = HubApp( + self, base_dir, base_uri=f"/derp/{base_dir.name}" + ) self.add_error_handler(Exception, self.print_exception) - def scramble(self, value): - if isinstance(value, str): - value = value.encode() - secret = self.secret - if isinstance(secret, str): - secret = secret.encode() - return urlsafe_b64encode( - pbkdf2_hmac("sha512", value, secret, 221100) - ).rstrip(b"=").decode("ascii") - async def print_exception(self, req, resp, ex, params): print_exception(*sys.exc_info()) @@ -58,46 +44,30 @@ class HubServer(Server): async def startup(self, sockets: list[socket.socket] | None = None) -> None: await super().startup(sockets) - config = self.config - protocol_name = "https" if config.ssl else "http" - host = "0.0.0.0" if config.host is None else config.host - if ":" in host: - # It's an IPv6 address. - host = f"[{host.rstrip(']').lstrip('[')}]" - - port = config.port - if port == 0: - try: - port = next( - chain.from_iterable( - (server.sockets for server in getattr(self, "servers", ())) - ) - ).getsockname()[1] - except StopIteration: - pass - if {"http": 80, "https": 443}[protocol_name] != port: + root_app = self.config.loaded_app.app.hubapps["root"] + print("Secret:", root_app.secret) + for uri, file in root_app.files_per_uris.items(): + if file.name == "index.html": + break + else: + raise ValueError("Root page not found!") + host, port, ssl = self.config.host, self.config.port, bool(self.config.ssl) + if port and port != (80, 443)[ssl]: host = f"{host}:{port}" - app = config.loaded_app.app - print("Secret:", app.secret) - for key, value in app.hubapps["root"].files_per_uris.items(): - if Path(value.path.name).stem == "index.html": - url = urlunsplit((protocol_name, host, key, "", "")) - print("URL:", url) - if self.browser: - run([self.browser, url]) - - -app: App + url = urlunsplit((f"http{'s'[:ssl]}", host, uri, "", "")) + print("URL:", url) + if self.browser: + run([self.browser, url]) async def main(): - global app ap = ArgumentParser() ap.add_argument("--secret") - ap.add_argument("--browser", default="firefox") + ap.add_argument("--browser", default="xdg-open") args = ap.parse_args() app = App(args.secret) - config = Config("hub.app:app", port=5000, log_level="info") + await app.hubapps["root"].setup() + config = Config(app, port=5000, log_level="info") config.setup_event_loop() hs = HubServer(config, browser=args.browser) await hs.serve() diff --git a/hub/hubapp.py b/hub/hubapp.py index f3c457c..591856c 100644 --- a/hub/hubapp.py +++ b/hub/hubapp.py @@ -1,14 +1,22 @@ +from base64 import urlsafe_b64encode +from hashlib import pbkdf2_hmac from pathlib import Path +from secrets import token_urlsafe from falcon.constants import MEDIA_HTML, MEDIA_JS, MEDIA_TEXT from falcon.status_codes import HTTP_OK from jinja2 import Template +from redis.asyncio import StrictRedis -from .websocket import WebSocketHub +from .websocket import WebSocketApp MEDIA_CSS = "text/css" + class StaticFile: + """ + Basic static file wrapper. + """ media_types_per_suffix = { ".html": MEDIA_HTML, ".js": MEDIA_JS, @@ -19,6 +27,7 @@ class StaticFile: def __init__(self, path): self.path = path + self.name = path.name self.media_type = self.get_media_type(path) self.mtime = None self.content = None @@ -39,14 +48,80 @@ class StaticFile: return f"<{type(self).__name__} {self.path}>" +class TreeFileApp: + """ + Map a directory tree base_dir to a base_uri and serve it statically. + Map index.html files to their relative roots: + index.html to "/" (root uri) to index.html and + "/a" (directory uri) to a/index.html + """ + @staticmethod + def is_ignored_filename(path): + return path.name.startswith(".") + + @classmethod + def scan_files(cls, base_dir): + stack = [base_dir.iterdir()] + while len(stack): + try: + path = next(stack[-1]) + except StopIteration: + stack.pop() + continue + if path.is_dir(): + stack.append(path.iterdir()) + elif not cls.is_ignored_filename(path): + yield path + + def get_file(self, path: Path) -> StaticFile: + return StaticFile(path) + + def uri_tail(self, path: Path) -> str: + """ + Return the "local" path, relative to self.base_dir, if applicable + """ + assert isinstance(path, Path) + if path.is_absolute(): + path = path.relative_to(self.base_dir) + return str(path) + + def uri(self, path: Path) -> str: + uri_tail = self.uri_tail(path) + if uri_tail == "index.html": + return self.base_uri or "/" + index_suffix = "/index.html" + if uri_tail.endswith(index_suffix): + uri_tail = uri_tail[:-len(index_suffix)] + return f"{self.base_uri}/{uri_tail.lstrip('/')}" + + def __init__(self, app, base_dir, base_uri="/"): + self.app = app + self.base_dir = base_dir + self.base_uri = base_uri.rstrip("/") + self.name = self.base_uri.replace("/", ".").strip(".") or "root" + self.files_per_uris = {} + for path in self.scan_files(base_dir): + static_file = self.get_file(path) + uri = self.uri(static_file.path) + self.files_per_uris[uri] = static_file + app.add_route(uri, self) + + async def on_get(self, req, resp): + resource = self.files_per_uris[req.path] + resp.content_type = resource.media_type + resp.data = resource.get().encode() + resp.status = HTTP_OK + + class StaticTemplateFile(StaticFile): TEMPLATE_SUFFIX = ".j2" content: Template | None def __init__(self, path, hubapp): super().__init__(path) + self.name = path.stem self.hubapp = hubapp - self.context = {"hubapp": self.hubapp} + self.context = {"static_file": self} def get(self) -> str: mtime = self.path.stat().st_mtime @@ -64,97 +139,76 @@ class StaticTemplateFile(StaticFile): return cls.media_types_per_suffix[path.suffixes[-2]] -class BaseHubApp: - SCAN_FILES_RECURSIVELY = True - - def __init__(self, app, base_dir, name=None): - self.app = app - self.base_dir = base_dir - self.name = name if name is not None else base_dir.name - self.app.hubapps[self.name] = self - self.files_per_uris = { - self.uri_from(file.path): file for file in self.scan_files(base_dir) - } - for uri in self.files_per_uris: - app.add_route(uri, self) +class TemplateTreeFileApp(TreeFileApp): + def get_file(self, path: Path) -> StaticFile: + if path.suffix == StaticTemplateFile.TEMPLATE_SUFFIX: + return StaticTemplateFile(path, self) + return super().get_file(path) - @staticmethod - def is_master_uri(uri_tail): - slash = "/" - start = uri_tail.rfind(slash) - pos = uri_tail.find(".", start if start >= 0 else 0) - return ( - uri_tail[start + len(slash):pos] == "master" - or "master" in uri_tail[:start].split(slash) - ) - - def _uri_tail(self, path, suffix): - if isinstance(path, Path): - if path.is_absolute(): - path = path.relative_to(self.base_dir) - uri_tail = str(path) - else: - uri_tail = str(path) - if uri_tail.endswith(suffix): - uri_tail = uri_tail[:-len(suffix)] - - if self.is_master_uri(uri_tail): - return self.app.scramble(uri_tail) - elif uri_tail == "index.html": - return "" + def uri_tail(self, path: Path) -> str: + uri_tail = super().uri_tail(path) + if uri_tail.endswith(StaticTemplateFile.TEMPLATE_SUFFIX): + uri_tail = uri_tail[:-len(StaticTemplateFile.TEMPLATE_SUFFIX)] return uri_tail - def uri_from(self, path) -> str | Path: - uri_tail = self._uri_tail(path, StaticTemplateFile.TEMPLATE_SUFFIX) - name = self.name - if name == "root": - name = "" - return f"/{name}{'/' if name and uri_tail else ''}{uri_tail}" - - def scan_files(self, base_dir): - stack = [base_dir.iterdir()] - while len(stack): - try: - path = next(stack[-1]) - except StopIteration: - stack.pop() - continue - if path.is_dir(): - if self.SCAN_FILES_RECURSIVELY: - stack.append(path.iterdir()) - elif not self.is_ignored_filename(path): - if path.suffix == StaticTemplateFile.TEMPLATE_SUFFIX: - static_file = StaticTemplateFile(path, self) - else: - static_file = StaticFile(path) - yield static_file - - @staticmethod - def is_ignored_filename(path): - return path.name.startswith(".") - - async def on_get(self, req, resp): - resource = self.files_per_uris[req.path] - resp.content_type = resource.media_type - resp.data = resource.get().encode() - resp.status = HTTP_OK - -class HubApp(BaseHubApp): - def __init__(self, app, base_dir): - super().__init__(app, base_dir) - self.wsh = WebSocketHub(self) - self.ws_uris = {self.uri_from(f"ws_{value}"): value for value in ("client", "master")} +class RootApp(TemplateTreeFileApp): + @classmethod + def scan_files(cls, base_dir): + for path in base_dir.iterdir(): + if not path.is_dir() and not cls.is_ignored_filename(path): + yield path + + def __init__(self, app, base_dir, base_uri="/", secret=None): + from .utils import get_redis_pass + + self.secret = secret or token_urlsafe(64) + super().__init__(app, base_dir, base_uri) + self.conn = StrictRedis(username="default", password=get_redis_pass("/etc/redis/redis.conf")) + + async def setup(self): + await self.conn.set("client_id", 0) + + def scramble(self, value): + if isinstance(value, str): + value = value.encode() + secret = self.secret + if isinstance(secret, str): + secret = secret.encode() + return urlsafe_b64encode( + pbkdf2_hmac("sha512", value, secret, 221100) + ).rstrip(b"=").decode("ascii") + + def uri_tail(self, path: Path) -> str: + return self.scramble(super().uri_tail(path)) + + +class HubApp(TemplateTreeFileApp): + def __init__(self, app, base_dir, base_uri="/"): + super().__init__(app, base_dir, base_uri) + self.ws_app = WebSocketApp(self) + self.ws_uris = { + self.uri(Path(f"ws_{value}")): value for value in ("client", "master") + } for uri, suffix in self.ws_uris.items(): - app.add_route(uri, self.wsh, suffix=suffix) - - -class RootApp(BaseHubApp): - SCAN_FILES_RECURSIVELY = False - - def __init__(self, app, base_dir): - super().__init__(app, base_dir, "root") + app.add_route(uri, self.ws_app, suffix=suffix) @staticmethod - def is_master_uri(uri_tail): - return True + def is_master_uri(path: Path) -> bool: + assert isinstance(path, Path) + basename = path.name + pos = basename.find(".") + if pos != -1: + basename = basename[:pos] + return basename in "master" + + def uri(self, path: Path) -> str: + uri = super().uri(path) + if self.is_master_uri(path): + pos = uri.rstrip("/").rfind("/") + scrambled_uri = self.app.hubapps["root"].scramble(uri) + if pos == -1: + uri = scrambled_uri + else: + uri = f"{uri[:pos]}/{scrambled_uri}" + return uri diff --git a/hub/utils.py b/hub/utils.py index ae448ed..b66fd97 100644 --- a/hub/utils.py +++ b/hub/utils.py @@ -1,24 +1,39 @@ -from pathlib import Path - from jinja2_simple_tags import StandaloneTag +from .hubapp import TreeFileApp class StaticTag(StandaloneTag): tags = {"static"} - def render(self, filename="/", hubapp=None): - if not hubapp: - hubapp = self.context["hubapp"] - elif isinstance(hubapp, str): - hubapp = self.context["hubapp"].app.hubapps[hubapp] - return hubapp.uri_from(Path(filename)) + def render( + self, filename: str = "", hubapp: str | TreeFileApp | None = None + ): + """ + If filename starts with '/', interpret the path as relative to hubapp.base_dir, + otherwise assume the path is relative to the current file. + """ + static_file = self.context["static_file"] + h = static_file.hubapp + if isinstance(hubapp, str): + h = h.app.hubapps[hubapp] + elif isinstance(hubapp, TreeFileApp): + h = hubapp + del hubapp + if filename.startswith("/") or h != static_file.hubapp: + path = h.base_dir / filename.lstrip("/") + else: + path = static_file.path.parent / filename + return h.uri(path) def get_redis_pass(redis_conf): + """ + Poor man's redis credentials: read the password from redis_conf. + Requires redis being configured with a `requirepass` password set. + """ prefix = "requirepass " with open(redis_conf, "rt") as fh: for line in fh: - if not line.startswith(prefix): - continue - return line[len(prefix) :].rstrip() + if line.startswith(prefix): + return line[len(prefix) :].rstrip() return None diff --git a/hub/websocket.py b/hub/websocket.py index 89f1192..0ecf987 100644 --- a/hub/websocket.py +++ b/hub/websocket.py @@ -7,36 +7,18 @@ from functools import partial from traceback import print_exception from falcon import WebSocketDisconnected -from redis.asyncio import StrictRedis -from .utils import get_redis_pass - -class BaseWebSocketHub: - client_ids_sem = asyncio.Semaphore(0) - - @classmethod - def _class_init(cls, redis): - if not hasattr(cls, "_class_init"): - return - delattr(cls, "_class_init") - asyncio.create_task(cls.initialize_client_ids(redis)) - - @classmethod - async def initialize_client_ids(cls, redis): - await redis.set("client_id", 0) - cls.client_ids_sem.release() - - def __init__(self): - self.redis = StrictRedis(password=get_redis_pass("/etc/redis/redis.conf")) - if hasattr(BaseWebSocketHub, "_class_init"): - BaseWebSocketHub._class_init(self.redis) +class BaseWebSocketApp: + def __init__(self, hubapp): + self.hubapp = hubapp + self.conn = self.hubapp.app.hubapps["root"].conn def task_done(self): self.task = None @staticmethod - async def process_websocket(redis, web_socket, extra_data={}, recipients=[]): + async def process_websocket(conn, web_socket, extra_data={}, recipients=[]): try: while True: data = json.loads(await web_socket.receive_text()) @@ -46,7 +28,7 @@ class BaseWebSocketHub: else: current_recipients = recipients for recipient in current_recipients: - await redis.publish(recipient, pickle.dumps(data)) + await conn.publish(recipient, pickle.dumps(data)) except (CancelledError, WebSocketDisconnected): pass @@ -72,7 +54,7 @@ class BaseWebSocketHub: leave_cb=None, ): await web_socket.accept() - pubsub = self.redis.pubsub() + pubsub = self.conn.pubsub() if pubsub_name: await pubsub.subscribe(pubsub_name) if callable(join_cb): @@ -80,7 +62,7 @@ class BaseWebSocketHub: try: await asyncio.gather( self.process_websocket( - self.redis, web_socket, **(process_websockets_kwargs or {}) + self.conn, web_socket, **(process_websockets_kwargs or {}) ), self.process_pubsub(pubsub, web_socket), return_exceptions=True, @@ -95,11 +77,7 @@ class BaseWebSocketHub: await leave_cb() -class WebSocketHub(BaseWebSocketHub): - def __init__(self, hubapp): - super().__init__() - self.hubapp = hubapp - +class WebSocketApp(BaseWebSocketApp): async def join_leave_client_notify(self, redis, action, client_id): await redis.publish( f"{self.hubapp.name}-master", @@ -107,11 +85,7 @@ class WebSocketHub(BaseWebSocketHub): ) async def on_websocket_client(self, req, web_socket): - await self.client_ids_sem.acquire() - try: - client_id = await self.redis.incr("client_id") - finally: - self.client_ids_sem.release() + client_id = await self.conn.incr("client_id") return await self.on_websocket( req, web_socket, @@ -120,8 +94,8 @@ class WebSocketHub(BaseWebSocketHub): "extra_data": {"client_id": client_id}, "recipients": [f"{self.hubapp.name}-master"], }, - partial(self.join_leave_client_notify, self.redis, "join", client_id), - partial(self.join_leave_client_notify, self.redis, "leave", client_id), + partial(self.join_leave_client_notify, self.conn, "join", client_id), + partial(self.join_leave_client_notify, self.conn, "leave", client_id), ) async def on_websocket_master(self, req, web_socket): diff --git a/requirements.txt b/requirements.txt index 9646ad0..95f864d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,24 +1,23 @@ -anyio==4.2.0 -async-timeout==4.0.3 -black==24.2.0 +anyio==4.4.0 +black==24.4.2 click==8.1.7 falcon==3.1.3 h11==0.14.0 httptools==0.6.1 -idna==3.6 +idna==3.7 isort==5.13.2 -Jinja2==3.1.3 -jinja2-simple-tags==0.5.0 -MarkupSafe==2.1.3 +Jinja2==3.1.4 +jinja2-simple-tags==0.6.1 +MarkupSafe==2.1.5 mypy-extensions==1.0.0 -packaging==23.2 +packaging==24.0 pathspec==0.12.1 -platformdirs==4.1.0 +platformdirs==4.2.2 python-dotenv==1.0.1 PyYAML==6.0.1 -redis==5.0.2 -sniffio==1.3.0 -uvicorn==0.27.1 +redis==5.0.5 +sniffio==1.3.1 +uvicorn==0.30.1 uvloop==0.19.0 -watchfiles==0.21.0 +watchfiles==0.22.0 websockets==12.0 diff --git a/setup_venv.sh b/setup_venv.sh index 139fde9..894aad5 100644 --- a/setup_venv.sh +++ b/setup_venv.sh @@ -1,8 +1,8 @@ venv_dir="${VIRTUAL_ENV:-venv}" -if type deactivate &>/dev/null; then +if type -f deactivate &>/dev/null; then deactivate fi -if [[ ! -d "${venv_dir}" ]]; then +if [[ ! -e "${venv_dir}" ]]; then "${PYTHON:-python}" -m venv "${venv_dir}" fi source "${venv_dir}/bin/activate" diff --git a/webroot/common.js b/webroot/common.js index 57ae142..8de282f 100644 --- a/webroot/common.js +++ b/webroot/common.js @@ -4,7 +4,9 @@ var key; for (key in obj) { if (key.substring(0, domconf_event_prefix.length) == domconf_event_prefix) { - domobj.addEventListener(key.substring(domconf_event_prefix.length), obj[key]); + domobj.addEventListener( + key.substring(domconf_event_prefix.length), obj[key] + ); } else { domobj.setAttribute(key, obj[key]) } diff --git a/webroot/first/index.html.j2 b/webroot/first/index.html.j2 index 5a338a5..37011da 100644 --- a/webroot/first/index.html.j2 +++ b/webroot/first/index.html.j2 @@ -5,7 +5,7 @@ Chat client - + diff --git a/webroot/first/index.js b/webroot/first/index.js index 8cae34e..03b1f49 100644 --- a/webroot/first/index.js +++ b/webroot/first/index.js @@ -3,7 +3,9 @@ var input_div = null; function write(msg) { - common.write(input_div.previousSibling, msg); + common.write( + input_div === null ? document.body : input_div.previousSibling, msg + ); } function change_name_button(hub) { @@ -74,7 +76,10 @@ function close() { write("connection lost"); - input_div.remove(); + if (input_div !== null) { + input_div.remove(); + input_div = null; + } } function message(msg) { diff --git a/webroot/first/master.html.j2 b/webroot/first/master.html.j2 index d0ab1f9..5a471de 100644 --- a/webroot/first/master.html.j2 +++ b/webroot/first/master.html.j2 @@ -5,7 +5,7 @@ Chat Master - + diff --git a/webroot/first/style.css b/webroot/first/style.css new file mode 100644 index 0000000..422e654 --- /dev/null +++ b/webroot/first/style.css @@ -0,0 +1,4 @@ +body { + background-color: black; + color: white; +} diff --git a/webroot/index.html.j2 b/webroot/index.html.j2 index 04f9059..ec1f96c 100644 --- a/webroot/index.html.j2 +++ b/webroot/index.html.j2 @@ -10,14 +10,12 @@

App list

diff --git a/webroot/style.css b/webroot/style.css index 4f7171b..5a1a0e6 100644 --- a/webroot/style.css +++ b/webroot/style.css @@ -1,3 +1,8 @@ +body { + background-color: black; + color: white; +} + li span { display: inline-block; width: 7em; -- 2.45.2