Merge pull request #508 from Textualize/msgpack-devtools

Msgpack devtools
This commit is contained in:
Will McGugan
2022-05-13 17:08:10 +01:00
committed by GitHub
8 changed files with 125 additions and 72 deletions

46
poetry.lock generated
View File

@@ -381,6 +381,14 @@ mkdocs-autorefs = ">=0.1"
pymdown-extensions = ">=6.3"
pytkdocs = ">=0.14.0"
[[package]]
name = "msgpack"
version = "1.0.3"
description = "MessagePack (de)serializer."
category = "main"
optional = false
python-versions = "*"
[[package]]
name = "multidict"
version = "6.0.2"
@@ -765,7 +773,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-
[metadata]
lock-version = "1.1"
python-versions = "^3.7"
content-hash = "d801e69bdd847115e92104a8cdd51ba1f207a1b7c25c4f6c9fb88434594be975"
content-hash = "05e80f2e4709cbc33327e1ddfaf7ae19a7f708fae251834d631317dc4cf4cd2f"
[metadata.files]
aiohttp = [
@@ -1121,6 +1129,42 @@ mkdocstrings = [
{file = "mkdocstrings-0.17.0-py3-none-any.whl", hash = "sha256:103fc1dd58cb23b7e0a6da5292435f01b29dc6fa0ba829132537f3f556f985de"},
{file = "mkdocstrings-0.17.0.tar.gz", hash = "sha256:75b5cfa2039aeaf3a5f5cf0aa438507b0330ce76c8478da149d692daa7213a98"},
]
msgpack = [
{file = "msgpack-1.0.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:96acc674bb9c9be63fa8b6dabc3248fdc575c4adc005c440ad02f87ca7edd079"},
{file = "msgpack-1.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2c3ca57c96c8e69c1a0d2926a6acf2d9a522b41dc4253a8945c4c6cd4981a4e3"},
{file = "msgpack-1.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0a792c091bac433dfe0a70ac17fc2087d4595ab835b47b89defc8bbabcf5c73"},
{file = "msgpack-1.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c58cdec1cb5fcea8c2f1771d7b5fec79307d056874f746690bd2bdd609ab147"},
{file = "msgpack-1.0.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f97c0f35b3b096a330bb4a1a9247d0bd7e1f3a2eba7ab69795501504b1c2c39"},
{file = "msgpack-1.0.3-cp310-cp310-win32.whl", hash = "sha256:36a64a10b16c2ab31dcd5f32d9787ed41fe68ab23dd66957ca2826c7f10d0b85"},
{file = "msgpack-1.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:c1ba333b4024c17c7591f0f372e2daa3c31db495a9b2af3cf664aef3c14354f7"},
{file = "msgpack-1.0.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c2140cf7a3ec475ef0938edb6eb363fa704159e0bf71dde15d953bacc1cf9d7d"},
{file = "msgpack-1.0.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f4c22717c74d44bcd7af353024ce71c6b55346dad5e2cc1ddc17ce8c4507c6b"},
{file = "msgpack-1.0.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d733a15ade190540c703de209ffbc42a3367600421b62ac0c09fde594da6ec"},
{file = "msgpack-1.0.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7e03b06f2982aa98d4ddd082a210c3db200471da523f9ac197f2828e80e7770"},
{file = "msgpack-1.0.3-cp36-cp36m-win32.whl", hash = "sha256:3d875631ecab42f65f9dce6f55ce6d736696ced240f2634633188de2f5f21af9"},
{file = "msgpack-1.0.3-cp36-cp36m-win_amd64.whl", hash = "sha256:40fb89b4625d12d6027a19f4df18a4de5c64f6f3314325049f219683e07e678a"},
{file = "msgpack-1.0.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6eef0cf8db3857b2b556213d97dd82de76e28a6524853a9beb3264983391dc1a"},
{file = "msgpack-1.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d8c332f53ffff01953ad25131272506500b14750c1d0ce8614b17d098252fbc"},
{file = "msgpack-1.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c0903bd93cbd34653dd63bbfcb99d7539c372795201f39d16fdfde4418de43a"},
{file = "msgpack-1.0.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bf1e6bfed4860d72106f4e0a1ab519546982b45689937b40257cfd820650b920"},
{file = "msgpack-1.0.3-cp37-cp37m-win32.whl", hash = "sha256:d02cea2252abc3756b2ac31f781f7a98e89ff9759b2e7450a1c7a0d13302ff50"},
{file = "msgpack-1.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:2f30dd0dc4dfe6231ad253b6f9f7128ac3202ae49edd3f10d311adc358772dba"},
{file = "msgpack-1.0.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f201d34dc89342fabb2a10ed7c9a9aaaed9b7af0f16a5923f1ae562b31258dea"},
{file = "msgpack-1.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bb87f23ae7d14b7b3c21009c4b1705ec107cb21ee71975992f6aca571fb4a42a"},
{file = "msgpack-1.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a3a5c4b16e9d0edb823fe54b59b5660cc8d4782d7bf2c214cb4b91a1940a8ef"},
{file = "msgpack-1.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f74da1e5fcf20ade12c6bf1baa17a2dc3604958922de8dc83cbe3eff22e8b611"},
{file = "msgpack-1.0.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:73a80bd6eb6bcb338c1ec0da273f87420829c266379c8c82fa14c23fb586cfa1"},
{file = "msgpack-1.0.3-cp38-cp38-win32.whl", hash = "sha256:9fce00156e79af37bb6db4e7587b30d11e7ac6a02cb5bac387f023808cd7d7f4"},
{file = "msgpack-1.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:9b6f2d714c506e79cbead331de9aae6837c8dd36190d02da74cb409b36162e8a"},
{file = "msgpack-1.0.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:89908aea5f46ee1474cc37fbc146677f8529ac99201bc2faf4ef8edc023c2bf3"},
{file = "msgpack-1.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:973ad69fd7e31159eae8f580f3f707b718b61141838321c6fa4d891c4a2cca52"},
{file = "msgpack-1.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da24375ab4c50e5b7486c115a3198d207954fe10aaa5708f7b65105df09109b2"},
{file = "msgpack-1.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a598d0685e4ae07a0672b59792d2cc767d09d7a7f39fd9bd37ff84e060b1a996"},
{file = "msgpack-1.0.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4c309a68cb5d6bbd0c50d5c71a25ae81f268c2dc675c6f4ea8ab2feec2ac4e2"},
{file = "msgpack-1.0.3-cp39-cp39-win32.whl", hash = "sha256:494471d65b25a8751d19c83f1a482fd411d7ca7a3b9e17d25980a74075ba0e88"},
{file = "msgpack-1.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:f01b26c2290cbd74316990ba84a14ac3d599af9cebefc543d241a66e785cf17d"},
{file = "msgpack-1.0.3.tar.gz", hash = "sha256:51fdc7fb93615286428ee7758cecc2f374d5ff363bdd884c7ea622a7a327a81e"},
]
multidict = [
{file = "multidict-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b9e95a740109c6047602f4db4da9949e6c5945cefbad34a1299775ddc9a62e2"},
{file = "multidict-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac0e27844758d7177989ce406acc6a83c16ed4524ebc363c1f748cba184d89d3"},

View File

@@ -28,6 +28,7 @@ rich = "^12.3.0"
click = "8.1.2"
importlib-metadata = "^4.11.3"
typing-extensions = { version = "^4.0.0", python = "<3.8" }
msgpack = "^1.0.3"
[tool.poetry.dev-dependencies]
pytest = "^6.2.3"

View File

@@ -1,11 +1,12 @@
from __future__ import annotations
import asyncio
import base64
import datetime
import inspect
import json
import msgpack
import pickle
from time import time
from asyncio import Queue, Task, QueueFull
from io import StringIO
from typing import Type, Any, NamedTuple
@@ -97,7 +98,7 @@ class DevtoolsClient:
self.update_console_task: Task | None = None
self.console: DevtoolsConsole = DevtoolsConsole(file=StringIO())
self.websocket: ClientWebSocketResponse | None = None
self.log_queue: Queue[str | Type[ClientShutdown]] | None = None
self.log_queue: Queue[str | bytes | Type[ClientShutdown]] | None = None
self.spillover: int = 0
async def connect(self) -> None:
@@ -144,7 +145,10 @@ class DevtoolsClient:
if log is ClientShutdown:
log_queue.task_done()
break
await websocket.send_str(log)
if isinstance(log, str):
await websocket.send_str(log)
else:
await websocket.send_bytes(log)
log_queue.task_done()
self.log_queue_task = asyncio.create_task(send_queued_logs())
@@ -203,17 +207,18 @@ class DevtoolsClient:
segments = self.console.export_segments()
encoded_segments = self._encode_segments(segments)
message = json.dumps(
message: bytes | None = msgpack.packb(
{
"type": "client_log",
"payload": {
"timestamp": int(datetime.datetime.utcnow().timestamp()),
"timestamp": int(time()),
"path": getattr(log.caller, "filename", ""),
"line_number": getattr(log.caller, "lineno", 0),
"encoded_segments": encoded_segments,
"segments": encoded_segments,
},
}
)
assert message is not None
try:
if self.log_queue:
self.log_queue.put_nowait(message)
@@ -233,15 +238,15 @@ class DevtoolsClient:
except QueueFull:
self.spillover += 1
def _encode_segments(self, segments: list[Segment]) -> str:
"""Pickle and Base64 encode the list of Segments
@classmethod
def _encode_segments(cls, segments: list[Segment]) -> bytes:
"""Pickle a list of Segments
Args:
segments (list[Segment]): A list of Segments to encode
Returns:
str: The Segment list pickled with pickle protocol v3, then base64 encoded
bytes: The Segment list pickled with the latest protocol.
"""
pickled = pickle.dumps(segments, protocol=3)
encoded = base64.b64encode(pickled)
return str(encoded, encoding="utf-8")
pickled = pickle.dumps(segments, protocol=4)
return pickled

View File

@@ -1,7 +1,7 @@
from __future__ import annotations
import sys
from datetime import datetime, timezone
from datetime import datetime
from pathlib import Path
from typing import Iterable
@@ -72,19 +72,13 @@ class DevConsoleLog:
def __rich_console__(
self, console: Console, options: ConsoleOptions
) -> RenderResult:
local_time = (
datetime.fromtimestamp(self.unix_timestamp)
.replace(tzinfo=timezone.utc)
.astimezone(tz=datetime.now().astimezone().tzinfo)
)
timezone_name = local_time.tzname()
local_time = datetime.fromtimestamp(self.unix_timestamp)
table = Table.grid(expand=True)
table.add_column()
table.add_column()
file_link = escape(f"file://{Path(self.path).absolute()}")
file_and_line = escape(f"{Path(self.path).name}:{self.line_number}")
table.add_row(
f"[dim]{local_time.time()} {timezone_name}",
f"[dim]{local_time.time()}",
Align.right(
Text(f"{file_and_line}", style=Style(dim=True, link=file_link))
),

View File

@@ -14,6 +14,7 @@ from aiohttp.abc import Request
from aiohttp.web_ws import WebSocketResponse
from rich.console import Console
from rich.markup import escape
import msgpack
from textual.devtools.renderables import (
DevConsoleLog,
@@ -160,26 +161,25 @@ class ClientHandler:
"""
last_message_time: float | None = None
while True:
message_json = await self.incoming_queue.get()
if message_json is None:
message = await self.incoming_queue.get()
if message is None:
self.incoming_queue.task_done()
break
type = message_json["type"]
type = message["type"]
if type == "client_log":
path = message_json["payload"]["path"]
line_number = message_json["payload"]["line_number"]
timestamp = message_json["payload"]["timestamp"]
encoded_segments = message_json["payload"]["encoded_segments"]
decoded_segments = base64.b64decode(encoded_segments)
segments = pickle.loads(decoded_segments)
path = message["payload"]["path"]
line_number = message["payload"]["line_number"]
timestamp = message["payload"]["timestamp"]
encoded_segments = message["payload"]["segments"]
segments = pickle.loads(encoded_segments)
message_time = time()
if (
last_message_time is not None
and message_time - last_message_time > 1
):
# Print a rule if it has been longer than a second since the last message
self.service.console.rule("")
self.service.console.rule()
self.service.console.print(
DevConsoleLog(
segments=segments,
@@ -190,7 +190,7 @@ class ClientHandler:
)
last_message_time = message_time
elif type == "client_spillover":
spillover = int(message_json["payload"]["spillover"])
spillover = int(message["payload"]["spillover"])
info_renderable = DevConsoleNotice(
f"Discarded {spillover} messages", level="warning"
)
@@ -219,21 +219,26 @@ class ClientHandler:
await self.service.send_server_info(client_handler=self)
async for message in self.websocket:
message = cast(WSMessage, message)
if message.type == WSMsgType.TEXT:
if message.type in (WSMsgType.TEXT, WSMsgType.BINARY):
try:
message_json = json.loads(message.data)
if isinstance(message.data, bytes):
message = msgpack.unpackb(message.data)
else:
message = json.loads(message.data)
except JSONDecodeError:
self.service.console.print(escape(str(message.data)))
continue
type = message_json.get("type")
type = message.get("type")
if not type:
continue
if (
type in QUEUEABLE_TYPES
and not self.service.shutdown_event.is_set()
):
await self.incoming_queue.put(message_json)
await self.incoming_queue.put(message)
elif message.type == WSMsgType.ERROR:
self.service.console.print(
DevConsoleNotice("Websocket error occurred", level="error")

View File

@@ -1,4 +1,4 @@
from datetime import datetime, timezone
from datetime import datetime
import pytest
import time_machine
@@ -6,22 +6,23 @@ from rich.align import Align
from rich.console import Console
from rich.segment import Segment
import msgpack
from tests.utilities.render import wait_for_predicate
from textual.devtools.renderables import DevConsoleLog, DevConsoleNotice
TIMESTAMP = 1649166819
WIDTH = 40
# The string "Hello, world!" is encoded in the payload below
EXAMPLE_LOG = {
_EXAMPLE_LOG = {
"type": "client_log",
"payload": {
"encoded_segments": "gASVQgAAAAAAAABdlCiMDHJpY2guc2VnbWVudJSMB1NlZ"
"21lbnSUk5SMDUhlbGxvLCB3b3JsZCGUTk6HlIGUaAOMAQqUTk6HlIGUZS4=",
"segments": b"\x80\x04\x955\x00\x00\x00\x00\x00\x00\x00]\x94\x8c\x0crich.segment\x94\x8c\x07Segment\x94\x93\x94\x8c\rHello, world!\x94NN\x87\x94\x81\x94a.",
"line_number": 123,
"path": "abc/hello.py",
"timestamp": TIMESTAMP,
},
}
EXAMPLE_LOG = msgpack.packb(_EXAMPLE_LOG)
@pytest.fixture(scope="module")
@@ -48,15 +49,10 @@ def test_log_message_render(console):
right: Align = right_cells[0]
# Since we can't guarantee the timezone the tests will run in...
local_time = (
datetime.fromtimestamp(TIMESTAMP)
.replace(tzinfo=timezone.utc)
.astimezone(tz=datetime.now().astimezone().tzinfo)
)
timezone_name = local_time.tzname()
local_time = datetime.fromtimestamp(TIMESTAMP)
string_timestamp = local_time.time()
assert left == f"[dim]{string_timestamp} {timezone_name}"
assert left == f"[dim]{string_timestamp}"
assert right.align == "right"
assert "hello.py:123" in right.renderable
@@ -69,7 +65,7 @@ def test_internal_message_render(console):
async def test_devtools_valid_client_log(devtools):
await devtools.websocket.send_json(EXAMPLE_LOG)
await devtools.websocket.send_bytes(EXAMPLE_LOG)
assert devtools.is_connected

View File

@@ -7,6 +7,7 @@ import time_machine
from aiohttp.web_ws import WebSocketResponse
from rich.console import ConsoleDimensions
from rich.panel import Panel
import msgpack
from tests.utilities.render import wait_for_predicate
from textual.devtools.client import DevtoolsClient
@@ -27,36 +28,39 @@ async def test_devtools_client_is_connected(devtools):
assert devtools.is_connected
@time_machine.travel(datetime.fromtimestamp(TIMESTAMP))
@time_machine.travel(datetime.utcfromtimestamp(TIMESTAMP))
async def test_devtools_log_places_encodes_and_queues_message(devtools):
await devtools._stop_log_queue_processing()
devtools.log(DevtoolsLog("Hello, world!", CALLER))
queued_log = await devtools.log_queue.get()
queued_log_json = json.loads(queued_log)
assert queued_log_json == {
queued_log_data = msgpack.unpackb(queued_log)
print(repr(queued_log_data))
assert queued_log_data == {
"type": "client_log",
"payload": {
"timestamp": TIMESTAMP,
"path": CALLER_PATH,
"line_number": CALLER_LINENO,
"encoded_segments": "gANdcQAoY3JpY2guc2VnbWVudApTZWdtZW50CnEBWA0AAABIZWxsbywgd29ybGQhcQJOTodxA4FxBGgBWAEAAAAKcQVOTodxBoFxB2Uu",
"timestamp": 1649166819,
"path": "a/b/c.py",
"line_number": 123,
"segments": b"\x80\x04\x95B\x00\x00\x00\x00\x00\x00\x00]\x94(\x8c\x0crich.segment\x94\x8c\x07Segment\x94\x93\x94\x8c\rHello, world!\x94NN\x87\x94\x81\x94h\x03\x8c\x01\n\x94NN\x87\x94\x81\x94e.",
},
}
@time_machine.travel(datetime.fromtimestamp(TIMESTAMP))
@time_machine.travel(datetime.utcfromtimestamp(TIMESTAMP))
async def test_devtools_log_places_encodes_and_queues_many_logs_as_string(devtools):
await devtools._stop_log_queue_processing()
devtools.log(DevtoolsLog(("hello", "world"), CALLER))
queued_log = await devtools.log_queue.get()
queued_log_json = json.loads(queued_log)
assert queued_log_json == {
queued_log_data = msgpack.unpackb(queued_log)
print(repr(queued_log_data))
assert queued_log_data == {
"type": "client_log",
"payload": {
"timestamp": TIMESTAMP,
"path": CALLER_PATH,
"line_number": CALLER_LINENO,
"encoded_segments": "gANdcQAoY3JpY2guc2VnbWVudApTZWdtZW50CnEBWAsAAABoZWxsbyB3b3JsZHECTk6HcQOBcQRoAVgBAAAACnEFTk6HcQaBcQdlLg==",
"timestamp": 1649166819,
"path": "a/b/c.py",
"line_number": 123,
"segments": b"\x80\x04\x95@\x00\x00\x00\x00\x00\x00\x00]\x94(\x8c\x0crich.segment\x94\x8c\x07Segment\x94\x93\x94\x8c\x0bhello world\x94NN\x87\x94\x81\x94h\x03\x8c\x01\n\x94NN\x87\x94\x81\x94e.",
},
}

View File

@@ -4,12 +4,13 @@ from datetime import datetime
import time_machine
import msgpack
from textual.devtools.redirect_output import StdoutRedirector
TIMESTAMP = 1649166819
@time_machine.travel(datetime.fromtimestamp(TIMESTAMP))
@time_machine.travel(datetime.utcfromtimestamp(TIMESTAMP))
async def test_print_redirect_to_devtools_only(devtools):
await devtools._stop_log_queue_processing()
@@ -19,14 +20,15 @@ async def test_print_redirect_to_devtools_only(devtools):
assert devtools.log_queue.qsize() == 1
queued_log = await devtools.log_queue.get()
queued_log_json = json.loads(queued_log)
payload = queued_log_json["payload"]
queued_log_data = msgpack.unpackb(queued_log)
print(repr(queued_log_data))
payload = queued_log_data["payload"]
assert queued_log_json["type"] == "client_log"
assert queued_log_data["type"] == "client_log"
assert payload["timestamp"] == TIMESTAMP
assert (
payload["encoded_segments"]
== "gANdcQAoY3JpY2guc2VnbWVudApTZWdtZW50CnEBWA0AAABIZWxsbywgd29ybGQhcQJOTodxA4FxBGgBWAEAAAAKcQVOTodxBoFxB2Uu"
payload["segments"]
== b"\x80\x04\x95B\x00\x00\x00\x00\x00\x00\x00]\x94(\x8c\x0crich.segment\x94\x8c\x07Segment\x94\x93\x94\x8c\rHello, world!\x94NN\x87\x94\x81\x94h\x03\x8c\x01\n\x94NN\x87\x94\x81\x94e."
)
@@ -86,8 +88,10 @@ async def test_print_multiple_args_batched_as_one_log(devtools, in_memory_logfil
assert queued_log_json["type"] == "client_log"
assert payload["timestamp"] == TIMESTAMP
assert payload[
"encoded_segments"] == "gANdcQAoY3JpY2guc2VnbWVudApTZWdtZW50CnEBWBQAAABIZWxsbyB3b3JsZCBtdWx0aXBsZXECTk6HcQOBcQRoAVgBAAAACnEFTk6HcQaBcQdlLg=="
assert (
payload["encoded_segments"]
== "gANdcQAoY3JpY2guc2VnbWVudApTZWdtZW50CnEBWBQAAABIZWxsbyB3b3JsZCBtdWx0aXBsZXECTk6HcQOBcQRoAVgBAAAACnEFTk6HcQaBcQdlLg=="
)
assert len(payload["path"]) > 0
assert payload["line_number"] != 0