alihan/textual
1
0
Fork 0
mirror of https://github.com/Textualize/textual.git synced 2025-10-17 02:38:12 +03:00
Code Issues Packages Projects Releases Wiki Activity

Tokenising variable declarations

Browse Source
This commit is contained in:
Darren Burns
2022-02-01 11:52:15 +00:00
parent ea2819f65b
commit a99d60f9f8
2 changed files with 105 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/textual/css/tokenize.py
View File

@@ -6,14 +6,28 @@ from typing import Iterable
from textual.css.tokenizer import Expect, Tokenizer, Token
expect_selector = Expect(
# Things we can match at the top-most scope in the CSS file
expect_root_scope = Expect(
whitespace=r"\s+",
comment_start=r"\/\*",
selector_start_id=r"\#[a-zA-Z_\-][a-zA-Z0-9_\-]*",
selector_start_class=r"\.[a-zA-Z_\-][a-zA-Z0-9_\-]*",
selector_start_universal=r"\*",
selector_start=r"[a-zA-Z_\-]+",
variable_declaration_start=r"\$[a-zA-Z0-9_\-]+\:",
).expect_eof(True)
expect_variable_declaration_continue = Expect(
variable_declaration_end=r"\n|;",
whitespace=r"\s+",
comment_start=r"\/\*",
scalar=r"\-?\d+\.?\d*(?:fr|%|w|h|vw|vh)",
duration=r"\d+\.?\d*(?:ms|s)",
number=r"\-?\d+\.?\d*",
color=r"\#[0-9a-fA-F]{6}|color\([0-9]{1,3}\)|rgb\(\d{1,3}\,\s?\d{1,3}\,\s?\d{1,3}\)",
key_value=r"[a-zA-Z_-][a-zA-Z0-9_-]*=[0-9a-zA-Z_\-\/]+",
token="[a-zA-Z_-]+",
string=r"\".*?\"",
).expect_eof(True)
expect_comment_end = Expect(
@@ -65,8 +79,19 @@ expect_declaration_content = Expect(
class TokenizerState:
EXPECT = expect_selector
"""State machine for the tokeniser.
Attributes:
EXPECT: The initial expectation of the tokenizer. Since we start tokenising
at the root scope, we'd expect to see either a variable or selector.
STATE_MAP: Maps token names to Expects, which are sets of regexes conveying
what we expect to see next in the tokenising process.
"""
EXPECT = expect_root_scope
STATE_MAP = {
"variable_declaration_start": expect_variable_declaration_continue,
"variable_declaration_end": expect_root_scope,
"selector_start": expect_selector_continue,
"selector_start_id": expect_selector_continue,
"selector_start_class": expect_selector_continue,
@@ -77,7 +102,7 @@ class TokenizerState:
"declaration_set_start": expect_declaration,
"declaration_name": expect_declaration_content,
"declaration_end": expect_declaration,
"declaration_set_end": expect_selector,
"declaration_set_end": expect_root_scope,
}
def __call__(self, code: str, path: str) -> Iterable[Token]:
@@ -108,7 +133,6 @@ class DeclarationTokenizerState(TokenizerState):
tokenize = TokenizerState()
tokenize_declarations = DeclarationTokenizerState()
# def tokenize(
# code: str, path: str, *, expect: Expect = expect_selector
# ) -> Iterable[Token]:

76
tests/css/test_tokenize.py
View File

@@ -0,0 +1,76 @@
import pytest
from textual.css.tokenize import tokenize
from textual.css.tokenizer import Token, TokenizeError
VALID_VARIABLE_NAMES = [
"warning-text",
"warning_text",
"warningtext1",
"1warningtext",
"WarningText1",
"warningtext_",
"warningtext-",
"_warningtext",
"-warningtext",
]
@pytest.mark.parametrize("name", VALID_VARIABLE_NAMES)
def test_variable_declaration_valid_names(name):
css = f"${name}: black on red;"
assert list(tokenize(css, "")) == [
Token(
name="variable_declaration_start", value=f"${name}:", path="", code=css, location=(0, 0)
),
Token(name="whitespace", value=" ", path="", code=css, location=(0, 14)),
Token(name="token", value="black", path="", code=css, location=(0, 15)),
Token(name="whitespace", value=" ", path="", code=css, location=(0, 20)),
Token(name="token", value="on", path="", code=css, location=(0, 21)),
Token(name="whitespace", value=" ", path="", code=css, location=(0, 23)),
Token(name="token", value="red", path="", code=css, location=(0, 24)),
Token(name="variable_declaration_end", value=";", path="", code=css, location=(0, 27)),
]
def test_variable_declaration_no_semicolon():
css = "$x: 1\n$y: 2"
assert list(tokenize(css, "")) == [
Token(name="variable_declaration_start", value="$x:", code=css, path="", location=(0, 0)),
Token(name="whitespace", value=" ", code=css, path="", location=(0, 3)),
Token(name="number", value="1", code=css, path="", location=(0, 4)),
Token(name="variable_declaration_end", value="\n", code=css, path="", location=(0, 5)),
Token(name="variable_declaration_start", value="$y:", code=css, path="", location=(1, 0)),
Token(name="whitespace", value=" ", code=css, path="", location=(1, 3)),
Token(name="number", value="2", code=css, path="", location=(1, 4)),
]
def test_variable_declaration_invalid_value():
css = "$x:(@$12x)"
with pytest.raises(TokenizeError):
list(tokenize(css, ""))
def test_variables_declarations_amongst_rulesets():
css = "$x:1; .thing{text:red;} $y:2;"
tokens = list(tokenize(css, ""))
assert tokens == [
Token(name='variable_declaration_start', value='$x:', path='', code=css, location=(0, 0)),
Token(name='number', value='1', path='', code=css, location=(0, 3)),
Token(name='variable_declaration_end', value=';', path='', code=css, location=(0, 4)),
Token(name='whitespace', value=' ', path='', code=css, location=(0, 5)),
Token(name='selector_start_class', value='.thing', path='', code=css, location=(0, 6)),
Token(name='declaration_set_start', value='{', path='', code=css, location=(0, 12)),
Token(name='declaration_name', value='text:', path='', code=css, location=(0, 13)),
Token(name='token', value='red', path='', code=css, location=(0, 18)),
Token(name='declaration_end', value=';', path='', code=css, location=(0, 21)),
Token(name='declaration_set_end', value='}', path='', code=css, location=(0, 22)),
Token(name='whitespace', value=' ', path='', code=css, location=(0, 23)),
Token(name='variable_declaration_start', value='$y:', path='', code=css, location=(0, 24)),
Token(name='number', value='2', path='', code=css, location=(0, 27)),
Token(name='variable_declaration_end', value=';', path='', code=css, location=(0, 28)),
]