From a99d60f9f8a883b81fcc5fd5bade51f2adc7e14d Mon Sep 17 00:00:00 2001 From: Darren Burns Date: Tue, 1 Feb 2022 11:52:15 +0000 Subject: [PATCH] Tokenising variable declarations --- src/textual/css/tokenize.py | 34 ++++++++++++++--- tests/css/test_tokenize.py | 76 +++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 5 deletions(-) diff --git a/src/textual/css/tokenize.py b/src/textual/css/tokenize.py index 2aae666e1..fd0e7a1fd 100644 --- a/src/textual/css/tokenize.py +++ b/src/textual/css/tokenize.py @@ -6,14 +6,28 @@ from typing import Iterable from textual.css.tokenizer import Expect, Tokenizer, Token - -expect_selector = Expect( +# Things we can match at the top-most scope in the CSS file +expect_root_scope = Expect( whitespace=r"\s+", comment_start=r"\/\*", selector_start_id=r"\#[a-zA-Z_\-][a-zA-Z0-9_\-]*", selector_start_class=r"\.[a-zA-Z_\-][a-zA-Z0-9_\-]*", selector_start_universal=r"\*", selector_start=r"[a-zA-Z_\-]+", + variable_declaration_start=r"\$[a-zA-Z0-9_\-]+\:", +).expect_eof(True) + +expect_variable_declaration_continue = Expect( + variable_declaration_end=r"\n|;", + whitespace=r"\s+", + comment_start=r"\/\*", + scalar=r"\-?\d+\.?\d*(?:fr|%|w|h|vw|vh)", + duration=r"\d+\.?\d*(?:ms|s)", + number=r"\-?\d+\.?\d*", + color=r"\#[0-9a-fA-F]{6}|color\([0-9]{1,3}\)|rgb\(\d{1,3}\,\s?\d{1,3}\,\s?\d{1,3}\)", + key_value=r"[a-zA-Z_-][a-zA-Z0-9_-]*=[0-9a-zA-Z_\-\/]+", + token="[a-zA-Z_-]+", + string=r"\".*?\"", ).expect_eof(True) expect_comment_end = Expect( @@ -65,8 +79,19 @@ expect_declaration_content = Expect( class TokenizerState: - EXPECT = expect_selector + """State machine for the tokeniser. + + Attributes: + EXPECT: The initial expectation of the tokenizer. Since we start tokenising + at the root scope, we'd expect to see either a variable or selector. + STATE_MAP: Maps token names to Expects, which are sets of regexes conveying + what we expect to see next in the tokenising process. + """ + + EXPECT = expect_root_scope STATE_MAP = { + "variable_declaration_start": expect_variable_declaration_continue, + "variable_declaration_end": expect_root_scope, "selector_start": expect_selector_continue, "selector_start_id": expect_selector_continue, "selector_start_class": expect_selector_continue, @@ -77,7 +102,7 @@ class TokenizerState: "declaration_set_start": expect_declaration, "declaration_name": expect_declaration_content, "declaration_end": expect_declaration, - "declaration_set_end": expect_selector, + "declaration_set_end": expect_root_scope, } def __call__(self, code: str, path: str) -> Iterable[Token]: @@ -108,7 +133,6 @@ class DeclarationTokenizerState(TokenizerState): tokenize = TokenizerState() tokenize_declarations = DeclarationTokenizerState() - # def tokenize( # code: str, path: str, *, expect: Expect = expect_selector # ) -> Iterable[Token]: diff --git a/tests/css/test_tokenize.py b/tests/css/test_tokenize.py index e69de29bb..a795df024 100644 --- a/tests/css/test_tokenize.py +++ b/tests/css/test_tokenize.py @@ -0,0 +1,76 @@ +import pytest + +from textual.css.tokenize import tokenize +from textual.css.tokenizer import Token, TokenizeError + +VALID_VARIABLE_NAMES = [ + "warning-text", + "warning_text", + "warningtext1", + "1warningtext", + "WarningText1", + "warningtext_", + "warningtext-", + "_warningtext", + "-warningtext", +] + + +@pytest.mark.parametrize("name", VALID_VARIABLE_NAMES) +def test_variable_declaration_valid_names(name): + css = f"${name}: black on red;" + + assert list(tokenize(css, "")) == [ + Token( + name="variable_declaration_start", value=f"${name}:", path="", code=css, location=(0, 0) + ), + Token(name="whitespace", value=" ", path="", code=css, location=(0, 14)), + Token(name="token", value="black", path="", code=css, location=(0, 15)), + Token(name="whitespace", value=" ", path="", code=css, location=(0, 20)), + Token(name="token", value="on", path="", code=css, location=(0, 21)), + Token(name="whitespace", value=" ", path="", code=css, location=(0, 23)), + Token(name="token", value="red", path="", code=css, location=(0, 24)), + Token(name="variable_declaration_end", value=";", path="", code=css, location=(0, 27)), + ] + + +def test_variable_declaration_no_semicolon(): + css = "$x: 1\n$y: 2" + + assert list(tokenize(css, "")) == [ + Token(name="variable_declaration_start", value="$x:", code=css, path="", location=(0, 0)), + Token(name="whitespace", value=" ", code=css, path="", location=(0, 3)), + Token(name="number", value="1", code=css, path="", location=(0, 4)), + Token(name="variable_declaration_end", value="\n", code=css, path="", location=(0, 5)), + Token(name="variable_declaration_start", value="$y:", code=css, path="", location=(1, 0)), + Token(name="whitespace", value=" ", code=css, path="", location=(1, 3)), + Token(name="number", value="2", code=css, path="", location=(1, 4)), + ] + + +def test_variable_declaration_invalid_value(): + css = "$x:(@$12x)" + with pytest.raises(TokenizeError): + list(tokenize(css, "")) + + +def test_variables_declarations_amongst_rulesets(): + css = "$x:1; .thing{text:red;} $y:2;" + tokens = list(tokenize(css, "")) + + assert tokens == [ + Token(name='variable_declaration_start', value='$x:', path='', code=css, location=(0, 0)), + Token(name='number', value='1', path='', code=css, location=(0, 3)), + Token(name='variable_declaration_end', value=';', path='', code=css, location=(0, 4)), + Token(name='whitespace', value=' ', path='', code=css, location=(0, 5)), + Token(name='selector_start_class', value='.thing', path='', code=css, location=(0, 6)), + Token(name='declaration_set_start', value='{', path='', code=css, location=(0, 12)), + Token(name='declaration_name', value='text:', path='', code=css, location=(0, 13)), + Token(name='token', value='red', path='', code=css, location=(0, 18)), + Token(name='declaration_end', value=';', path='', code=css, location=(0, 21)), + Token(name='declaration_set_end', value='}', path='', code=css, location=(0, 22)), + Token(name='whitespace', value=' ', path='', code=css, location=(0, 23)), + Token(name='variable_declaration_start', value='$y:', path='', code=css, location=(0, 24)), + Token(name='number', value='2', path='', code=css, location=(0, 27)), + Token(name='variable_declaration_end', value=';', path='', code=css, location=(0, 28)), + ]