Tokenising variable declarations

src/textual/css/tokenize.py

@@ -6,14 +6,28 @@ from typing import Iterable
 
 from textual.css.tokenizer import Expect, Tokenizer, Token
 
-
+# Things we can match at the top-most scope in the CSS file
-expect_selector = Expect(
+expect_root_scope = Expect(
     whitespace=r"\s+",
     comment_start=r"\/\*",
     selector_start_id=r"\#[a-zA-Z_\-][a-zA-Z0-9_\-]*",
     selector_start_class=r"\.[a-zA-Z_\-][a-zA-Z0-9_\-]*",
     selector_start_universal=r"\*",
     selector_start=r"[a-zA-Z_\-]+",
+    variable_declaration_start=r"\$[a-zA-Z0-9_\-]+\:",
+).expect_eof(True)
+
+expect_variable_declaration_continue = Expect(
+    variable_declaration_end=r"\n|;",
+    whitespace=r"\s+",
+    comment_start=r"\/\*",
+    scalar=r"\-?\d+\.?\d*(?:fr|%|w|h|vw|vh)",
+    duration=r"\d+\.?\d*(?:ms|s)",
+    number=r"\-?\d+\.?\d*",
+    color=r"\#[0-9a-fA-F]{6}|color\([0-9]{1,3}\)|rgb\(\d{1,3}\,\s?\d{1,3}\,\s?\d{1,3}\)",
+    key_value=r"[a-zA-Z_-][a-zA-Z0-9_-]*=[0-9a-zA-Z_\-\/]+",
+    token="[a-zA-Z_-]+",
+    string=r"\".*?\"",
 ).expect_eof(True)
 
 expect_comment_end = Expect(
@@ -65,8 +79,19 @@ expect_declaration_content = Expect(
 
 
 class TokenizerState:
-    EXPECT = expect_selector
+    """State machine for the tokeniser.
+
+    Attributes:
+        EXPECT: The initial expectation of the tokenizer. Since we start tokenising
+            at the root scope, we'd expect to see either a variable or selector.
+        STATE_MAP: Maps token names to Expects, which are sets of regexes conveying
+            what we expect to see next in the tokenising process.
+    """
+
+    EXPECT = expect_root_scope
     STATE_MAP = {
+        "variable_declaration_start": expect_variable_declaration_continue,
+        "variable_declaration_end": expect_root_scope,
         "selector_start": expect_selector_continue,
         "selector_start_id": expect_selector_continue,
         "selector_start_class": expect_selector_continue,
@@ -77,7 +102,7 @@ class TokenizerState:
         "declaration_set_start": expect_declaration,
         "declaration_name": expect_declaration_content,
         "declaration_end": expect_declaration,
-        "declaration_set_end": expect_selector,
+        "declaration_set_end": expect_root_scope,
     }
 
     def __call__(self, code: str, path: str) -> Iterable[Token]:
@@ -108,7 +133,6 @@ class DeclarationTokenizerState(TokenizerState):
 tokenize = TokenizerState()
 tokenize_declarations = DeclarationTokenizerState()
 
-
 # def tokenize(
 #     code: str, path: str, *, expect: Expect = expect_selector
 # ) -> Iterable[Token]:

tests/css/test_tokenize.py

@@ -0,0 +1,76 @@
+import pytest
+
+from textual.css.tokenize import tokenize
+from textual.css.tokenizer import Token, TokenizeError
+
+VALID_VARIABLE_NAMES = [
+    "warning-text",
+    "warning_text",
+    "warningtext1",
+    "1warningtext",
+    "WarningText1",
+    "warningtext_",
+    "warningtext-",
+    "_warningtext",
+    "-warningtext",
+]
+
+
+@pytest.mark.parametrize("name", VALID_VARIABLE_NAMES)
+def test_variable_declaration_valid_names(name):
+    css = f"${name}: black on red;"
+
+    assert list(tokenize(css, "")) == [
+        Token(
+            name="variable_declaration_start", value=f"${name}:", path="", code=css, location=(0, 0)
+        ),
+        Token(name="whitespace", value=" ", path="", code=css, location=(0, 14)),
+        Token(name="token", value="black", path="", code=css, location=(0, 15)),
+        Token(name="whitespace", value=" ", path="", code=css, location=(0, 20)),
+        Token(name="token", value="on", path="", code=css, location=(0, 21)),
+        Token(name="whitespace", value=" ", path="", code=css, location=(0, 23)),
+        Token(name="token", value="red", path="", code=css, location=(0, 24)),
+        Token(name="variable_declaration_end", value=";", path="", code=css, location=(0, 27)),
+    ]
+
+
+def test_variable_declaration_no_semicolon():
+    css = "$x: 1\n$y: 2"
+
+    assert list(tokenize(css, "")) == [
+        Token(name="variable_declaration_start", value="$x:", code=css, path="", location=(0, 0)),
+        Token(name="whitespace", value=" ", code=css, path="", location=(0, 3)),
+        Token(name="number", value="1", code=css, path="", location=(0, 4)),
+        Token(name="variable_declaration_end", value="\n", code=css, path="", location=(0, 5)),
+        Token(name="variable_declaration_start", value="$y:", code=css, path="", location=(1, 0)),
+        Token(name="whitespace", value=" ", code=css, path="", location=(1, 3)),
+        Token(name="number", value="2", code=css, path="", location=(1, 4)),
+    ]
+
+
+def test_variable_declaration_invalid_value():
+    css = "$x:(@$12x)"
+    with pytest.raises(TokenizeError):
+        list(tokenize(css, ""))
+
+
+def test_variables_declarations_amongst_rulesets():
+    css = "$x:1; .thing{text:red;} $y:2;"
+    tokens = list(tokenize(css, ""))
+
+    assert tokens == [
+        Token(name='variable_declaration_start', value='$x:', path='', code=css, location=(0, 0)),
+        Token(name='number', value='1', path='', code=css, location=(0, 3)),
+        Token(name='variable_declaration_end', value=';', path='', code=css, location=(0, 4)),
+        Token(name='whitespace', value=' ', path='', code=css, location=(0, 5)),
+        Token(name='selector_start_class', value='.thing', path='', code=css, location=(0, 6)),
+        Token(name='declaration_set_start', value='{', path='', code=css, location=(0, 12)),
+        Token(name='declaration_name', value='text:', path='', code=css, location=(0, 13)),
+        Token(name='token', value='red', path='', code=css, location=(0, 18)),
+        Token(name='declaration_end', value=';', path='', code=css, location=(0, 21)),
+        Token(name='declaration_set_end', value='}', path='', code=css, location=(0, 22)),
+        Token(name='whitespace', value=' ', path='', code=css, location=(0, 23)),
+        Token(name='variable_declaration_start', value='$y:', path='', code=css, location=(0, 24)),
+        Token(name='number', value='2', path='', code=css, location=(0, 27)),
+        Token(name='variable_declaration_end', value=';', path='', code=css, location=(0, 28)),
+    ]