This commit is contained in:
Kovid Goyal 2021-06-23 10:59:29 +05:30
parent 53d0d1c249
commit 2e235a9e7a
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C

View File

@ -13,6 +13,7 @@ if c_tokenize_flat is None:
else: else:
tokenizers = (python_tokenize_flat, c_tokenize_flat) tokenizers = (python_tokenize_flat, c_tokenize_flat)
def token_api(self, tokenize): def token_api(self, tokenize):
for css_source in [ for css_source in [
'(8, foo, [z])', '[8, foo, (z)]', '{8, foo, [z]}', 'func(8, foo, [z])' '(8, foo, [z])', '[8, foo, (z)]', '{8, foo, [z]}', 'func(8, foo, [z])'
@ -21,6 +22,7 @@ def token_api(self, tokenize):
self.ae(len(tokens), 1) self.ae(len(tokens), 1)
self.ae(len(tokens[0].content), 7) self.ae(len(tokens[0].content), 7)
def token_serialize_css(self, tokenize): def token_serialize_css(self, tokenize):
for tokenize in tokenizers: for tokenize in tokenizers:
for css_source in [ for css_source in [
@ -40,6 +42,7 @@ foo(int x) {\
result = ''.join(token.as_css() for token in tokens) result = ''.join(token.as_css() for token in tokens)
self.ae(result, css_source) self.ae(result, css_source)
def comments(self, tokenize): def comments(self, tokenize):
for ignore_comments, expected_tokens in [ for ignore_comments, expected_tokens in [
(False, [ (False, [
@ -65,6 +68,7 @@ def comments(self, tokenize):
result = list(jsonify(tokens)) result = list(jsonify(tokens))
self.ae(result, expected_tokens) self.ae(result, expected_tokens)
def token_grouping(self, tokenize): def token_grouping(self, tokenize):
for css_source, expected_tokens in [ for css_source, expected_tokens in [
('', []), ('', []),
@ -126,6 +130,7 @@ def token_grouping(self, tokenize):
result = list(jsonify(tokens)) result = list(jsonify(tokens))
self.ae(result, expected_tokens) self.ae(result, expected_tokens)
def positions(self, tokenize): def positions(self, tokenize):
css = '/* Lorem\nipsum */\fa {\n color: red;\tcontent: "dolor\\\fsit" }' css = '/* Lorem\nipsum */\fa {\n color: red;\tcontent: "dolor\\\fsit" }'
tokens = tokenize(css, ignore_comments=False) tokens = tokenize(css, ignore_comments=False)
@ -138,6 +143,7 @@ def positions(self, tokenize):
('IDENT', 4, 17), (':', 4, 24), ('S', 4, 25), ('STRING', 4, 26), ('IDENT', 4, 17), (':', 4, 24), ('S', 4, 25), ('STRING', 4, 26),
('S', 5, 5), ('}', 5, 6)]) ('S', 5, 5), ('}', 5, 6)])
def tokens(self, tokenize): def tokens(self, tokenize):
for css_source, expected_tokens in [ for css_source, expected_tokens in [
('', []), ('', []),
@ -175,7 +181,7 @@ foo(int x) {\
# Escaping # Escaping
(r'/* Comment with a \ backslash */', (r'/* Comment with a \ backslash */',
[('COMMENT', '/* Comment with a \ backslash */')]), # Unchanged [('COMMENT', r'/* Comment with a \ backslash */')]), # Unchanged
# backslash followed by a newline in a string: ignored # backslash followed by a newline in a string: ignored
('"Lorem\\\nIpsum"', [('STRING', 'LoremIpsum')]), ('"Lorem\\\nIpsum"', [('STRING', 'LoremIpsum')]),
@ -266,4 +272,3 @@ class TestTokenizer(BaseTest):
def test_tokens(self): def test_tokens(self):
self.run_test(tokens) self.run_test(tokens)