Generate tokenizer tests from testdata files

Author: gsnedders

.pytest.expect

@@ -2,20 +2,20 @@ pytest-expect file v1
 (2, 7, 11, 'final', 0)
 b'html5lib/tests/test_encoding.py::test_encoding::[110]': FAIL
 b'html5lib/tests/test_encoding.py::test_encoding::[111]': FAIL
-b'html5lib/tests/test_tokenizer.py::testTokenizer::[4718]': FAIL
-b'html5lib/tests/test_tokenizer.py::testTokenizer::[4990]': FAIL
-b'html5lib/tests/test_tokenizer.py::testTokenizer::[4993]': FAIL
-b'html5lib/tests/test_tokenizer.py::testTokenizer::[4994]': FAIL
-b'html5lib/tests/test_tokenizer.py::testTokenizer::[4996]': FAIL
-b'html5lib/tests/test_tokenizer.py::testTokenizer::[4997]': FAIL
-b'html5lib/tests/test_tokenizer.py::testTokenizer::[4999]': FAIL
-b'html5lib/tests/test_tokenizer.py::testTokenizer::[5002]': FAIL
-b'html5lib/tests/test_tokenizer.py::testTokenizer::[5003]': FAIL
-b'html5lib/tests/test_tokenizer.py::testTokenizer::[5005]': FAIL
-b'html5lib/tests/test_tokenizer.py::testTokenizer::[5006]': FAIL
-b'html5lib/tests/test_tokenizer.py::testTokenizer::[5008]': FAIL
-b'html5lib/tests/test_tokenizer.py::testTokenizer::[5020]': FAIL
-b'html5lib/tests/test_tokenizer.py::testTokenizer::[5418]': FAIL
+u'html5lib/tests/testdata/tokenizer/test2.test::0::dataState': FAIL
+u'html5lib/tests/testdata/tokenizer/test3.test::228::dataState': FAIL
+u'html5lib/tests/testdata/tokenizer/test3.test::231::dataState': FAIL
+u'html5lib/tests/testdata/tokenizer/test3.test::232::dataState': FAIL
+u'html5lib/tests/testdata/tokenizer/test3.test::234::dataState': FAIL
+u'html5lib/tests/testdata/tokenizer/test3.test::235::dataState': FAIL
+u'html5lib/tests/testdata/tokenizer/test3.test::237::dataState': FAIL
+u'html5lib/tests/testdata/tokenizer/test3.test::240::dataState': FAIL
+u'html5lib/tests/testdata/tokenizer/test3.test::241::dataState': FAIL
+u'html5lib/tests/testdata/tokenizer/test3.test::243::dataState': FAIL
+u'html5lib/tests/testdata/tokenizer/test3.test::244::dataState': FAIL
+u'html5lib/tests/testdata/tokenizer/test3.test::246::dataState': FAIL
+u'html5lib/tests/testdata/tokenizer/test3.test::258::dataState': FAIL
+u'html5lib/tests/testdata/tokenizer/test3.test::656::dataState': FAIL
 u'html5lib/tests/testdata/tree-construction/foreign-fragment.dat::0::DOM::parser::namespaced': FAIL
 u'html5lib/tests/testdata/tree-construction/foreign-fragment.dat::0::DOM::parser::void-namespace': FAIL
 u'html5lib/tests/testdata/tree-construction/foreign-fragment.dat::0::ElementTree::parser::namespaced': FAIL

html5lib/tests/conftest.py

@@ -1,10 +1,12 @@
 import os.path
 
 from .tree_construction import TreeConstructionFile
+from .tokenizer import TokenizerFile
 
 _dir = os.path.abspath(os.path.dirname(__file__))
 _testdata = os.path.join(_dir, "testdata")
 _tree_construction = os.path.join(_testdata, "tree-construction")
+_tokenizer = os.path.join(_testdata, "tokenizer")
 
 
 def pytest_collectstart():
@@ -19,3 +21,6 @@ def pytest_collect_file(path, parent):
             return
         if path.ext == ".dat":
             return TreeConstructionFile(path, parent)
+    elif dir == _tokenizer:
+        if path.ext == ".test":
+            return TokenizerFile(path, parent)

html5lib/tests/test_tokenizer.py renamed to html5lib/tests/tokenizer.py

@@ -1,13 +1,13 @@
 from __future__ import absolute_import, division, unicode_literals
 
+import codecs
 import json
 import warnings
 import re
 
+import pytest
 from six import unichr
 
-from .support import get_data_files
-
 from html5lib.tokenizer import HTMLTokenizer
 from html5lib import constants, utils
 
@@ -172,27 +172,6 @@ def repl(m):
     return test
 
 
-def runTokenizerTest(test):
-    warnings.resetwarnings()
-    warnings.simplefilter("error")
-
-    expected = test['output']
-    if 'lastStartTag' not in test:
-        test['lastStartTag'] = None
-    parser = TokenizerTestParser(test['initialState'],
-                                 test['lastStartTag'])
-    tokens = parser.parse(test['input'])
-    received = normalizeTokens(tokens)
-    errorMsg = "\n".join(["\n\nInitial state:",
-                          test['initialState'],
-                          "\nInput:", test['input'],
-                          "\nExpected:", repr(expected),
-                          "\nreceived:", repr(tokens)])
-    errorMsg = errorMsg
-    ignoreErrorOrder = test.get('ignoreErrorOrder', False)
-    assert tokensMatch(expected, received, ignoreErrorOrder, True), errorMsg
-
-
 def _doCapitalize(match):
     return match.group(1).upper()
 
@@ -205,18 +184,68 @@ def capitalize(s):
     return s
 
 
-def testTokenizer():
-    for filename in get_data_files('tokenizer', '*.test'):
-        with open(filename) as fp:
+class TokenizerFile(pytest.File):
+    def collect(self):
+        with codecs.open(str(self.fspath), "r", encoding="utf-8") as fp:
             tests = json.load(fp)
-            if 'tests' in tests:
-                for index, test in enumerate(tests['tests']):
-                    if 'initialStates' not in test:
-                        test["initialStates"] = ["Data state"]
-                    if 'doubleEscaped' in test:
-                        test = unescape(test)
-                        if test["input"] is None:
-                            continue  # Not valid input for this platform
-                    for initialState in test["initialStates"]:
-                        test["initialState"] = capitalize(initialState)
-                        yield runTokenizerTest, test
+        if 'tests' in tests:
+            for i, test in enumerate(tests['tests']):
+                yield TokenizerTestCollector(str(i), self, testdata=test)
+
+
+class TokenizerTestCollector(pytest.Collector):
+    def __init__(self, name, parent=None, config=None, session=None, testdata=None):
+        super(TokenizerTestCollector, self).__init__(name, parent, config, session)
+        if 'initialStates' not in testdata:
+            testdata["initialStates"] = ["Data state"]
+        if 'doubleEscaped' in testdata:
+            testdata = unescape(testdata)
+        self.testdata = testdata
+
+    def collect(self):
+        for initialState in self.testdata["initialStates"]:
+            initialState = capitalize(initialState)
+            item = TokenizerTest(initialState,
+                                 self,
+                                 self.testdata,
+                                 initialState)
+            if self.testdata["input"] is None:
+                item.add_marker(pytest.mark.skipif(True, reason="Relies on lone surrogates"))
+            yield item
+
+
+class TokenizerTest(pytest.Item):
+    def __init__(self, name, parent, test, initialState):
+        super(TokenizerTest, self).__init__(name, parent)
+        self.obj = lambda: 1  # this is to hack around skipif needing a function!
+        self.test = test
+        self.initialState = initialState
+
+    def runtest(self):
+        warnings.resetwarnings()
+        warnings.simplefilter("error")
+
+        expected = self.test['output']
+        if 'lastStartTag' not in self.test:
+            self.test['lastStartTag'] = None
+        parser = TokenizerTestParser(self.initialState,
+                                     self.test['lastStartTag'])
+        tokens = parser.parse(self.test['input'])
+        received = normalizeTokens(tokens)
+        errorMsg = "\n".join(["\n\nInitial state:",
+                              self.initialState,
+                              "\nInput:", self.test['input'],
+                              "\nExpected:", repr(expected),
+                              "\nreceived:", repr(tokens)])
+        errorMsg = errorMsg
+        ignoreErrorOrder = self.test.get('ignoreErrorOrder', False)
+        assert tokensMatch(expected, received, ignoreErrorOrder, True), errorMsg
+
+    def repr_failure(self, excinfo):
+        traceback = excinfo.traceback
+        ntraceback = traceback.cut(path=__file__)
+        excinfo.traceback = ntraceback.filter()
+
+        return excinfo.getrepr(funcargs=True,
+                               showlocals=False,
+                               style="short", tbfilter=False)