From dc2963d2a0002d4ddacf50b1f80c470d5de7ec61 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Wed, 12 Jun 2019 15:10:59 +0100 Subject: [PATCH] Fix double word hacking test At present, 'pycodestyle' feeds the following string into the 'tokenizer' library: ["'This is the the best comment'"] (note the added quotes because this isn't valid Python otherwise) On previous versions of Python, this tokenizer would parse the string like so: (3, "'This is the the best comment'", (1, 0), (1, 30), "'This is the the best comment'") (0, '', (2, 0), (2, 0), '') where (3 = 'STRING', 0 = 'ENDMARKER') However, with the fix [1] backported to recent versions of Python, this now resolves to: (3, "'This is the the best comment'", (1, 0), (1, 30), "'This is the the best comment'") (4, '', (1, 30), (1, 31), '') (0, '', (2, 0), (2, 0), '') where (3 = 'STRING', 4 = 'NEWLINE', 0 = 'ENDMARKER') Typically, 'pycodestyle' will run physical line checks on each line as it parses the token: https://github.com/PyCQA/pycodestyle/blob/2.5.0/pycodestyle.py#L2036 For the former case above, the line doesn't include a newline which means we never parse a 'NEWLINE' token with a logical line (the fifth element of the token tuple) corresponding to our full line. This means we don't here but that wasn't an issue previously since there's a fallthrough case that handled tokens remaining at the end of the parse: https://github.com/PyCQA/pycodestyle/blob/2.5.0/pycodestyle.py#L2114-L2116 Unfortunately, because we now have an additional newline character to parse, one that's on a separate line to our test string no less, we run logical checks on it: https://github.com/PyCQA/pycodestyle/blob/2.5.0/pycodestyle.py#L2105-L2107 This is an issue since the logical check wipes stored tokens meaning we've nothing to check when we get to the fallthrough case: https://github.com/PyCQA/pycodestyle/blob/2.5.0/pycodestyle.py#L2012 This fixes changes things so that a newline is included (and also adds quotes so it's valid Python, but that's mostly unrelated). This means we end up with the following instead: ["'This is the the best comment'\n"] On both Python without the bugfix and with it, this parses as: (3, "'This is the the best comment'", (1, 0), (1, 30), "'This is the the best comment'\n") (4, '\n', (1, 30), (1, 31), "'This is the the best comment'\n") (0, '', (2, 0), (2, 0), '') where (3 = 'STRING', 4 = 'NEWLINE', 0 = 'ENDMARKER') Which triggers things in 'pycodestyle' correctly. https://github.com/PyCQA/pycodestyle/blob/2.5.0/pycodestyle.py#L2044-L2046 This isn't _really_ a fix since there's clearly still a bug in either 'pycodestyle' or Python (I think the latter, since it's adding a newline to a file that explicitly doesn't have one), but the chances of us hitting this bug in practice are rather low - you'd need to make a mistake on the very last line of a file without a newline at the end which is something Vim, for example, won't even let you do without setting special flags - and therefore it can be reasonably ignored. [1] https://bugs.python.org/issue33899 Change-Id: Ia597594e0469c0e83d7ad22b0678390aaebaffe7 Signed-off-by: Stephen Finucane Closes-Bug: #1804062 (cherry picked from f545a25cc443c41dcd9bdd028064c28b53f56037) (cherry picked from commit 0cb6106b83c33bded9e6cdec7737964c36be8de5) (cherry picked from commit 4858074c89838eadeb9eaf9f39917e9fb90acd93) --- nova/tests/unit/test_hacking.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nova/tests/unit/test_hacking.py b/nova/tests/unit/test_hacking.py index 82b822a7931a..c8dd1104a4b0 100644 --- a/nova/tests/unit/test_hacking.py +++ b/nova/tests/unit/test_hacking.py @@ -275,7 +275,7 @@ class HackingTestCase(test.NoDBTestCase): def _run_check(self, code, checker, filename=None): pep8.register_check(checker) - lines = textwrap.dedent(code).strip().splitlines(True) + lines = textwrap.dedent(code).lstrip().splitlines(True) checker = pep8.Checker(filename=filename, lines=lines) # NOTE(sdague): the standard reporter has printing to stdout @@ -571,12 +571,14 @@ class HackingTestCase(test.NoDBTestCase): def test_check_doubled_words(self): errors = [(1, 0, "N343")] - # Artificial break to stop pep8 detecting the test ! - code = "This is the" + " the best comment" + # Explicit addition of line-ending here and below since this isn't a + # block comment and without it we trigger #1804062. Artificial break is + # necessary to stop flake8 detecting the test + code = "'This is the" + " the best comment'\n" self._assert_has_errors(code, checks.check_doubled_words, expected_errors=errors) - code = "This is the then best comment" + code = "'This is the then best comment'\n" self._assert_has_no_errors(code, checks.check_doubled_words) def test_dict_iteritems(self):