mirrors
/
cpython
mirror of https://github.com/python/cpython.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

gh-116042: Fix location for SyntaxErrors of invalid escapes in the tokenizer (#116049)

This commit is contained in:
Pablo Galindo Salgado 2025-02-13 01:07:37 +00:00 committed by GitHub
parent 49b11033bd
commit 56eda25633
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 80 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Lib/test/test_cmd_line_script.py
View File

@ -660,7 +660,7 @@ def test_syntaxerror_invalid_escape_sequence_multi_line(self):
self.assertEqual(
stderr.splitlines()[-3:],
[ b' foo = """\\q"""',
b' ^^^^^^^^',
b' ^^',
b'SyntaxError: "\\q" is an invalid escape sequence. '
b'Did you mean "\\\\q"? A raw string is also an option.'
],

39
Lib/test/test_string_literals.py
View File

@ -120,7 +120,7 @@ def test_eval_str_invalid_escape(self):
r'Such sequences will not work in the future. '
r'Did you mean "\\z"? A raw string is also an option.')
self.assertEqual(w[0].filename, '<string>')
self.assertEqual(w[0].lineno, 1)
self.assertEqual(w[0].lineno, 2)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('error', category=SyntaxWarning)
@ -131,7 +131,7 @@ def test_eval_str_invalid_escape(self):
self.assertEqual(exc.msg, r'"\z" is an invalid escape sequence. '
r'Did you mean "\\z"? A raw string is also an option.')
self.assertEqual(exc.filename, '<string>')
self.assertEqual(exc.lineno, 1)
self.assertEqual(exc.lineno, 2)
self.assertEqual(exc.offset, 1)
# Check that the warning is raised only once if there are syntax errors
@ -160,7 +160,7 @@ def test_eval_str_invalid_octal_escape(self):
r'Such sequences will not work in the future. '
r'Did you mean "\\407"? A raw string is also an option.')
self.assertEqual(w[0].filename, '<string>')
self.assertEqual(w[0].lineno, 1)
self.assertEqual(w[0].lineno, 2)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('error', category=SyntaxWarning)
@ -171,9 +171,32 @@ def test_eval_str_invalid_octal_escape(self):
self.assertEqual(exc.msg, r'"\407" is an invalid octal escape sequence. '
r'Did you mean "\\407"? A raw string is also an option.')
self.assertEqual(exc.filename, '<string>')
self.assertEqual(exc.lineno, 1)
self.assertEqual(exc.lineno, 2)
self.assertEqual(exc.offset, 1)
def test_invalid_escape_locations_with_offset(self):
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always', category=SyntaxWarning)
eval("\"'''''''''''''''''''''invalid\ Escape\"")
self.assertEqual(len(w), 1)
self.assertEqual(str(w[0].message),
r'"\ " is an invalid escape sequence. Such sequences '
r'will not work in the future. Did you mean "\\ "? '
r'A raw string is also an option.')
self.assertEqual(w[0].filename, '<string>')
self.assertEqual(w[0].lineno, 1)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always', category=SyntaxWarning)
eval("\"''Incorrect \ logic?\"")
self.assertEqual(len(w), 1)
self.assertEqual(str(w[0].message),
r'"\ " is an invalid escape sequence. Such sequences '
r'will not work in the future. Did you mean "\\ "? '
r'A raw string is also an option.')
self.assertEqual(w[0].filename, '<string>')
self.assertEqual(w[0].lineno, 1)
def test_eval_str_raw(self):
self.assertEqual(eval(""" r'x' """), 'x')
self.assertEqual(eval(r""" r'\x01' """), '\\' + 'x01')
@ -215,7 +238,7 @@ def test_eval_bytes_invalid_escape(self):
r'Such sequences will not work in the future. '
r'Did you mean "\\z"? A raw string is also an option.')
self.assertEqual(w[0].filename, '<string>')
self.assertEqual(w[0].lineno, 1)
self.assertEqual(w[0].lineno, 2)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('error', category=SyntaxWarning)
@ -226,7 +249,7 @@ def test_eval_bytes_invalid_escape(self):
self.assertEqual(exc.msg, r'"\z" is an invalid escape sequence. '
r'Did you mean "\\z"? A raw string is also an option.')
self.assertEqual(exc.filename, '<string>')
self.assertEqual(exc.lineno, 1)
self.assertEqual(exc.lineno, 2)
def test_eval_bytes_invalid_octal_escape(self):
for i in range(0o400, 0o1000):
@ -241,7 +264,7 @@ def test_eval_bytes_invalid_octal_escape(self):
r'Such sequences will not work in the future. '
r'Did you mean "\\407"? A raw string is also an option.')
self.assertEqual(w[0].filename, '<string>')
self.assertEqual(w[0].lineno, 1)
self.assertEqual(w[0].lineno, 2)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('error', category=SyntaxWarning)
@ -252,7 +275,7 @@ def test_eval_bytes_invalid_octal_escape(self):
self.assertEqual(exc.msg, r'"\407" is an invalid octal escape sequence. '
r'Did you mean "\\407"? A raw string is also an option.')
self.assertEqual(exc.filename, '<string>')
self.assertEqual(exc.lineno, 1)
self.assertEqual(exc.lineno, 2)
def test_eval_bytes_raw(self):
self.assertEqual(eval(""" br'x' """), b'x')

2
Misc/NEWS.d/next/Core_and_Builtins/2025-02-13-00-28-43.gh-issue-116042.861juq.rst Normal file
View File

@ -0,0 +1,2 @@
Fix location for SyntaxErrors of invalid escapes in the tokenizer. Patch by
Pablo Galindo

4
Parser/pegen_errors.c
View File

@ -352,8 +352,8 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
assert(p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done == E_EOF);
if (p->tok->lineno <= lineno && p->tok->inp > p->tok->buf) {
Py_ssize_t size = p->tok->inp - p->tok->buf;
error_line = PyUnicode_DecodeUTF8(p->tok->buf, size, "replace");
Py_ssize_t size = p->tok->inp - p->tok->line_start;
error_line = PyUnicode_DecodeUTF8(p->tok->line_start, size, "replace");
}
else if (p->tok->fp == NULL || p->tok->fp == stdin) {
error_line = get_error_line_from_tokenizer_buffers(p, lineno);

50
Parser/string_parser.c
View File

@ -11,7 +11,7 @@
//// STRING HANDLING FUNCTIONS ////
static int
warn_invalid_escape_sequence(Parser *p, const char *first_invalid_escape, Token *t)
warn_invalid_escape_sequence(Parser *p, const char* buffer, const char *first_invalid_escape, Token *t)
{
if (p->call_invalid_rules) {
// Do not report warnings if we are in the second pass of the parser
@ -48,8 +48,46 @@ warn_invalid_escape_sequence(Parser *p, const char *first_invalid_escape, Token
else {
category = PyExc_DeprecationWarning;
}
// Calculate the lineno and the col_offset of the invalid escape sequence
const char *start = buffer;
const char *end = first_invalid_escape;
int lineno = t->lineno;
int col_offset = t->col_offset;
while (start < end) {
if (*start == '\n') {
lineno++;
col_offset = 0;
}
else {
col_offset++;
}
start++;
}
// Count the number of quotes in the token
char first_quote = 0;
if (lineno == t->lineno) {
int quote_count = 0;
char* tok = PyBytes_AsString(t->bytes);
for (int i = 0; i < PyBytes_Size(t->bytes); i++) {
if (tok[i] == '\'' || tok[i] == '\"') {
if (quote_count == 0) {
first_quote = tok[i];
}
if (tok[i] == first_quote) {
quote_count++;
}
} else {
break;
}
}
col_offset += quote_count;
}
if (PyErr_WarnExplicitObject(category, msg, p->tok->filename,
t->lineno, NULL, NULL) < 0) {
lineno, NULL, NULL) < 0) {
if (PyErr_ExceptionMatches(category)) {
/* Replace the Syntax/DeprecationWarning exception with a SyntaxError
to get a more accurate error report */
@ -60,13 +98,13 @@ warn_invalid_escape_sequence(Parser *p, const char *first_invalid_escape, Token
error location, if p->known_err_token is not set. */
p->known_err_token = t;
if (octal) {
RAISE_SYNTAX_ERROR(
RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, lineno, col_offset-1, lineno, col_offset+1,
"\"\\%.3s\" is an invalid octal escape sequence. "
"Did you mean \"\\\\%.3s\"? A raw string is also an option.",
first_invalid_escape, first_invalid_escape);
}
else {
RAISE_SYNTAX_ERROR(
RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, lineno, col_offset-1, lineno, col_offset+1,
"\"\\%c\" is an invalid escape sequence. "
"Did you mean \"\\\\%c\"? A raw string is also an option.",
c, c);
@ -163,7 +201,7 @@ decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token *t)
// HACK: later we can simply pass the line no, since we don't preserve the tokens
// when we are decoding the string but we preserve the line numbers.
if (v != NULL && first_invalid_escape != NULL && t != NULL) {
if (warn_invalid_escape_sequence(parser, first_invalid_escape, t) < 0) {
if (warn_invalid_escape_sequence(parser, s, first_invalid_escape, t) < 0) {
/* We have not decref u before because first_invalid_escape points
inside u. */
Py_XDECREF(u);
@ -185,7 +223,7 @@ decode_bytes_with_escapes(Parser *p, const char *s, Py_ssize_t len, Token *t)
}
if (first_invalid_escape != NULL) {
if (warn_invalid_escape_sequence(p, first_invalid_escape, t) < 0) {
if (warn_invalid_escape_sequence(p, s, first_invalid_escape, t) < 0) {
Py_DECREF(result);
return NULL;
}