mirrors
/
cpython
mirror of https://github.com/python/cpython.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added shift and mask ops. 

Allow numbers starting with a period.
This commit is contained in:
Guido van Rossum 1991-10-24 14:59:40 +00:00
parent 7928cd7636
commit baf0ebf43c
1 changed files with 25 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
Parser/tokenizer.c
View File

@ -85,6 +85,10 @@ char *tok_name[] = {
"NOTEQUAL", "NOTEQUAL",
"LESSEQUAL", "LESSEQUAL",
"GREATEREQUAL", "GREATEREQUAL",
"TILDE",
"CIRCUMFLEX",
"LEFTSHIFT",
"RIGHTSHIFT",
/* This table must match the #defines in token.h! */ /* This table must match the #defines in token.h! */
"OP", "OP",
"<ERRORTOKEN>", "<ERRORTOKEN>",
@ -301,6 +305,8 @@ tok_1char(c)
case '`': return BACKQUOTE; case '`': return BACKQUOTE;
case '{': return LBRACE; case '{': return LBRACE;
case '}': return RBRACE; case '}': return RBRACE;
case '^': return CIRCUMFLEX;
case '~': return TILDE;
default: return OP; default: return OP;
} }
} }
@ -325,11 +331,13 @@ tok_2char(c1, c2)
switch (c2) { switch (c2) {
case '>': return NOTEQUAL; case '>': return NOTEQUAL;
case '=': return LESSEQUAL; case '=': return LESSEQUAL;
case '<': return LEFTSHIFT;
} }
break; break;
case '>': case '>':
switch (c2) { switch (c2) {
case '=': return GREATEREQUAL; case '=': return GREATEREQUAL;
case '>': return RIGHTSHIFT;
} }
break; break;
} }
@ -438,7 +446,7 @@ tok_get(tok, p_start, p_end)
beginning or end of the file. (Will vi never die...?) beginning or end of the file. (Will vi never die...?)
For Python it must be at the beginning of the file! */ For Python it must be at the beginning of the file! */
int x; int x;
/* XXX The case to (unsigned char *) is needed by THINK C 3.0 */ /* XXX The cast to (unsigned char *) is needed by THINK C 3.0 */
if (sscanf(/*(unsigned char *)*/tok->cur, if (sscanf(/*(unsigned char *)*/tok->cur,
" vi:set tabsize=%d:", &x) == 1 && " vi:set tabsize=%d:", &x) == 1 &&
x >= 1 && x <= 40) { x >= 1 && x <= 40) {
@ -451,8 +459,10 @@ tok_get(tok, p_start, p_end)
} }
/* Check for EOF and errors now */ /* Check for EOF and errors now */
if (c == EOF) if (c == EOF) {
*p_start = *p_end = tok->cur;
return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN; return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
}
/* Identifier (most frequent token!) */ /* Identifier (most frequent token!) */
if (isalpha(c) || c == '_') { if (isalpha(c) || c == '_') {
@ -473,6 +483,19 @@ tok_get(tok, p_start, p_end)
return NEWLINE; return NEWLINE;
} }
/* Period or number starting with period? */
if (c == '.') {
c = tok_nextc(tok);
if (isdigit(c)) {
goto fraction;
}
else {
tok_backup(tok, c);
*p_end = tok->cur;
return DOT;
}
}
/* Number */ /* Number */
if (isdigit(c)) { if (isdigit(c)) {
if (c == '0') { if (c == '0') {