Skip to content

Commit 5ca576e

Browse files
tim-onetim-one
committed
Merging the gen-branch into the main line, at Guido's direction. Yay!
Bugfix candidate in inspect.py: it was referencing "self" outside of a method.
1 parent 1dad6a8 commit 5ca576e

16 files changed

Lines changed: 1263 additions & 1082 deletions

File tree

‎Grammar/Grammar‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,11 @@ augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>='
4343
print_stmt: 'print' ( [ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ] )
4444
del_stmt: 'del' exprlist
4545
pass_stmt: 'pass'
46-
flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
46+
flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
4747
break_stmt: 'break'
4848
continue_stmt: 'continue'
4949
return_stmt: 'return' [testlist]
50+
yield_stmt: 'yield' testlist
5051
raise_stmt: 'raise' [test [',' test [',' test]]]
5152
import_stmt: 'import' dotted_as_name (',' dotted_as_name)* | 'from' dotted_name 'import' ('*' | import_as_name (',' import_as_name)*)
5253
import_as_name: NAME [NAME NAME]

‎Include/compile.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ typedef struct {
3333
#define CO_VARARGS 0x0004
3434
#define CO_VARKEYWORDS 0x0008
3535
#define CO_NESTED 0x0010
36+
#define CO_GENERATOR 0x0020
3637

3738
extern DL_IMPORT(PyTypeObject) PyCode_Type;
3839

‎Include/frameobject.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ typedef struct _frame {
2121
PyObject *f_globals; /* global symbol table (PyDictObject) */
2222
PyObject *f_locals; /* local symbol table (PyDictObject) */
2323
PyObject **f_valuestack; /* points after the last local */
24+
PyObject **f_stackbottom; /* points to the last item on the stack if
25+
frame has yielded. */
2426
PyObject *f_trace; /* Trace function */
2527
PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
2628
PyThreadState *f_tstate;

‎Include/graminit.h‎

Lines changed: 45 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -18,47 +18,48 @@
1818
#define break_stmt 273
1919
#define continue_stmt 274
2020
#define return_stmt 275
21-
#define raise_stmt 276
22-
#define import_stmt 277
23-
#define import_as_name 278
24-
#define dotted_as_name 279
25-
#define dotted_name 280
26-
#define global_stmt 281
27-
#define exec_stmt 282
28-
#define assert_stmt 283
29-
#define compound_stmt 284
30-
#define if_stmt 285
31-
#define while_stmt 286
32-
#define for_stmt 287
33-
#define try_stmt 288
34-
#define except_clause 289
35-
#define suite 290
36-
#define test 291
37-
#define and_test 292
38-
#define not_test 293
39-
#define comparison 294
40-
#define comp_op 295
41-
#define expr 296
42-
#define xor_expr 297
43-
#define and_expr 298
44-
#define shift_expr 299
45-
#define arith_expr 300
46-
#define term 301
47-
#define factor 302
48-
#define power 303
49-
#define atom 304
50-
#define listmaker 305
51-
#define lambdef 306
52-
#define trailer 307
53-
#define subscriptlist 308
54-
#define subscript 309
55-
#define sliceop 310
56-
#define exprlist 311
57-
#define testlist 312
58-
#define dictmaker 313
59-
#define classdef 314
60-
#define arglist 315
61-
#define argument 316
62-
#define list_iter 317
63-
#define list_for 318
64-
#define list_if 319
21+
#define yield_stmt 276
22+
#define raise_stmt 277
23+
#define import_stmt 278
24+
#define import_as_name 279
25+
#define dotted_as_name 280
26+
#define dotted_name 281
27+
#define global_stmt 282
28+
#define exec_stmt 283
29+
#define assert_stmt 284
30+
#define compound_stmt 285
31+
#define if_stmt 286
32+
#define while_stmt 287
33+
#define for_stmt 288
34+
#define try_stmt 289
35+
#define except_clause 290
36+
#define suite 291
37+
#define test 292
38+
#define and_test 293
39+
#define not_test 294
40+
#define comparison 295
41+
#define comp_op 296
42+
#define expr 297
43+
#define xor_expr 298
44+
#define and_expr 299
45+
#define shift_expr 300
46+
#define arith_expr 301
47+
#define term 302
48+
#define factor 303
49+
#define power 304
50+
#define atom 305
51+
#define listmaker 306
52+
#define lambdef 307
53+
#define trailer 308
54+
#define subscriptlist 309
55+
#define subscript 310
56+
#define sliceop 311
57+
#define exprlist 312
58+
#define testlist 313
59+
#define dictmaker 314
60+
#define classdef 315
61+
#define arglist 316
62+
#define argument 317
63+
#define list_iter 318
64+
#define list_for 319
65+
#define list_if 320

‎Include/opcode.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ extern "C" {
7171
#define RETURN_VALUE 83
7272
#define IMPORT_STAR 84
7373
#define EXEC_STMT 85
74+
#define YIELD_VALUE 86
7475

7576
#define POP_BLOCK 87
7677
#define END_FINALLY 88

‎Include/symtable.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ typedef struct _symtable_entry {
4646
int ste_nested; /* true if scope is nested */
4747
int ste_child_free; /* true if a child scope has free variables,
4848
including free refs to globals */
49+
int ste_generator; /* true if namespace is a generator */
4950
int ste_opt_lineno; /* lineno of last exec or import * */
5051
struct symtable *ste_table;
5152
} PySymtableEntryObject;

‎Lib/dis.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,7 @@ def jabs_op(name, op):
223223
def_op('RETURN_VALUE', 83)
224224
def_op('IMPORT_STAR', 84)
225225
def_op('EXEC_STMT', 85)
226+
def_op('YIELD_STMT', 86)
226227

227228
def_op('POP_BLOCK', 87)
228229
def_op('END_FINALLY', 88)

‎Lib/inspect.py‎

Lines changed: 19 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -349,32 +349,28 @@ def readline(self):
349349
return self.lines[i]
350350
else: return ''
351351

352-
class EndOfBlock(Exception): pass
353-
354-
class BlockFinder:
355-
"""Provide a tokeneater() method to detect the end of a code block."""
356-
def __init__(self):
357-
self.indent = 0
358-
self.started = 0
359-
self.last = 0
360-
361-
def tokeneater(self, type, token, (srow, scol), (erow, ecol), line):
362-
if not self.started:
363-
if type == tokenize.NAME: self.started = 1
352+
def getblock(lines):
353+
"""Extract the block of code at the top of the given list of lines."""
354+
355+
indent = 0
356+
started = 0
357+
last = 0
358+
tokens = tokenize.generate_tokens(ListReader(lines).readline)
359+
360+
for (type, token, (srow, scol), (erow, ecol), line) in tokens:
361+
if not started:
362+
if type == tokenize.NAME:
363+
started = 1
364364
elif type == tokenize.NEWLINE:
365-
self.last = srow
365+
last = srow
366366
elif type == tokenize.INDENT:
367-
self.indent = self.indent + 1
367+
indent = indent + 1
368368
elif type == tokenize.DEDENT:
369-
self.indent = self.indent - 1
370-
if self.indent == 0: raise EndOfBlock, self.last
371-
372-
def getblock(lines):
373-
"""Extract the block of code at the top of the given list of lines."""
374-
try:
375-
tokenize.tokenize(ListReader(lines).readline, BlockFinder().tokeneater)
376-
except EndOfBlock, eob:
377-
return lines[:eob.args[0]]
369+
indent = indent - 1
370+
if indent == 0:
371+
return lines[:last]
372+
else:
373+
raise ValueError, "unable to find block"
378374

379375
def getsourcelines(object):
380376
"""Return a list of source lines and starting line number for an object.

‎Lib/tabnanny.py‎

Lines changed: 9 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -77,9 +77,8 @@ def check(file):
7777
if verbose > 1:
7878
print "checking", `file`, "..."
7979

80-
reset_globals()
8180
try:
82-
tokenize.tokenize(f.readline, tokeneater)
81+
process_tokens(tokenize.generate_tokens(f.readline))
8382

8483
except tokenize.TokenError, msg:
8584
errprint("%s: Token Error: %s" % (`file`, str(msg)))
@@ -244,28 +243,19 @@ def format_witnesses(w):
244243
prefix = prefix + "s"
245244
return prefix + " " + string.join(firsts, ', ')
246245

247-
# The collection of globals, the reset_globals() function, and the
248-
# tokeneater() function, depend on which version of tokenize is
249-
# in use.
246+
# Need Guido's enhancement
247+
assert hasattr(tokenize, 'NL'), "tokenize module too old"
250248

251-
if hasattr(tokenize, 'NL'):
252-
# take advantage of Guido's patch!
253-
254-
indents = []
255-
check_equal = 0
256-
257-
def reset_globals():
258-
global indents, check_equal
259-
check_equal = 0
260-
indents = [Whitespace("")]
261-
262-
def tokeneater(type, token, start, end, line,
249+
def process_tokens(tokens,
263250
INDENT=tokenize.INDENT,
264251
DEDENT=tokenize.DEDENT,
265252
NEWLINE=tokenize.NEWLINE,
266-
JUNK=(tokenize.COMMENT, tokenize.NL) ):
267-
global indents, check_equal
253+
JUNK=(tokenize.COMMENT, tokenize.NL)):
268254

255+
indents = [Whitespace("")]
256+
check_equal = 0
257+
258+
for (type, token, start, end, line) in tokens:
269259
if type == NEWLINE:
270260
# a program statement, or ENDMARKER, will eventually follow,
271261
# after some (possibly empty) run of tokens of the form
@@ -311,62 +301,6 @@ def tokeneater(type, token, start, end, line,
311301
msg = "indent not equal e.g. " + format_witnesses(witness)
312302
raise NannyNag(start[0], msg, line)
313303

314-
else:
315-
# unpatched version of tokenize
316-
317-
nesting_level = 0
318-
indents = []
319-
check_equal = 0
320-
321-
def reset_globals():
322-
global nesting_level, indents, check_equal
323-
nesting_level = check_equal = 0
324-
indents = [Whitespace("")]
325-
326-
def tokeneater(type, token, start, end, line,
327-
INDENT=tokenize.INDENT,
328-
DEDENT=tokenize.DEDENT,
329-
NEWLINE=tokenize.NEWLINE,
330-
COMMENT=tokenize.COMMENT,
331-
OP=tokenize.OP):
332-
global nesting_level, indents, check_equal
333-
334-
if type == INDENT:
335-
check_equal = 0
336-
thisguy = Whitespace(token)
337-
if not indents[-1].less(thisguy):
338-
witness = indents[-1].not_less_witness(thisguy)
339-
msg = "indent not greater e.g. " + format_witnesses(witness)
340-
raise NannyNag(start[0], msg, line)
341-
indents.append(thisguy)
342-
343-
elif type == DEDENT:
344-
del indents[-1]
345-
346-
elif type == NEWLINE:
347-
if nesting_level == 0:
348-
check_equal = 1
349-
350-
elif type == COMMENT:
351-
pass
352-
353-
elif check_equal:
354-
check_equal = 0
355-
thisguy = Whitespace(line)
356-
if not indents[-1].equal(thisguy):
357-
witness = indents[-1].not_equal_witness(thisguy)
358-
msg = "indent not equal e.g. " + format_witnesses(witness)
359-
raise NannyNag(start[0], msg, line)
360-
361-
if type == OP and token in ('{', '[', '('):
362-
nesting_level = nesting_level + 1
363-
364-
elif type == OP and token in ('}', ']', ')'):
365-
if nesting_level == 0:
366-
raise NannyNag(start[0],
367-
"unbalanced bracket '" + token + "'",
368-
line)
369-
nesting_level = nesting_level - 1
370304

371305
if __name__ == '__main__':
372306
main()

0 commit comments

Comments
 (0)