mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-11-15 00:00:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Make the example a little more interesting and useful.

Browse source
This commit is contained in:
Raymond Hettinger 2010-12-07 09:44:21 +00:00
parent 673ccf20dc
commit c2c7c37377
1 changed files with 3 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Doc/library/re.rst
View file

@ -1300,6 +1300,7 @@ successive matches::
Token = collections.namedtuple('Token', 'typ value line column') Token = collections.namedtuple('Token', 'typ value line column')
def tokenize(s): def tokenize(s):
keywords = {'IF', 'THEN', 'FOR', 'NEXT', 'GOSUB', 'RETURN'}
tok_spec = [ tok_spec = [
('NUMBER', r'\d+(\.\d*)?'), # Integer or decimal number ('NUMBER', r'\d+(\.\d*)?'), # Integer or decimal number
('ASSIGN', r':='), # Assignment operator ('ASSIGN', r':='), # Assignment operator
@ -1320,6 +1321,8 @@ successive matches::
line_start = pos line_start = pos
line += 1 line += 1
elif typ != 'SKIP': elif typ != 'SKIP':
if typ == 'ID' and val in keywords:
typ = val
yield Token(typ, mo.group(typ), line, mo.start()-line_start) yield Token(typ, mo.group(typ), line, mo.start()-line_start)
pos = mo.end() pos = mo.end()
mo = gettok(s, pos) mo = gettok(s, pos)