sre.Scanner fixes (from Greg Chapman). also added a Scanner sanity

check to the test suite.

added a few missing exception checks in the _sre module

Lib/test/test_sre.py

@@ -223,6 +223,26 @@ pat = sre.compile(sre.escape(p))
 test(r"""pat.match(p) is not None""", 1)
 test(r"""pat.match(p).span()""", (0,256))
 
+if verbose:
+    print 'Running tests on sre.Scanner'
+
+def s_ident(scanner, token): return token
+def s_operator(scanner, token): return "op%s" % token
+def s_float(scanner, token): return float(token)
+def s_int(scanner, token): return int(token)
+
+scanner = sre.Scanner([
+    (r"[a-zA-Z_]\w*", s_ident),
+    (r"\d+\.\d*", s_float),
+    (r"\d+", s_int),
+    (r"=|\+|-|\*|/", s_operator),
+    (r"\s+", None),
+    ])
+
+# sanity check
+test('scanner.scan("sum = 3*foo + 312.50 + bar")',
+     (['sum', 'op=', 3, 'op*', 'foo', 'op+', 312.5, 'op+', 'bar'], ''))
+
 if verbose:
     print 'Pickling a SRE_Pattern instance'