mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-04 17:08:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

gh-97669: Remove outdated example scripts (#97675)

Browse source 
Remove outdated example scripts of the Tools/scripts/ directory. A
copy can be found in the old-demos project:
https://github.com/gvanrossum/old-demos

Removed scripts (39):

* byext.py
* byteyears.py
* cleanfuture.py
* copytime.py
* crlf.py
* db2pickle.py
* dutree.doc
* dutree.py
* find-uname.py
* find_recursionlimit.py
* finddiv.py
* findlinksto.py
* findnocoding.py
* fixcid.py
* fixdiv.py
* fixheader.py
* fixnotice.py
* fixps.py
* get-remote-certificate.py
* google.py
* highlight.py
* ifdef.py
* import_diagnostics.py
* lfcr.py
* linktree.py
* lll.py
* mailerdaemon.py
* make_ctype.py
* mkreal.py
* objgraph.py
* pdeps.py
* pickle2db.py
* pindent.py
* pysource.py
* reindent-rst.py
* rgrep.py
* suff.py
* texi2html.py
* which.py

Changes:

* Remove test_fixcid, test_lll, test_pdeps and test_pindent
  of test.test_tools.
* Remove get-remote-certificate.py changelog entry, since the script
  was removed.

Note: there is a copy of crlf.py in Lib/test/test_lib2to3/data/.
This commit is contained in:
Victor Stinner 2022-10-04 10:49:00 +02:00 • committed by GitHub
parent 49802605f8
commit 6cbbc26a73
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
48 changed files with 9 additions and 7050 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Doc/whatsnew/3.12.rst
View file

@ -178,6 +178,11 @@ Demos and Tools
<https://github.com/gvanrossum/old-demos>`_.
(Contributed by Victor Stinner in :gh:`97681`.)
* Remove outdated example scripts of the ``Tools/scripts/`` directory.
A copy can be found in the `old-demos project
<https://github.com/gvanrossum/old-demos>`_.
(Contributed by Victor Stinner in :gh:`97669`.)
Deprecated
==========

94
Lib/test/test_tools/test_fixcid.py
View file

@ -1,94 +0,0 @@
'''Test Tools/scripts/fixcid.py.'''
from io import StringIO
import os, os.path
import runpy
import sys
from test import support
from test.support import os_helper
from test.test_tools import skip_if_missing, scriptsdir
import unittest
skip_if_missing()
class Test(unittest.TestCase):
def test_parse_strings(self):
old1 = 'int xx = "xx\\"xx"[xx];\n'
old2 = "int xx = 'x\\'xx' + xx;\n"
output = self.run_script(old1 + old2)
new1 = 'int yy = "xx\\"xx"[yy];\n'
new2 = "int yy = 'x\\'xx' + yy;\n"
self.assertMultiLineEqual(output,
"1\n"
"< {old1}"
"> {new1}"
"{new1}"
"2\n"
"< {old2}"
"> {new2}"
"{new2}".format(old1=old1, old2=old2, new1=new1, new2=new2)
)
def test_alter_comments(self):
output = self.run_script(
substfile=
"xx yy\n"
"*aa bb\n",
args=("-c", "-",),
input=
"/* xx altered */\n"
"int xx;\n"
"/* aa unaltered */\n"
"int aa;\n",
)
self.assertMultiLineEqual(output,
"1\n"
"< /* xx altered */\n"
"> /* yy altered */\n"
"/* yy altered */\n"
"2\n"
"< int xx;\n"
"> int yy;\n"
"int yy;\n"
"/* aa unaltered */\n"
"4\n"
"< int aa;\n"
"> int bb;\n"
"int bb;\n"
)
def test_directory(self):
os.mkdir(os_helper.TESTFN)
self.addCleanup(os_helper.rmtree, os_helper.TESTFN)
c_filename = os.path.join(os_helper.TESTFN, "file.c")
with open(c_filename, "w", encoding="utf-8") as file:
file.write("int xx;\n")
with open(os.path.join(os_helper.TESTFN, "file.py"), "w",
encoding="utf-8") as file:
file.write("xx = 'unaltered'\n")
script = os.path.join(scriptsdir, "fixcid.py")
output = self.run_script(args=(os_helper.TESTFN,))
self.assertMultiLineEqual(output,
"{}:\n"
"1\n"
'< int xx;\n'
'> int yy;\n'.format(c_filename)
)
def run_script(self, input="", *, args=("-",), substfile="xx yy\n"):
substfilename = os_helper.TESTFN + ".subst"
with open(substfilename, "w", encoding="utf-8") as file:
file.write(substfile)
self.addCleanup(os_helper.unlink, substfilename)
argv = ["fixcid.py", "-s", substfilename] + list(args)
script = os.path.join(scriptsdir, "fixcid.py")
with support.swap_attr(sys, "argv", argv), \
support.swap_attr(sys, "stdin", StringIO(input)), \
support.captured_stdout() as output, \
support.captured_stderr():
try:
runpy.run_path(script, run_name="__main__")
except SystemExit as exit:
self.assertEqual(exit.code, 0)
return output.getvalue()

41
Lib/test/test_tools/test_lll.py
View file

@ -1,41 +0,0 @@
"""Tests for the lll script in the Tools/script directory."""
import os
import tempfile
from test import support
from test.support import os_helper
from test.test_tools import skip_if_missing, import_tool
import unittest
skip_if_missing()
class lllTests(unittest.TestCase):
def setUp(self):
self.lll = import_tool('lll')
@os_helper.skip_unless_symlink
def test_lll_multiple_dirs(self):
with tempfile.TemporaryDirectory() as dir1, \
tempfile.TemporaryDirectory() as dir2:
fn1 = os.path.join(dir1, 'foo1')
fn2 = os.path.join(dir2, 'foo2')
for fn, dir in (fn1, dir1), (fn2, dir2):
open(fn, 'wb').close()
os.symlink(fn, os.path.join(dir, 'symlink'))
with support.captured_stdout() as output:
self.lll.main([dir1, dir2])
prefix = '\\\\?\\' if os.name == 'nt' else ''
self.assertEqual(output.getvalue(),
f'{dir1}:\n'
f'symlink -> {prefix}{fn1}\n'
f'\n'
f'{dir2}:\n'
f'symlink -> {prefix}{fn2}\n'
)
if __name__ == '__main__':
unittest.main()

32
Lib/test/test_tools/test_pdeps.py
View file

@ -1,32 +0,0 @@
"""Tests for the pdeps script in the Tools directory."""
import os
import unittest
import tempfile
from test.test_tools import skip_if_missing, import_tool
skip_if_missing()
class PdepsTests(unittest.TestCase):
@classmethod
def setUpClass(self):
self.pdeps = import_tool('pdeps')
def test_process_errors(self):
# Issue #14492: m_import.match(line) can be None.
with tempfile.TemporaryDirectory() as tmpdir:
fn = os.path.join(tmpdir, 'foo')
with open(fn, 'w', encoding='utf-8') as stream:
stream.write("#!/this/will/fail")
self.pdeps.process(fn, {})
def test_inverse_attribute_error(self):
# Issue #14492: this used to fail with an AttributeError.
self.pdeps.inverse({'a': []})
if __name__ == '__main__':
unittest.main()

339
Lib/test/test_tools/test_pindent.py
View file

@ -1,339 +0,0 @@
"""Tests for the pindent script in the Tools directory."""
import os
import sys
import unittest
import subprocess
import textwrap
from test.support import os_helper
from test.support.script_helper import assert_python_ok
from test.test_tools import scriptsdir, skip_if_missing
skip_if_missing()
class PindentTests(unittest.TestCase):
script = os.path.join(scriptsdir, 'pindent.py')
def assertFileEqual(self, fn1, fn2):
with open(fn1) as f1, open(fn2) as f2:
self.assertEqual(f1.readlines(), f2.readlines())
def pindent(self, source, *args):
with subprocess.Popen(
(sys.executable, self.script) + args,
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
universal_newlines=True) as proc:
out, err = proc.communicate(source)
self.assertIsNone(err)
return out
def lstriplines(self, data):
return '\n'.join(line.lstrip() for line in data.splitlines()) + '\n'
def test_selftest(self):
self.maxDiff = None
with os_helper.temp_dir() as directory:
data_path = os.path.join(directory, '_test.py')
with open(self.script, encoding='utf-8') as f:
closed = f.read()
with open(data_path, 'w', encoding='utf-8') as f:
f.write(closed)
rc, out, err = assert_python_ok(self.script, '-d', data_path)
self.assertEqual(out, b'')
self.assertEqual(err, b'')
backup = data_path + '~'
self.assertTrue(os.path.exists(backup))
with open(backup, encoding='utf-8') as f:
self.assertEqual(f.read(), closed)
with open(data_path, encoding='utf-8') as f:
clean = f.read()
compile(clean, '_test.py', 'exec')
self.assertEqual(self.pindent(clean, '-c'), closed)
self.assertEqual(self.pindent(closed, '-d'), clean)
rc, out, err = assert_python_ok(self.script, '-c', data_path)
self.assertEqual(out, b'')
self.assertEqual(err, b'')
with open(backup, encoding='utf-8') as f:
self.assertEqual(f.read(), clean)
with open(data_path, encoding='utf-8') as f:
self.assertEqual(f.read(), closed)
broken = self.lstriplines(closed)
with open(data_path, 'w', encoding='utf-8') as f:
f.write(broken)
rc, out, err = assert_python_ok(self.script, '-r', data_path)
self.assertEqual(out, b'')
self.assertEqual(err, b'')
with open(backup, encoding='utf-8') as f:
self.assertEqual(f.read(), broken)
with open(data_path, encoding='utf-8') as f:
indented = f.read()
compile(indented, '_test.py', 'exec')
self.assertEqual(self.pindent(broken, '-r'), indented)
def pindent_test(self, clean, closed):
self.assertEqual(self.pindent(clean, '-c'), closed)
self.assertEqual(self.pindent(closed, '-d'), clean)
broken = self.lstriplines(closed)
self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '4'), closed)
def test_statements(self):
clean = textwrap.dedent("""\
if a:
pass
if a:
pass
else:
pass
if a:
pass
elif:
pass
else:
pass
while a:
break
while a:
break
else:
pass
for i in a:
break
for i in a:
break
else:
pass
try:
pass
finally:
pass
try:
pass
except TypeError:
pass
except ValueError:
pass
else:
pass
try:
pass
except TypeError:
pass
except ValueError:
pass
finally:
pass
with a:
pass
class A:
pass
def f():
pass
""")
closed = textwrap.dedent("""\
if a:
pass
# end if
if a:
pass
else:
pass
# end if
if a:
pass
elif:
pass
else:
pass
# end if
while a:
break
# end while
while a:
break
else:
pass
# end while
for i in a:
break
# end for
for i in a:
break
else:
pass
# end for
try:
pass
finally:
pass
# end try
try:
pass
except TypeError:
pass
except ValueError:
pass
else:
pass
# end try
try:
pass
except TypeError:
pass
except ValueError:
pass
finally:
pass
# end try
with a:
pass
# end with
class A:
pass
# end class A
def f():
pass
# end def f
""")
self.pindent_test(clean, closed)
def test_multilevel(self):
clean = textwrap.dedent("""\
def foobar(a, b):
if a == b:
a = a+1
elif a < b:
b = b-1
if b > a: a = a-1
else:
print 'oops!'
""")
closed = textwrap.dedent("""\
def foobar(a, b):
if a == b:
a = a+1
elif a < b:
b = b-1
if b > a: a = a-1
# end if
else:
print 'oops!'
# end if
# end def foobar
""")
self.pindent_test(clean, closed)
def test_preserve_indents(self):
clean = textwrap.dedent("""\
if a:
if b:
pass
""")
closed = textwrap.dedent("""\
if a:
if b:
pass
# end if
# end if
""")
self.assertEqual(self.pindent(clean, '-c'), closed)
self.assertEqual(self.pindent(closed, '-d'), clean)
broken = self.lstriplines(closed)
self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '9'), closed)
clean = textwrap.dedent("""\
if a:
\tif b:
\t\tpass
""")
closed = textwrap.dedent("""\
if a:
\tif b:
\t\tpass
\t# end if
# end if
""")
self.assertEqual(self.pindent(clean, '-c'), closed)
self.assertEqual(self.pindent(closed, '-d'), clean)
broken = self.lstriplines(closed)
self.assertEqual(self.pindent(broken, '-r'), closed)
def test_escaped_newline(self):
clean = textwrap.dedent("""\
class\\
\\
A:
def\
\\
f:
pass
""")
closed = textwrap.dedent("""\
class\\
\\
A:
def\
\\
f:
pass
# end def f
# end class A
""")
self.assertEqual(self.pindent(clean, '-c'), closed)
self.assertEqual(self.pindent(closed, '-d'), clean)
def test_empty_line(self):
clean = textwrap.dedent("""\
if a:
pass
""")
closed = textwrap.dedent("""\
if a:
pass
# end if
""")
self.pindent_test(clean, closed)
def test_oneline(self):
clean = textwrap.dedent("""\
if a: pass
""")
closed = textwrap.dedent("""\
if a: pass
# end if
""")
self.pindent_test(clean, closed)
if __name__ == '__main__':
unittest.main()

3
Misc/NEWS.d/next/Security/2022-09-28-12-10-57.gh-issue-97612.y6NvOQ.rst
View file

@ -1,3 +0,0 @@
Fix a shell code injection vulnerability in the ``get-remote-certificate.py``
example script. The script no longer uses a shell to run ``openssl`` commands.
Issue reported and initial fix by Caleb Shortt. Patch by Victor Stinner.

3
Misc/NEWS.d/next/Tools-Demos/2022-09-30-14-30-12.gh-issue-97669.gvbgcg.rst Normal file
View file

@ -0,0 +1,3 @@
Remove outdated example scripts of the ``Tools/scripts/`` directory. A copy can
be found in the `old-demos project <https://github.com/gvanrossum/old-demos>`_.
Patch by Victor Stinner.

3
PCbuild/lib.pyproj
View file

@ -1334,12 +1334,9 @@
<Compile Include="test\test_tkinter\test_widgets.py" />
<Compile Include="test\test_tkinter\widget_tests.py" />
<Compile Include="test\test_tokenize.py" />
<Compile Include="test\test_tools\test_fixcid.py" />
<Compile Include="test\test_tools\test_gprof2html.py" />
<Compile Include="test\test_tools\test_i18n.py" />
<Compile Include="test\test_tools\test_md5sum.py" />
<Compile Include="test\test_tools\test_pdeps.py" />
<Compile Include="test\test_tools\test_pindent.py" />
<Compile Include="test\test_tools\test_reindent.py" />
<Compile Include="test\test_tools\test_sundry.py" />
<Compile Include="test\test_tools\test_unparse.py" />

42
Tools/scripts/README
View file

@ -1,65 +1,25 @@
This directory contains a collection of executable Python scripts that are
useful while building, extending or managing Python. Some (e.g., dutree or lll)
are also generally useful UNIX tools.
useful while building, extending or managing Python.
2to3 Main script for running the 2to3 conversion tool
abitype.py Converts a C file to use the PEP 384 type definition API
analyze_dxp.py Analyzes the result of sys.getdxp()
byext.py Print lines/words/chars stats of files by extension
byteyears.py Print product of a file's size and age
cleanfuture.py Fix redundant Python __future__ statements
combinerefs.py A helper for analyzing PYTHONDUMPREFS output
copytime.py Copy one file's atime and mtime to another
crlf.py Change CRLF line endings to LF (Windows to Unix)
db2pickle.py Dump a database file to a pickle
diff.py Print file diffs in context, unified, or ndiff formats
dutree.py Format du(1) output as a tree sorted by size
eptags.py Create Emacs TAGS file for Python modules
finddiv.py A grep-like tool that looks for division operators
findlinksto.py Recursively find symbolic links to a given path prefix
findnocoding.py Find source files which need an encoding declaration
find_recursionlimit.py Find the maximum recursion limit on this machine
find-uname.py Look for the given arguments in the sets of all Unicode names
fixcid.py Massive identifier substitution on C source files
fixdiv.py Tool to fix division operators.
fixheader.py Add some cpp magic to a C include file
fixnotice.py Fix the copyright notice in source files
fixps.py Fix Python scripts' first line (if #!)
ftpmirror.py FTP mirror script
get-remote-certificate.py Fetch the certificate that the server(s) are providing in PEM form
google.py Open a webbrowser with Google
gprof2html.py Transform gprof(1) output into useful HTML
highlight.py Python syntax highlighting with HTML output
idle3 Main program to start IDLE
ifdef.py Remove #if(n)def groups from C sources
import_diagnostics.py Miscellaneous diagnostics for the import system
lfcr.py Change LF line endings to CRLF (Unix to Windows)
linktree.py Make a copy of a tree with links to original files
lll.py Find and list symbolic links in current directory
mailerdaemon.py Parse error messages from mailer daemons (Sjoerd&Jack)
make_ctype.py Generate ctype.h replacement in stringobject.c
md5sum.py Print MD5 checksums of argument files
mkreal.py Turn a symbolic link into a real file or directory
ndiff.py Intelligent diff between text files (Tim Peters)
nm2def.py Create a template for PC/python_nt.def (Marc Lemburg)
objgraph.py Print object graph from nm output on a library
parseentities.py Utility for parsing HTML entity definitions
parse_html5_entities.py Utility for parsing HTML5 entity definitions
patchcheck.py Perform common checks and cleanup before committing
pathfix.py Change #!/usr/local/bin/python into something else
pdeps.py Print dependencies between Python modules
pickle2db.py Load a pickle generated by db2pickle.py to a database
pindent.py Indent Python code, giving block-closing comments
ptags.py Create vi tags file for Python modules
pydoc3 Python documentation browser
pysource.py Find Python source files
reindent.py Change .py files to use 4-space indents
reindent-rst.py Fix-up reStructuredText file whitespace
rgrep.py Reverse grep through a file (useful for big logfiles)
run_tests.py Run the test suite with more sensible default options
stable_abi.py Stable ABI checks and file generators.
suff.py Sort a list of files by suffix
texi2html.py Convert GNU texinfo files into HTML
untabify.py Replace tabs with spaces in argument files
which.py Find a program in $PATH
win_add2path.py Add Python to the search path on Windows

132
Tools/scripts/byext.py
View file

@ -1,132 +0,0 @@
#! /usr/bin/env python3
"""Show file statistics by extension."""
import os
import sys
class Stats:
def __init__(self):
self.stats = {}
def statargs(self, args):
for arg in args:
if os.path.isdir(arg):
self.statdir(arg)
elif os.path.isfile(arg):
self.statfile(arg)
else:
sys.stderr.write("Can't find %s\n" % arg)
self.addstats("<???>", "unknown", 1)
def statdir(self, dir):
self.addstats("<dir>", "dirs", 1)
try:
names = os.listdir(dir)
except OSError as err:
sys.stderr.write("Can't list %s: %s\n" % (dir, err))
self.addstats("<dir>", "unlistable", 1)
return
for name in sorted(names):
if name.startswith(".#"):
continue # Skip CVS temp files
if name.endswith("~"):
continue # Skip Emacs backup files
full = os.path.join(dir, name)
if os.path.islink(full):
self.addstats("<lnk>", "links", 1)
elif os.path.isdir(full):
self.statdir(full)
else:
self.statfile(full)
def statfile(self, filename):
head, ext = os.path.splitext(filename)
head, base = os.path.split(filename)
if ext == base:
ext = "" # E.g. .cvsignore is deemed not to have an extension
ext = os.path.normcase(ext)
if not ext:
ext = "<none>"
self.addstats(ext, "files", 1)
try:
with open(filename, "rb") as f:
data = f.read()
except IOError as err:
sys.stderr.write("Can't open %s: %s\n" % (filename, err))
self.addstats(ext, "unopenable", 1)
return
self.addstats(ext, "bytes", len(data))
if b'\0' in data:
self.addstats(ext, "binary", 1)
return
if not data:
self.addstats(ext, "empty", 1)
# self.addstats(ext, "chars", len(data))
lines = str(data, "latin-1").splitlines()
self.addstats(ext, "lines", len(lines))
del lines
words = data.split()
self.addstats(ext, "words", len(words))
def addstats(self, ext, key, n):
d = self.stats.setdefault(ext, {})
d[key] = d.get(key, 0) + n
def report(self):
exts = sorted(self.stats)
# Get the column keys
columns = {}
for ext in exts:
columns.update(self.stats[ext])
cols = sorted(columns)
colwidth = {}
colwidth["ext"] = max(map(len, exts))
minwidth = 6
self.stats["TOTAL"] = {}
for col in cols:
total = 0
cw = max(minwidth, len(col))
for ext in exts:
value = self.stats[ext].get(col)
if value is None:
w = 0
else:
w = len("%d" % value)
total += value
cw = max(cw, w)
cw = max(cw, len(str(total)))
colwidth[col] = cw
self.stats["TOTAL"][col] = total
exts.append("TOTAL")
for ext in exts:
self.stats[ext]["ext"] = ext
cols.insert(0, "ext")
def printheader():
for col in cols:
print("%*s" % (colwidth[col], col), end=' ')
print()
printheader()
for ext in exts:
for col in cols:
value = self.stats[ext].get(col, "")
print("%*s" % (colwidth[col], value), end=' ')
print()
printheader() # Another header at the bottom
def main():
args = sys.argv[1:]
if not args:
args = [os.curdir]
s = Stats()
s.statargs(args)
s.report()
if __name__ == "__main__":
main()

61
Tools/scripts/byteyears.py
View file

@ -1,61 +0,0 @@
#! /usr/bin/env python3
# Print the product of age and size of each file, in suitable units.
#
# Usage: byteyears [ -a | -m | -c ] file ...
#
# Options -[amc] select atime, mtime (default) or ctime as age.
import sys, os, time
from stat import *
def main():
# Use lstat() to stat files if it exists, else stat()
try:
statfunc = os.lstat
except AttributeError:
statfunc = os.stat
# Parse options
if sys.argv[1] == '-m':
itime = ST_MTIME
del sys.argv[1]
elif sys.argv[1] == '-c':
itime = ST_CTIME
del sys.argv[1]
elif sys.argv[1] == '-a':
itime = ST_CTIME
del sys.argv[1]
else:
itime = ST_MTIME
secs_per_year = 365.0 * 24.0 * 3600.0 # Scale factor
now = time.time() # Current time, for age computations
status = 0 # Exit status, set to 1 on errors
# Compute max file name length
maxlen = 1
for filename in sys.argv[1:]:
maxlen = max(maxlen, len(filename))
# Process each argument in turn
for filename in sys.argv[1:]:
try:
st = statfunc(filename)
except OSError as msg:
sys.stderr.write("can't stat %r: %r\n" % (filename, msg))
status = 1
st = ()
if st:
anytime = st[itime]
size = st[ST_SIZE]
age = now - anytime
byteyears = float(size) * float(age) / secs_per_year
print(filename.ljust(maxlen), end=' ')
print(repr(int(byteyears)).rjust(8))
sys.exit(status)
if __name__ == '__main__':
main()

275
Tools/scripts/cleanfuture.py
View file

@ -1,275 +0,0 @@
#! /usr/bin/env python3
"""cleanfuture [-d][-r][-v] path ...
-d Dry run. Analyze, but don't make any changes to, files.
-r Recurse. Search for all .py files in subdirectories too.
-v Verbose. Print informative msgs.
Search Python (.py) files for future statements, and remove the features
from such statements that are already mandatory in the version of Python
you're using.
Pass one or more file and/or directory paths. When a directory path, all
.py files within the directory will be examined, and, if the -r option is
given, likewise recursively for subdirectories.
Overwrites files in place, renaming the originals with a .bak extension. If
cleanfuture finds nothing to change, the file is left alone. If cleanfuture
does change a file, the changed file is a fixed-point (i.e., running
cleanfuture on the resulting .py file won't change it again, at least not
until you try it again with a later Python release).
Limitations: You can do these things, but this tool won't help you then:
+ A future statement cannot be mixed with any other statement on the same
physical line (separated by semicolon).
+ A future statement cannot contain an "as" clause.
Example: Assuming you're using Python 2.2, if a file containing
from __future__ import nested_scopes, generators
is analyzed by cleanfuture, the line is rewritten to
from __future__ import generators
because nested_scopes is no longer optional in 2.2 but generators is.
"""
import __future__
import tokenize
import os
import sys
dryrun = 0
recurse = 0
verbose = 0
def errprint(*args):
strings = map(str, args)
msg = ' '.join(strings)
if msg[-1:] != '\n':
msg += '\n'
sys.stderr.write(msg)
def main():
import getopt
global verbose, recurse, dryrun
try:
opts, args = getopt.getopt(sys.argv[1:], "drv")
except getopt.error as msg:
errprint(msg)
return
for o, a in opts:
if o == '-d':
dryrun += 1
elif o == '-r':
recurse += 1
elif o == '-v':
verbose += 1
if not args:
errprint("Usage:", __doc__)
return
for arg in args:
check(arg)
def check(file):
if os.path.isdir(file) and not os.path.islink(file):
if verbose:
print("listing directory", file)
names = os.listdir(file)
for name in names:
fullname = os.path.join(file, name)
if ((recurse and os.path.isdir(fullname) and
not os.path.islink(fullname))
or name.lower().endswith(".py")):
check(fullname)
return
if verbose:
print("checking", file, "...", end=' ')
try:
f = open(file)
except IOError as msg:
errprint("%r: I/O Error: %s" % (file, str(msg)))
return
with f:
ff = FutureFinder(f, file)
changed = ff.run()
if changed:
ff.gettherest()
if changed:
if verbose:
print("changed.")
if dryrun:
print("But this is a dry run, so leaving it alone.")
for s, e, line in changed:
print("%r lines %d-%d" % (file, s+1, e+1))
for i in range(s, e+1):
print(ff.lines[i], end=' ')
if line is None:
print("-- deleted")
else:
print("-- change to:")
print(line, end=' ')
if not dryrun:
bak = file + ".bak"
if os.path.exists(bak):
os.remove(bak)
os.rename(file, bak)
if verbose:
print("renamed", file, "to", bak)
with open(file, "w") as g:
ff.write(g)
if verbose:
print("wrote new", file)
else:
if verbose:
print("unchanged.")
class FutureFinder:
def __init__(self, f, fname):
self.f = f
self.fname = fname
self.ateof = 0
self.lines = [] # raw file lines
# List of (start_index, end_index, new_line) triples.
self.changed = []
# Line-getter for tokenize.
def getline(self):
if self.ateof:
return ""
line = self.f.readline()
if line == "":
self.ateof = 1
else:
self.lines.append(line)
return line
def run(self):
STRING = tokenize.STRING
NL = tokenize.NL
NEWLINE = tokenize.NEWLINE
COMMENT = tokenize.COMMENT
NAME = tokenize.NAME
OP = tokenize.OP
changed = self.changed
get = tokenize.generate_tokens(self.getline).__next__
type, token, (srow, scol), (erow, ecol), line = get()
# Chew up initial comments and blank lines (if any).
while type in (COMMENT, NL, NEWLINE):
type, token, (srow, scol), (erow, ecol), line = get()
# Chew up docstring (if any -- and it may be implicitly catenated!).
while type is STRING:
type, token, (srow, scol), (erow, ecol), line = get()
# Analyze the future stmts.
while 1:
# Chew up comments and blank lines (if any).
while type in (COMMENT, NL, NEWLINE):
type, token, (srow, scol), (erow, ecol), line = get()
if not (type is NAME and token == "from"):
break
startline = srow - 1 # tokenize is one-based
type, token, (srow, scol), (erow, ecol), line = get()
if not (type is NAME and token == "__future__"):
break
type, token, (srow, scol), (erow, ecol), line = get()
if not (type is NAME and token == "import"):
break
type, token, (srow, scol), (erow, ecol), line = get()
# Get the list of features.
features = []
while type is NAME:
features.append(token)
type, token, (srow, scol), (erow, ecol), line = get()
if not (type is OP and token == ','):
break
type, token, (srow, scol), (erow, ecol), line = get()
# A trailing comment?
comment = None
if type is COMMENT:
comment = token
type, token, (srow, scol), (erow, ecol), line = get()
if type is not NEWLINE:
errprint("Skipping file %r; can't parse line %d:\n%s" %
(self.fname, srow, line))
return []
endline = srow - 1
# Check for obsolete features.
okfeatures = []
for f in features:
object = getattr(__future__, f, None)
if object is None:
# A feature we don't know about yet -- leave it in.
# They'll get a compile-time error when they compile
# this program, but that's not our job to sort out.
okfeatures.append(f)
else:
released = object.getMandatoryRelease()
if released is None or released <= sys.version_info:
# Withdrawn or obsolete.
pass
else:
okfeatures.append(f)
# Rewrite the line if at least one future-feature is obsolete.
if len(okfeatures) < len(features):
if len(okfeatures) == 0:
line = None
else:
line = "from __future__ import "
line += ', '.join(okfeatures)
if comment is not None:
line += ' ' + comment
line += '\n'
changed.append((startline, endline, line))
# Loop back for more future statements.
return changed
def gettherest(self):
if self.ateof:
self.therest = ''
else:
self.therest = self.f.read()
def write(self, f):
changed = self.changed
assert changed
# Prevent calling this again.
self.changed = []
# Apply changes in reverse order.
changed.reverse()
for s, e, line in changed:
if line is None:
# pure deletion
del self.lines[s:e+1]
else:
self.lines[s:e+1] = [line]
f.writelines(self.lines)
# Copy over the remainder of the file.
if self.therest:
f.write(self.therest)
if __name__ == '__main__':
main()

26
Tools/scripts/copytime.py
View file

@ -1,26 +0,0 @@
#! /usr/bin/env python3
# Copy one file's atime and mtime to another
import sys
import os
from stat import ST_ATIME, ST_MTIME # Really constants 7 and 8
def main():
if len(sys.argv) != 3:
sys.stderr.write('usage: copytime source destination\n')
sys.exit(2)
file1, file2 = sys.argv[1], sys.argv[2]
try:
stat1 = os.stat(file1)
except OSError:
sys.stderr.write(file1 + ': cannot stat\n')
sys.exit(1)
try:
os.utime(file2, (stat1[ST_ATIME], stat1[ST_MTIME]))
except OSError:
sys.stderr.write(file2 + ': cannot change time\n')
sys.exit(2)
if __name__ == '__main__':
main()

23
Tools/scripts/crlf.py
View file

@ -1,23 +0,0 @@
#! /usr/bin/env python3
"Replace CRLF with LF in argument files. Print names of changed files."
import sys, os
def main():
for filename in sys.argv[1:]:
if os.path.isdir(filename):
print(filename, "Directory!")
continue
with open(filename, "rb") as f:
data = f.read()
if b'\0' in data:
print(filename, "Binary!")
continue
newdata = data.replace(b"\r\n", b"\n")
if newdata != data:
print(filename)
with open(filename, "wb") as f:
f.write(newdata)
if __name__ == '__main__':
main()

135
Tools/scripts/db2pickle.py
View file

@ -1,135 +0,0 @@
#!/usr/bin/env python3
"""
Synopsis: %(prog)s [-h|-g|-b|-r|-a] dbfile [ picklefile ]
Convert the database file given on the command line to a pickle
representation. The optional flags indicate the type of the database:
-a - open using dbm (any supported format)
-b - open as bsddb btree file
-d - open as dbm file
-g - open as gdbm file
-h - open as bsddb hash file
-r - open as bsddb recno file
The default is hash. If a pickle file is named it is opened for write
access (deleting any existing data). If no pickle file is named, the pickle
output is written to standard output.
"""
import getopt
try:
import bsddb
except ImportError:
bsddb = None
try:
import dbm.ndbm as dbm
except ImportError:
dbm = None
try:
import dbm.gnu as gdbm
except ImportError:
gdbm = None
try:
import dbm.ndbm as anydbm
except ImportError:
anydbm = None
import sys
try:
import pickle as pickle
except ImportError:
import pickle
prog = sys.argv[0]
def usage():
sys.stderr.write(__doc__ % globals())
def main(args):
try:
opts, args = getopt.getopt(args, "hbrdag",
["hash", "btree", "recno", "dbm",
"gdbm", "anydbm"])
except getopt.error:
usage()
return 1
if len(args) == 0 or len(args) > 2:
usage()
return 1
elif len(args) == 1:
dbfile = args[0]
pfile = sys.stdout
else:
dbfile = args[0]
try:
pfile = open(args[1], 'wb')
except IOError:
sys.stderr.write("Unable to open %s\n" % args[1])
return 1
dbopen = None
for opt, arg in opts:
if opt in ("-h", "--hash"):
try:
dbopen = bsddb.hashopen
except AttributeError:
sys.stderr.write("bsddb module unavailable.\n")
return 1
elif opt in ("-b", "--btree"):
try:
dbopen = bsddb.btopen
except AttributeError:
sys.stderr.write("bsddb module unavailable.\n")
return 1
elif opt in ("-r", "--recno"):
try:
dbopen = bsddb.rnopen
except AttributeError:
sys.stderr.write("bsddb module unavailable.\n")
return 1
elif opt in ("-a", "--anydbm"):
try:
dbopen = anydbm.open
except AttributeError:
sys.stderr.write("dbm module unavailable.\n")
return 1
elif opt in ("-g", "--gdbm"):
try:
dbopen = gdbm.open
except AttributeError:
sys.stderr.write("dbm.gnu module unavailable.\n")
return 1
elif opt in ("-d", "--dbm"):
try:
dbopen = dbm.open
except AttributeError:
sys.stderr.write("dbm.ndbm module unavailable.\n")
return 1
if dbopen is None:
if bsddb is None:
sys.stderr.write("bsddb module unavailable - ")
sys.stderr.write("must specify dbtype.\n")
return 1
else:
dbopen = bsddb.hashopen
try:
db = dbopen(dbfile, 'r')
except bsddb.error:
sys.stderr.write("Unable to open %s. " % dbfile)
sys.stderr.write("Check for format or version mismatch.\n")
return 1
for k in db.keys():
pickle.dump((k, db[k]), pfile, 1==1)
db.close()
pfile.close()
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))

54
Tools/scripts/dutree.doc
View file

@ -1,54 +0,0 @@
Path: cwi.nl!sun4nl!mcsun!uunet!cs.utexas.edu!convex!usenet
From: tchrist@convex.COM (Tom Christiansen)
Newsgroups: comp.lang.perl
Subject: Re: The problems of Perl (Re: Question (silly?))
Message-ID: <1992Jan17.053115.4220@convex.com>
Date: 17 Jan 92 05:31:15 GMT
References: <17458@ector.cs.purdue.edu> <1992Jan16.165347.25583@cherokee.uswest.com> <=#Hues+4@cs.psu.edu>
Sender: usenet@convex.com (news access account)
Reply-To: tchrist@convex.COM (Tom Christiansen)
Organization: CONVEX Realtime Development, Colorado Springs, CO
Lines: 83
Nntp-Posting-Host: pixel.convex.com
From the keyboard of flee@cs.psu.edu (Felix Lee):
:And Perl is definitely awkward with data types. I haven't yet found a
:pleasant way of shoving non-trivial data types into Perl's grammar.
Yes, it's pretty awful at that, alright. Sometimes I write perl programs
that need them, and sometimes it just takes a little creativity. But
sometimes it's not worth it. I actually wrote a C program the other day
(gasp) because I didn't want to deal with a game matrix with six links per node.
:Here's a very simple problem that's tricky to express in Perl: process
:the output of "du" to produce output that's indented to reflect the
:tree structure, and with each subtree sorted by size. Something like:
: 434 /etc
: | 344 .
: | 50 install
: | 35 uucp
: | 3 nserve
: | | 2 .
: | | 1 auth.info
: | 1 sm
: | 1 sm.bak
At first I thought I could just keep one local list around
at once, but this seems inherently recursive. Which means
I need an real recursive data structure. Maybe you could
do it with one of the %assoc arrays Larry uses in the begat
programs, but I broke down and got dirty. I think the hardest
part was matching Felix's desired output exactly. It's not
blazingly fast: I should probably inline the &childof routine,
but it *was* faster to write than I could have written the
equivalent C program.
--tom
--
"GUIs normally make it simple to accomplish simple actions and impossible
to accomplish complex actions." --Doug Gwyn (22/Jun/91 in comp.unix.wizards)
Tom Christiansen tchrist@convex.com convex!tchrist

60
Tools/scripts/dutree.py
View file

@ -1,60 +0,0 @@
#! /usr/bin/env python3
# Format du output in a tree shape
import os, sys, errno
def main():
total, d = None, {}
with os.popen('du ' + ' '.join(sys.argv[1:])) as p:
for line in p:
i = 0
while line[i] in '0123456789': i = i+1
size = eval(line[:i])
while line[i] in ' \t': i = i+1
filename = line[i:-1]
comps = filename.split('/')
if comps[0] == '': comps[0] = '/'
if comps[len(comps)-1] == '': del comps[len(comps)-1]
total, d = store(size, comps, total, d)
try:
display(total, d)
except IOError as e:
if e.errno != errno.EPIPE:
raise
def store(size, comps, total, d):
if comps == []:
return size, d
if comps[0] not in d:
d[comps[0]] = None, {}
t1, d1 = d[comps[0]]
d[comps[0]] = store(size, comps[1:], t1, d1)
return total, d
def display(total, d):
show(total, d, '')
def show(total, d, prefix):
if not d: return
list = []
sum = 0
for key in d.keys():
tsub, dsub = d[key]
list.append((tsub, key))
if tsub is not None: sum = sum + tsub
## if sum < total:
## list.append((total - sum, os.curdir))
list.sort()
list.reverse()
width = len(repr(list[0][0]))
for tsub, key in list:
if tsub is None:
psub = prefix
else:
print(prefix + repr(tsub).rjust(width) + ' ' + key)
psub = prefix + ' '*(width-1) + '|' + ' '*(len(key)+1)
if key in d:
show(tsub, d[key][1], psub)
if __name__ == '__main__':
main()

40
Tools/scripts/find-uname.py
View file

@ -1,40 +0,0 @@
#!/usr/bin/env python3
"""
For each argument on the command line, look for it in the set of all Unicode
names. Arguments are treated as case-insensitive regular expressions, e.g.:
% find-uname 'small letter a$' 'horizontal line'
*** small letter a$ matches ***
LATIN SMALL LETTER A (97)
COMBINING LATIN SMALL LETTER A (867)
CYRILLIC SMALL LETTER A (1072)
PARENTHESIZED LATIN SMALL LETTER A (9372)
CIRCLED LATIN SMALL LETTER A (9424)
FULLWIDTH LATIN SMALL LETTER A (65345)
*** horizontal line matches ***
HORIZONTAL LINE EXTENSION (9135)
"""
import unicodedata
import sys
import re
def main(args):
unicode_names = []
for ix in range(sys.maxunicode+1):
try:
unicode_names.append((ix, unicodedata.name(chr(ix))))
except ValueError: # no name for the character
pass
for arg in args:
pat = re.compile(arg, re.I)
matches = [(y,x) for (x,y) in unicode_names
if pat.search(y) is not None]
if matches:
print("***", arg, "matches", "***")
for match in matches:
print("%s (%d)" % match)
if __name__ == "__main__":
main(sys.argv[1:])

128
Tools/scripts/find_recursionlimit.py
View file

@ -1,128 +0,0 @@
#! /usr/bin/env python3
"""Find the maximum recursion limit that prevents interpreter termination.
This script finds the maximum safe recursion limit on a particular
platform. If you need to change the recursion limit on your system,
this script will tell you a safe upper bound. To use the new limit,
call sys.setrecursionlimit().
This module implements several ways to create infinite recursion in
Python. Different implementations end up pushing different numbers of
C stack frames, depending on how many calls through Python's abstract
C API occur.
After each round of tests, it prints a message:
"Limit of NNNN is fine".
The highest printed value of "NNNN" is therefore the highest potentially
safe limit for your system (which depends on the OS, architecture, but also
the compilation flags). Please note that it is practically impossible to
test all possible recursion paths in the interpreter, so the results of
this test should not be trusted blindly -- although they give a good hint
of which values are reasonable.
NOTE: When the C stack space allocated by your system is exceeded due
to excessive recursion, exact behaviour depends on the platform, although
the interpreter will always fail in a likely brutal way: either a
segmentation fault, a MemoryError, or just a silent abort.
NB: A program that does not use __methods__ can set a higher limit.
"""
import sys
import itertools
class RecursiveBlowup1:
def __init__(self):
self.__init__()
def test_init():
return RecursiveBlowup1()
class RecursiveBlowup2:
def __repr__(self):
return repr(self)
def test_repr():
return repr(RecursiveBlowup2())
class RecursiveBlowup4:
def __add__(self, x):
return x + self
def test_add():
return RecursiveBlowup4() + RecursiveBlowup4()
class RecursiveBlowup5:
def __getattr__(self, attr):
return getattr(self, attr)
def test_getattr():
return RecursiveBlowup5().attr
class RecursiveBlowup6:
def __getitem__(self, item):
return self[item - 2] + self[item - 1]
def test_getitem():
return RecursiveBlowup6()[5]
def test_recurse():
return test_recurse()
def test_cpickle(_cache={}):
import io
try:
import _pickle
except ImportError:
print("cannot import _pickle, skipped!")
return
k, l = None, None
for n in itertools.count():
try:
l = _cache[n]
continue # Already tried and it works, let's save some time
except KeyError:
for i in range(100):
l = [k, l]
k = {i: l}
_pickle.Pickler(io.BytesIO(), protocol=-1).dump(l)
_cache[n] = l
def test_compiler_recursion():
# The compiler uses a scaling factor to support additional levels
# of recursion. This is a sanity check of that scaling to ensure
# it still raises RecursionError even at higher recursion limits
compile("()" * (10 * sys.getrecursionlimit()), "<single>", "single")
def check_limit(n, test_func_name):
sys.setrecursionlimit(n)
if test_func_name.startswith("test_"):
print(test_func_name[5:])
else:
print(test_func_name)
test_func = globals()[test_func_name]
try:
test_func()
# AttributeError can be raised because of the way e.g. PyDict_GetItem()
# silences all exceptions and returns NULL, which is usually interpreted
# as "missing attribute".
except (RecursionError, AttributeError):
pass
else:
print("Yikes!")
if __name__ == '__main__':
limit = 1000
while 1:
check_limit(limit, "test_recurse")
check_limit(limit, "test_add")
check_limit(limit, "test_repr")
check_limit(limit, "test_init")
check_limit(limit, "test_getattr")
check_limit(limit, "test_getitem")
check_limit(limit, "test_cpickle")
check_limit(limit, "test_compiler_recursion")
print("Limit of %d is fine" % limit)
limit = limit + 100

89
Tools/scripts/finddiv.py
View file

@ -1,89 +0,0 @@
#! /usr/bin/env python3
"""finddiv - a grep-like tool that looks for division operators.
Usage: finddiv [-l] file_or_directory ...
For directory arguments, all files in the directory whose name ends in
.py are processed, and subdirectories are processed recursively.
This actually tokenizes the files to avoid false hits in comments or
strings literals.
By default, this prints all lines containing a / or /= operator, in
grep -n style. With the -l option specified, it prints the filename
of files that contain at least one / or /= operator.
"""
import os
import sys
import getopt
import tokenize
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], "lh")
except getopt.error as msg:
usage(msg)
return 2
if not args:
usage("at least one file argument is required")
return 2
listnames = 0
for o, a in opts:
if o == "-h":
print(__doc__)
return
if o == "-l":
listnames = 1
exit = None
for filename in args:
x = process(filename, listnames)
exit = exit or x
return exit
def usage(msg):
sys.stderr.write("%s: %s\n" % (sys.argv[0], msg))
sys.stderr.write("Usage: %s [-l] file ...\n" % sys.argv[0])
sys.stderr.write("Try `%s -h' for more information.\n" % sys.argv[0])
def process(filename, listnames):
if os.path.isdir(filename):
return processdir(filename, listnames)
try:
fp = open(filename)
except IOError as msg:
sys.stderr.write("Can't open: %s\n" % msg)
return 1
with fp:
g = tokenize.generate_tokens(fp.readline)
lastrow = None
for type, token, (row, col), end, line in g:
if token in ("/", "/="):
if listnames:
print(filename)
break
if row != lastrow:
lastrow = row
print("%s:%d:%s" % (filename, row, line), end=' ')
def processdir(dir, listnames):
try:
names = os.listdir(dir)
except OSError as msg:
sys.stderr.write("Can't list directory: %s\n" % dir)
return 1
files = []
for name in names:
fn = os.path.join(dir, name)
if os.path.normcase(fn).endswith(".py") or os.path.isdir(fn):
files.append(fn)
files.sort(key=os.path.normcase)
exit = None
for fn in files:
x = process(fn, listnames)
exit = exit or x
return exit
if __name__ == "__main__":
sys.exit(main())

43
Tools/scripts/findlinksto.py
View file

@ -1,43 +0,0 @@
#! /usr/bin/env python3
# findlinksto
#
# find symbolic links to a path matching a regular expression
import os
import sys
import re
import getopt
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], '')
if len(args) < 2:
raise getopt.GetoptError('not enough arguments', None)
except getopt.GetoptError as msg:
sys.stdout = sys.stderr
print(msg)
print('usage: findlinksto pattern directory ...')
sys.exit(2)
pat, dirs = args[0], args[1:]
prog = re.compile(pat)
for dirname in dirs:
os.walk(dirname, visit, prog)
def visit(prog, dirname, names):
if os.path.islink(dirname):
names[:] = []
return
if os.path.ismount(dirname):
print('descend into', dirname)
for name in names:
name = os.path.join(dirname, name)
try:
linkto = os.readlink(name)
if prog.search(linkto) is not None:
print(name, '->', linkto)
except OSError:
pass
if __name__ == '__main__':
main()

107
Tools/scripts/findnocoding.py
View file

@ -1,107 +0,0 @@
#!/usr/bin/env python3
"""List all those Python files that require a coding directive
Usage: findnocoding.py dir1 [dir2...]
"""
__author__ = "Oleg Broytmann, Georg Brandl"
import sys, os, re, getopt
# our pysource module finds Python source files
try:
import pysource
except ImportError:
# emulate the module with a simple os.walk
class pysource:
has_python_ext = looks_like_python = can_be_compiled = None
def walk_python_files(self, paths, *args, **kwargs):
for path in paths:
if os.path.isfile(path):
yield path.endswith(".py")
elif os.path.isdir(path):
for root, dirs, files in os.walk(path):
for filename in files:
if filename.endswith(".py"):
yield os.path.join(root, filename)
pysource = pysource()
print("The pysource module is not available; "
"no sophisticated Python source file search will be done.", file=sys.stderr)
decl_re = re.compile(rb'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)')
blank_re = re.compile(rb'^[ \t\f]*(?:[#\r\n]|$)')
def get_declaration(line):
match = decl_re.match(line)
if match:
return match.group(1)
return b''
def has_correct_encoding(text, codec):
try:
str(text, codec)
except UnicodeDecodeError:
return False
else:
return True
def needs_declaration(fullpath):
try:
infile = open(fullpath, 'rb')
except IOError: # Oops, the file was removed - ignore it
return None
with infile:
line1 = infile.readline()
line2 = infile.readline()
if (get_declaration(line1) or
blank_re.match(line1) and get_declaration(line2)):
# the file does have an encoding declaration, so trust it
return False
# check the whole file for non utf-8 characters
rest = infile.read()
if has_correct_encoding(line1+line2+rest, "utf-8"):
return False
return True
usage = """Usage: %s [-cd] paths...
-c: recognize Python source files trying to compile them
-d: debug output""" % sys.argv[0]
if __name__ == '__main__':
try:
opts, args = getopt.getopt(sys.argv[1:], 'cd')
except getopt.error as msg:
print(msg, file=sys.stderr)
print(usage, file=sys.stderr)
sys.exit(1)
is_python = pysource.looks_like_python
debug = False
for o, a in opts:
if o == '-c':
is_python = pysource.can_be_compiled
elif o == '-d':
debug = True
if not args:
print(usage, file=sys.stderr)
sys.exit(1)
for fullpath in pysource.walk_python_files(args, is_python):
if debug:
print("Testing for coding: %s" % fullpath)
result = needs_declaration(fullpath)
if result:
print(fullpath)

316
Tools/scripts/fixcid.py
View file

@ -1,316 +0,0 @@
#! /usr/bin/env python3
# Perform massive identifier substitution on C source files.
# This actually tokenizes the files (to some extent) so it can
# avoid making substitutions inside strings or comments.
# Inside strings, substitutions are never made; inside comments,
# it is a user option (off by default).
#
# The substitutions are read from one or more files whose lines,
# when not empty, after stripping comments starting with #,
# must contain exactly two words separated by whitespace: the
# old identifier and its replacement.
#
# The option -r reverses the sense of the substitutions (this may be
# useful to undo a particular substitution).
#
# If the old identifier is prefixed with a '*' (with no intervening
# whitespace), then it will not be substituted inside comments.
#
# Command line arguments are files or directories to be processed.
# Directories are searched recursively for files whose name looks
# like a C file (ends in .h or .c). The special filename '-' means
# operate in filter mode: read stdin, write stdout.
#
# Symbolic links are always ignored (except as explicit directory
# arguments).
#
# The original files are kept as back-up with a "~" suffix.
#
# Changes made are reported to stdout in a diff-like format.
#
# NB: by changing only the function fixline() you can turn this
# into a program for different changes to C source files; by
# changing the function wanted() you can make a different selection of
# files.
import sys
import re
import os
from stat import *
import getopt
err = sys.stderr.write
dbg = err
rep = sys.stdout.write
def usage():
progname = sys.argv[0]
err('Usage: ' + progname +
' [-c] [-r] [-s file] ... file-or-directory ...\n')
err('\n')
err('-c : substitute inside comments\n')
err('-r : reverse direction for following -s options\n')
err('-s substfile : add a file of substitutions\n')
err('\n')
err('Each non-empty non-comment line in a substitution file must\n')
err('contain exactly two words: an identifier and its replacement.\n')
err('Comments start with a # character and end at end of line.\n')
err('If an identifier is preceded with a *, it is not substituted\n')
err('inside a comment even when -c is specified.\n')
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], 'crs:')
except getopt.error as msg:
err('Options error: ' + str(msg) + '\n')
usage()
sys.exit(2)
bad = 0
if not args: # No arguments
usage()
sys.exit(2)
for opt, arg in opts:
if opt == '-c':
setdocomments()
if opt == '-r':
setreverse()
if opt == '-s':
addsubst(arg)
for arg in args:
if os.path.isdir(arg):
if recursedown(arg): bad = 1
elif os.path.islink(arg):
err(arg + ': will not process symbolic links\n')
bad = 1
else:
if fix(arg): bad = 1
sys.exit(bad)
# Change this regular expression to select a different set of files
Wanted = r'^[a-zA-Z0-9_]+\.[ch]$'
def wanted(name):
return re.match(Wanted, name)
def recursedown(dirname):
dbg('recursedown(%r)\n' % (dirname,))
bad = 0
try:
names = os.listdir(dirname)
except OSError as msg:
err(dirname + ': cannot list directory: ' + str(msg) + '\n')
return 1
names.sort()
subdirs = []
for name in names:
if name in (os.curdir, os.pardir): continue
fullname = os.path.join(dirname, name)
if os.path.islink(fullname): pass
elif os.path.isdir(fullname):
subdirs.append(fullname)
elif wanted(name):
if fix(fullname): bad = 1
for fullname in subdirs:
if recursedown(fullname): bad = 1
return bad
def fix(filename):
## dbg('fix(%r)\n' % (filename,))
if filename == '-':
# Filter mode
f = sys.stdin
g = sys.stdout
else:
# File replacement mode
try:
f = open(filename, 'r')
except IOError as msg:
err(filename + ': cannot open: ' + str(msg) + '\n')
return 1
head, tail = os.path.split(filename)
tempname = os.path.join(head, '@' + tail)
g = None
# If we find a match, we rewind the file and start over but
# now copy everything to a temp file.
lineno = 0
initfixline()
while 1:
line = f.readline()
if not line: break
lineno = lineno + 1
while line[-2:] == '\\\n':
nextline = f.readline()
if not nextline: break
line = line + nextline
lineno = lineno + 1
newline = fixline(line)
if newline != line:
if g is None:
try:
g = open(tempname, 'w')
except IOError as msg:
f.close()
err(tempname+': cannot create: '+
str(msg)+'\n')
return 1
f.seek(0)
lineno = 0
initfixline()
rep(filename + ':\n')
continue # restart from the beginning
rep(repr(lineno) + '\n')
rep('< ' + line)
rep('> ' + newline)
if g is not None:
g.write(newline)
# End of file
if filename == '-': return 0 # Done in filter mode
f.close()
if not g: return 0 # No changes
g.close()
# Finishing touch -- move files
# First copy the file's mode to the temp file
try:
statbuf = os.stat(filename)
os.chmod(tempname, statbuf[ST_MODE] & 0o7777)
except OSError as msg:
err(tempname + ': warning: chmod failed (' + str(msg) + ')\n')
# Then make a backup of the original file as filename~
try:
os.rename(filename, filename + '~')
except OSError as msg:
err(filename + ': warning: backup failed (' + str(msg) + ')\n')
# Now move the temp file to the original file
try:
os.rename(tempname, filename)
except OSError as msg:
err(filename + ': rename failed (' + str(msg) + ')\n')
return 1
# Return success
return 0
# Tokenizing ANSI C (partly)
Identifier = '(struct )?[a-zA-Z_][a-zA-Z0-9_]+'
String = r'"([^\n\\"]|\\.)*"'
Char = r"'([^\n\\']|\\.)*'"
CommentStart = r'/\*'
CommentEnd = r'\*/'
Hexnumber = '0[xX][0-9a-fA-F]*[uUlL]*'
Octnumber = '0[0-7]*[uUlL]*'
Decnumber = '[1-9][0-9]*[uUlL]*'
Intnumber = Hexnumber + '|' + Octnumber + '|' + Decnumber
Exponent = '[eE][-+]?[0-9]+'
Pointfloat = r'([0-9]+\.[0-9]*|\.[0-9]+)(' + Exponent + r')?'
Expfloat = '[0-9]+' + Exponent
Floatnumber = Pointfloat + '|' + Expfloat
Number = Floatnumber + '|' + Intnumber
# Anything else is an operator -- don't list this explicitly because of '/*'
OutsideComment = (Identifier, Number, String, Char, CommentStart)
OutsideCommentPattern = '(' + '|'.join(OutsideComment) + ')'
OutsideCommentProgram = re.compile(OutsideCommentPattern)
InsideComment = (Identifier, Number, CommentEnd)
InsideCommentPattern = '(' + '|'.join(InsideComment) + ')'
InsideCommentProgram = re.compile(InsideCommentPattern)
def initfixline():
global Program
Program = OutsideCommentProgram
def fixline(line):
global Program
## print('-->', repr(line))
i = 0
while i < len(line):
match = Program.search(line, i)
if match is None: break
i = match.start()
found = match.group(0)
## if Program is InsideCommentProgram: print(end='... ')
## else: print(end=' ')
## print(found)
if len(found) == 2:
if found == '/*':
Program = InsideCommentProgram
elif found == '*/':
Program = OutsideCommentProgram
n = len(found)
if found in Dict:
subst = Dict[found]
if Program is InsideCommentProgram:
if not Docomments:
print('Found in comment:', found)
i = i + n
continue
if found in NotInComment:
## print(end='Ignored in comment: ')
## print(found, '-->', subst)
## print('Line:', line, end='')
subst = found
## else:
## print(end='Substituting in comment: ')
## print(found, '-->', subst)
## print('Line:', line, end='')
line = line[:i] + subst + line[i+n:]
n = len(subst)
i = i + n
return line
Docomments = 0
def setdocomments():
global Docomments
Docomments = 1
Reverse = 0
def setreverse():
global Reverse
Reverse = (not Reverse)
Dict = {}
NotInComment = {}
def addsubst(substfile):
try:
fp = open(substfile, 'r')
except IOError as msg:
err(substfile + ': cannot read substfile: ' + str(msg) + '\n')
sys.exit(1)
with fp:
lineno = 0
while 1:
line = fp.readline()
if not line: break
lineno = lineno + 1
try:
i = line.index('#')
except ValueError:
i = -1 # Happens to delete trailing \n
words = line[:i].split()
if not words: continue
if len(words) == 3 and words[0] == 'struct':
words[:2] = [words[0] + ' ' + words[1]]
elif len(words) != 2:
err(substfile + '%s:%r: warning: bad line: %r' % (substfile, lineno, line))
continue
if Reverse:
[value, key] = words
else:
[key, value] = words
if value[0] == '*':
value = value[1:]
if key[0] == '*':
key = key[1:]
NotInComment[key] = value
if key in Dict:
err('%s:%r: warning: overriding: %r %r\n' % (substfile, lineno, key, value))
err('%s:%r: warning: previous: %r\n' % (substfile, lineno, Dict[key]))
Dict[key] = value
if __name__ == '__main__':
main()

378
Tools/scripts/fixdiv.py
View file

@ -1,378 +0,0 @@
#! /usr/bin/env python3
"""fixdiv - tool to fix division operators.
To use this tool, first run `python -Qwarnall yourscript.py 2>warnings'.
This runs the script `yourscript.py' while writing warning messages
about all uses of the classic division operator to the file
`warnings'. The warnings look like this:
<file>:<line>: DeprecationWarning: classic <type> division
The warnings are written to stderr, so you must use `2>' for the I/O
redirect. I know of no way to redirect stderr on Windows in a DOS
box, so you will have to modify the script to set sys.stderr to some
kind of log file if you want to do this on Windows.
The warnings are not limited to the script; modules imported by the
script may also trigger warnings. In fact a useful technique is to
write a test script specifically intended to exercise all code in a
particular module or set of modules.
Then run `python fixdiv.py warnings'. This first reads the warnings,
looking for classic division warnings, and sorts them by file name and
line number. Then, for each file that received at least one warning,
it parses the file and tries to match the warnings up to the division
operators found in the source code. If it is successful, it writes
its findings to stdout, preceded by a line of dashes and a line of the
form:
Index: <file>
If the only findings found are suggestions to change a / operator into
a // operator, the output is acceptable input for the Unix 'patch'
program.
Here are the possible messages on stdout (N stands for a line number):
- A plain-diff-style change ('NcN', a line marked by '<', a line
containing '---', and a line marked by '>'):
A / operator was found that should be changed to //. This is the
recommendation when only int and/or long arguments were seen.
- 'True division / operator at line N' and a line marked by '=':
A / operator was found that can remain unchanged. This is the
recommendation when only float and/or complex arguments were seen.
- 'Ambiguous / operator (..., ...) at line N', line marked by '?':
A / operator was found for which int or long as well as float or
complex arguments were seen. This is highly unlikely; if it occurs,
you may have to restructure the code to keep the classic semantics,
or maybe you don't care about the classic semantics.
- 'No conclusive evidence on line N', line marked by '*':
A / operator was found for which no warnings were seen. This could
be code that was never executed, or code that was only executed
with user-defined objects as arguments. You will have to
investigate further. Note that // can be overloaded separately from
/, using __floordiv__. True division can also be separately
overloaded, using __truediv__. Classic division should be the same
as either of those. (XXX should I add a warning for division on
user-defined objects, to disambiguate this case from code that was
never executed?)
- 'Phantom ... warnings for line N', line marked by '*':
A warning was seen for a line not containing a / operator. The most
likely cause is a warning about code executed by 'exec' or eval()
(see note below), or an indirect invocation of the / operator, for
example via the div() function in the operator module. It could
also be caused by a change to the file between the time the test
script was run to collect warnings and the time fixdiv was run.
- 'More than one / operator in line N'; or
'More than one / operator per statement in lines N-N':
The scanner found more than one / operator on a single line, or in a
statement split across multiple lines. Because the warnings
framework doesn't (and can't) show the offset within the line, and
the code generator doesn't always give the correct line number for
operations in a multi-line statement, we can't be sure whether all
operators in the statement were executed. To be on the safe side,
by default a warning is issued about this case. In practice, these
cases are usually safe, and the -m option suppresses these warning.
- 'Can't find the / operator in line N', line marked by '*':
This really shouldn't happen. It means that the tokenize module
reported a '/' operator but the line it returns didn't contain a '/'
character at the indicated position.
- 'Bad warning for line N: XYZ', line marked by '*':
This really shouldn't happen. It means that a 'classic XYZ
division' warning was read with XYZ being something other than
'int', 'long', 'float', or 'complex'.
Notes:
- The augmented assignment operator /= is handled the same way as the
/ operator.
- This tool never looks at the // operator; no warnings are ever
generated for use of this operator.
- This tool never looks at the / operator when a future division
statement is in effect; no warnings are generated in this case, and
because the tool only looks at files for which at least one classic
division warning was seen, it will never look at files containing a
future division statement.
- Warnings may be issued for code not read from a file, but executed
using the exec() or eval() functions. These may have
<string> in the filename position, in which case the fixdiv script
will attempt and fail to open a file named '<string>' and issue a
warning about this failure; or these may be reported as 'Phantom'
warnings (see above). You're on your own to deal with these. You
could make all recommended changes and add a future division
statement to all affected files, and then re-run the test script; it
should not issue any warnings. If there are any, and you have a
hard time tracking down where they are generated, you can use the
-Werror option to force an error instead of a first warning,
generating a traceback.
- The tool should be run from the same directory as that from which
the original script was run, otherwise it won't be able to open
files given by relative pathnames.
"""
import sys
import getopt
import re
import tokenize
multi_ok = 0
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], "hm")
except getopt.error as msg:
usage(msg)
return 2
for o, a in opts:
if o == "-h":
print(__doc__)
return
if o == "-m":
global multi_ok
multi_ok = 1
if not args:
usage("at least one file argument is required")
return 2
if args[1:]:
sys.stderr.write("%s: extra file arguments ignored\n", sys.argv[0])
warnings = readwarnings(args[0])
if warnings is None:
return 1
files = list(warnings.keys())
if not files:
print("No classic division warnings read from", args[0])
return
files.sort()
exit = None
for filename in files:
x = process(filename, warnings[filename])
exit = exit or x
return exit
def usage(msg):
sys.stderr.write("%s: %s\n" % (sys.argv[0], msg))
sys.stderr.write("Usage: %s [-m] warnings\n" % sys.argv[0])
sys.stderr.write("Try `%s -h' for more information.\n" % sys.argv[0])
PATTERN = (r"^(.+?):(\d+): DeprecationWarning: "
r"classic (int|long|float|complex) division$")
def readwarnings(warningsfile):
prog = re.compile(PATTERN)
warnings = {}
try:
f = open(warningsfile)
except IOError as msg:
sys.stderr.write("can't open: %s\n" % msg)
return
with f:
while 1:
line = f.readline()
if not line:
break
m = prog.match(line)
if not m:
if line.find("division") >= 0:
sys.stderr.write("Warning: ignored input " + line)
continue
filename, lineno, what = m.groups()
list = warnings.get(filename)
if list is None:
warnings[filename] = list = []
list.append((int(lineno), sys.intern(what)))
return warnings
def process(filename, list):
print("-"*70)
assert list # if this fails, readwarnings() is broken
try:
fp = open(filename)
except IOError as msg:
sys.stderr.write("can't open: %s\n" % msg)
return 1
with fp:
print("Index:", filename)
f = FileContext(fp)
list.sort()
index = 0 # list[:index] has been processed, list[index:] is still to do
g = tokenize.generate_tokens(f.readline)
while 1:
startlineno, endlineno, slashes = lineinfo = scanline(g)
if startlineno is None:
break
assert startlineno <= endlineno is not None
orphans = []
while index < len(list) and list[index][0] < startlineno:
orphans.append(list[index])
index += 1
if orphans:
reportphantomwarnings(orphans, f)
warnings = []
while index < len(list) and list[index][0] <= endlineno:
warnings.append(list[index])
index += 1
if not slashes and not warnings:
pass
elif slashes and not warnings:
report(slashes, "No conclusive evidence")
elif warnings and not slashes:
reportphantomwarnings(warnings, f)
else:
if len(slashes) > 1:
if not multi_ok:
rows = []
lastrow = None
for (row, col), line in slashes:
if row == lastrow:
continue
rows.append(row)
lastrow = row
assert rows
if len(rows) == 1:
print("*** More than one / operator in line", rows[0])
else:
print("*** More than one / operator per statement", end=' ')
print("in lines %d-%d" % (rows[0], rows[-1]))
intlong = []
floatcomplex = []
bad = []
for lineno, what in warnings:
if what in ("int", "long"):
intlong.append(what)
elif what in ("float", "complex"):
floatcomplex.append(what)
else:
bad.append(what)
lastrow = None
for (row, col), line in slashes:
if row == lastrow:
continue
lastrow = row
line = chop(line)
if line[col:col+1] != "/":
print("*** Can't find the / operator in line %d:" % row)
print("*", line)
continue
if bad:
print("*** Bad warning for line %d:" % row, bad)
print("*", line)
elif intlong and not floatcomplex:
print("%dc%d" % (row, row))
print("<", line)
print("---")
print(">", line[:col] + "/" + line[col:])
elif floatcomplex and not intlong:
print("True division / operator at line %d:" % row)
print("=", line)
elif intlong and floatcomplex:
print("*** Ambiguous / operator (%s, %s) at line %d:" %
("|".join(intlong), "|".join(floatcomplex), row))
print("?", line)
def reportphantomwarnings(warnings, f):
blocks = []
lastrow = None
lastblock = None
for row, what in warnings:
if row != lastrow:
lastblock = [row]
blocks.append(lastblock)
lastblock.append(what)
for block in blocks:
row = block[0]
whats = "/".join(block[1:])
print("*** Phantom %s warnings for line %d:" % (whats, row))
f.report(row, mark="*")
def report(slashes, message):
lastrow = None
for (row, col), line in slashes:
if row != lastrow:
print("*** %s on line %d:" % (message, row))
print("*", chop(line))
lastrow = row
class FileContext:
def __init__(self, fp, window=5, lineno=1):
self.fp = fp
self.window = 5
self.lineno = 1
self.eoflookahead = 0
self.lookahead = []
self.buffer = []
def fill(self):
while len(self.lookahead) < self.window and not self.eoflookahead:
line = self.fp.readline()
if not line:
self.eoflookahead = 1
break
self.lookahead.append(line)
def readline(self):
self.fill()
if not self.lookahead:
return ""
line = self.lookahead.pop(0)
self.buffer.append(line)
self.lineno += 1
return line
def __getitem__(self, index):
self.fill()
bufstart = self.lineno - len(self.buffer)
lookend = self.lineno + len(self.lookahead)
if bufstart <= index < self.lineno:
return self.buffer[index - bufstart]
if self.lineno <= index < lookend:
return self.lookahead[index - self.lineno]
raise KeyError
def report(self, first, last=None, mark="*"):
if last is None:
last = first
for i in range(first, last+1):
try:
line = self[first]
except KeyError:
line = "<missing line>"
print(mark, chop(line))
def scanline(g):
slashes = []
startlineno = None
endlineno = None
for type, token, start, end, line in g:
endlineno = end[0]
if startlineno is None:
startlineno = endlineno
if token in ("/", "/="):
slashes.append((start, line))
if type == tokenize.NEWLINE:
break
return startlineno, endlineno, slashes
def chop(line):
if line.endswith("\n"):
return line[:-1]
else:
return line
if __name__ == "__main__":
sys.exit(main())

49
Tools/scripts/fixheader.py
View file

@ -1,49 +0,0 @@
#! /usr/bin/env python3
# Add some standard cpp magic to a header file
import sys
def main():
args = sys.argv[1:]
for filename in args:
process(filename)
def process(filename):
try:
f = open(filename, 'r')
except IOError as msg:
sys.stderr.write('%s: can\'t open: %s\n' % (filename, str(msg)))
return
with f:
data = f.read()
if data[:2] != '/*':
sys.stderr.write('%s does not begin with C comment\n' % filename)
return
try:
f = open(filename, 'w')
except IOError as msg:
sys.stderr.write('%s: can\'t write: %s\n' % (filename, str(msg)))
return
with f:
sys.stderr.write('Processing %s ...\n' % filename)
magic = 'Py_'
for c in filename:
if ord(c)<=0x80 and c.isalnum():
magic = magic + c.upper()
else: magic = magic + '_'
print('#ifndef', magic, file=f)
print('#define', magic, file=f)
print('#ifdef __cplusplus', file=f)
print('extern "C" {', file=f)
print('#endif', file=f)
print(file=f)
f.write(data)
print(file=f)
print('#ifdef __cplusplus', file=f)
print('}', file=f)
print('#endif', file=f)
print('#endif /*', '!'+magic, '*/', file=f)
if __name__ == '__main__':
main()

109
Tools/scripts/fixnotice.py
View file

@ -1,109 +0,0 @@
#! /usr/bin/env python3
"""(Ostensibly) fix copyright notices in files.
Actually, this script will simply replace a block of text in a file from one
string to another. It will only do this once though, i.e. not globally
throughout the file. It writes a backup file and then does an os.rename()
dance for atomicity.
Usage: fixnotices.py [options] [filenames]
Options:
-h / --help
Print this message and exit
--oldnotice=file
Use the notice in the file as the old (to be replaced) string, instead
of the hard coded value in the script.
--newnotice=file
Use the notice in the file as the new (replacement) string, instead of
the hard coded value in the script.
--dry-run
Don't actually make the changes, but print out the list of files that
would change. When used with -v, a status will be printed for every
file.
-v / --verbose
Print a message for every file looked at, indicating whether the file
is changed or not.
"""
OLD_NOTICE = """/***********************************************************
Copyright (c) 2000, BeOpen.com.
Copyright (c) 1995-2000, Corporation for National Research Initiatives.
Copyright (c) 1990-1995, Stichting Mathematisch Centrum.
All rights reserved.
See the file "Misc/COPYRIGHT" for information on usage and
redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES.
******************************************************************/
"""
import os
import sys
import getopt
NEW_NOTICE = ""
DRYRUN = 0
VERBOSE = 0
def usage(code, msg=''):
print(__doc__ % globals())
if msg:
print(msg)
sys.exit(code)
def main():
global DRYRUN, OLD_NOTICE, NEW_NOTICE, VERBOSE
try:
opts, args = getopt.getopt(sys.argv[1:], 'hv',
['help', 'oldnotice=', 'newnotice=',
'dry-run', 'verbose'])
except getopt.error as msg:
usage(1, msg)
for opt, arg in opts:
if opt in ('-h', '--help'):
usage(0)
elif opt in ('-v', '--verbose'):
VERBOSE = 1
elif opt == '--dry-run':
DRYRUN = 1
elif opt == '--oldnotice':
with open(arg) as fp:
OLD_NOTICE = fp.read()
elif opt == '--newnotice':
with open(arg) as fp:
NEW_NOTICE = fp.read()
for arg in args:
process(arg)
def process(file):
with open(file) as f:
data = f.read()
i = data.find(OLD_NOTICE)
if i < 0:
if VERBOSE:
print('no change:', file)
return
elif DRYRUN or VERBOSE:
print(' change:', file)
if DRYRUN:
# Don't actually change the file
return
data = data[:i] + NEW_NOTICE + data[i+len(OLD_NOTICE):]
new = file + ".new"
backup = file + ".bak"
with open(new, "w") as f:
f.write(data)
os.rename(file, backup)
os.rename(new, file)
if __name__ == '__main__':
main()

31
Tools/scripts/fixps.py
View file

@ -1,31 +0,0 @@
#!/usr/bin/env python3
# Fix Python script(s) to reference the interpreter via /usr/bin/env python.
# Warning: this overwrites the file without making a backup.
import sys
import re
def main():
for filename in sys.argv[1:]:
try:
f = open(filename, 'r')
except IOError as msg:
print(filename, ': can\'t open :', msg)
continue
with f:
line = f.readline()
if not re.match('^#! */usr/local/bin/python', line):
print(filename, ': not a /usr/local/bin/python script')
continue
rest = f.read()
line = re.sub('/usr/local/bin/python',
'/usr/bin/env python', line)
print(filename, ':', repr(line))
with open(filename, "w") as f:
f.write(line)
f.write(rest)
if __name__ == '__main__':
main()

70
Tools/scripts/get-remote-certificate.py
View file

@ -1,70 +0,0 @@
#!/usr/bin/env python3
#
# fetch the certificate that the server(s) are providing in PEM form
#
# args are HOST:PORT [, HOST:PORT...]
#
# By Bill Janssen.
import re
import os
import sys
import tempfile
def fetch_server_certificate (host, port):
def subproc(cmd):
from subprocess import Popen, PIPE, STDOUT, DEVNULL
proc = Popen(cmd, stdout=PIPE, stderr=STDOUT, stdin=DEVNULL)
status = proc.wait()
output = proc.stdout.read()
return status, output
def strip_to_x509_cert(certfile_contents, outfile=None):
m = re.search(br"^([-]+BEGIN CERTIFICATE[-]+[\r]*\n"
br".*[\r]*^[-]+END CERTIFICATE[-]+)$",
certfile_contents, re.MULTILINE | re.DOTALL)
if not m:
return None
else:
tn = tempfile.mktemp()
with open(tn, "wb") as fp:
fp.write(m.group(1) + b"\n")
try:
tn2 = (outfile or tempfile.mktemp())
cmd = ['openssl', 'x509', '-in', tn, '-out', tn2]
status, output = subproc(cmd)
if status != 0:
raise RuntimeError('OpenSSL x509 failed with status %s and '
'output: %r' % (status, output))
with open(tn2, 'rb') as fp:
data = fp.read()
os.unlink(tn2)
return data
finally:
os.unlink(tn)
cmd = ['openssl', 's_client', '-connect', '%s:%s' % (host, port), '-showcerts']
status, output = subproc(cmd)
if status != 0:
raise RuntimeError('OpenSSL connect failed with status %s and '
'output: %r' % (status, output))
certtext = strip_to_x509_cert(output)
if not certtext:
raise ValueError("Invalid response received from server at %s:%s" %
(host, port))
return certtext
if __name__ == "__main__":
if len(sys.argv) < 2:
sys.stderr.write(
"Usage: %s HOSTNAME:PORTNUMBER [, HOSTNAME:PORTNUMBER...]\n" %
sys.argv[0])
sys.exit(1)
for arg in sys.argv[1:]:
host, port = arg.split(":")
sys.stdout.buffer.write(fetch_server_certificate(host, int(port)))
sys.exit(0)

25
Tools/scripts/google.py
View file

@ -1,25 +0,0 @@
#! /usr/bin/env python3
"""Script to search with Google
Usage:
python3 google.py [search terms]
"""
import sys
import urllib.parse
import webbrowser
def main(args):
def quote(arg):
if ' ' in arg:
arg = '"%s"' % arg
return urllib.parse.quote_plus(arg)
qstring = '+'.join(quote(arg) for arg in args)
url = urllib.parse.urljoin('https://www.google.com/search', '?q=' + qstring)
webbrowser.open(url)
if __name__ == '__main__':
main(sys.argv[1:])

265
Tools/scripts/highlight.py
View file

@ -1,265 +0,0 @@
#!/usr/bin/env python3
'''Add syntax highlighting to Python source code'''
__author__ = 'Raymond Hettinger'
import builtins
import functools
import html as html_module
import keyword
import re
import tokenize
#### Analyze Python Source #################################
def is_builtin(s):
'Return True if s is the name of a builtin'
return hasattr(builtins, s)
def combine_range(lines, start, end):
'Join content from a range of lines between start and end'
(srow, scol), (erow, ecol) = start, end
if srow == erow:
return lines[srow-1][scol:ecol], end
rows = [lines[srow-1][scol:]] + lines[srow: erow-1] + [lines[erow-1][:ecol]]
return ''.join(rows), end
def analyze_python(source):
'''Generate and classify chunks of Python for syntax highlighting.
Yields tuples in the form: (category, categorized_text).
'''
lines = source.splitlines(True)
lines.append('')
readline = functools.partial(next, iter(lines), '')
kind = tok_str = ''
tok_type = tokenize.COMMENT
written = (1, 0)
for tok in tokenize.generate_tokens(readline):
prev_tok_type, prev_tok_str = tok_type, tok_str
tok_type, tok_str, (srow, scol), (erow, ecol), logical_lineno = tok
kind = ''
if tok_type == tokenize.COMMENT:
kind = 'comment'
elif tok_type == tokenize.OP and tok_str[:1] not in '{}[](),.:;@':
kind = 'operator'
elif tok_type == tokenize.STRING:
kind = 'string'
if prev_tok_type == tokenize.INDENT or scol==0:
kind = 'docstring'
elif tok_type == tokenize.NAME:
if tok_str in ('def', 'class', 'import', 'from'):
kind = 'definition'
elif prev_tok_str in ('def', 'class'):
kind = 'defname'
elif keyword.iskeyword(tok_str):
kind = 'keyword'
elif is_builtin(tok_str) and prev_tok_str != '.':
kind = 'builtin'
if kind:
text, written = combine_range(lines, written, (srow, scol))
yield '', text
text, written = tok_str, (erow, ecol)
yield kind, text
line_upto_token, written = combine_range(lines, written, (erow, ecol))
yield '', line_upto_token
#### Raw Output ###########################################
def raw_highlight(classified_text):
'Straight text display of text classifications'
result = []
for kind, text in classified_text:
result.append('%15s: %r\n' % (kind or 'plain', text))
return ''.join(result)
#### ANSI Output ###########################################
default_ansi = {
'comment': ('\033[0;31m', '\033[0m'),
'string': ('\033[0;32m', '\033[0m'),
'docstring': ('\033[0;32m', '\033[0m'),
'keyword': ('\033[0;33m', '\033[0m'),
'builtin': ('\033[0;35m', '\033[0m'),
'definition': ('\033[0;33m', '\033[0m'),
'defname': ('\033[0;34m', '\033[0m'),
'operator': ('\033[0;33m', '\033[0m'),
}
def ansi_highlight(classified_text, colors=default_ansi):
'Add syntax highlighting to source code using ANSI escape sequences'
# http://en.wikipedia.org/wiki/ANSI_escape_code
result = []
for kind, text in classified_text:
opener, closer = colors.get(kind, ('', ''))
result += [opener, text, closer]
return ''.join(result)
#### HTML Output ###########################################
def html_highlight(classified_text,opener='<pre class="python">\n', closer='</pre>\n'):
'Convert classified text to an HTML fragment'
result = [opener]
for kind, text in classified_text:
if kind:
result.append('<span class="%s">' % kind)
result.append(html_module.escape(text))
if kind:
result.append('</span>')
result.append(closer)
return ''.join(result)
default_css = {
'.comment': '{color: crimson;}',
'.string': '{color: forestgreen;}',
'.docstring': '{color: forestgreen; font-style:italic;}',
'.keyword': '{color: darkorange;}',
'.builtin': '{color: purple;}',
'.definition': '{color: darkorange; font-weight:bold;}',
'.defname': '{color: blue;}',
'.operator': '{color: brown;}',
}
default_html = '''\
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
"http://www.w3.org/TR/html4/strict.dtd">
<html>
<head>
<meta http-equiv="Content-type" content="text/html;charset=UTF-8">
<title> {title} </title>
<style type="text/css">
{css}
</style>
</head>
<body>
{body}
</body>
</html>
'''
def build_html_page(classified_text, title='python',
css=default_css, html=default_html):
'Create a complete HTML page with colorized source code'
css_str = '\n'.join(['%s %s' % item for item in css.items()])
result = html_highlight(classified_text)
title = html_module.escape(title)
return html.format(title=title, css=css_str, body=result)
#### LaTeX Output ##########################################
default_latex_commands = {
'comment': r'{\color{red}#1}',
'string': r'{\color{ForestGreen}#1}',
'docstring': r'{\emph{\color{ForestGreen}#1}}',
'keyword': r'{\color{orange}#1}',
'builtin': r'{\color{purple}#1}',
'definition': r'{\color{orange}#1}',
'defname': r'{\color{blue}#1}',
'operator': r'{\color{brown}#1}',
}
default_latex_document = r'''
\documentclass{article}
\usepackage{alltt}
\usepackage{upquote}
\usepackage{color}
\usepackage[usenames,dvipsnames]{xcolor}
\usepackage[cm]{fullpage}
%(macros)s
\begin{document}
\center{\LARGE{%(title)s}}
\begin{alltt}
%(body)s
\end{alltt}
\end{document}
'''
def alltt_escape(s):
'Replace backslash and braces with their escaped equivalents'
xlat = {'{': r'\{', '}': r'\}', '\\': r'\textbackslash{}'}
return re.sub(r'[\\{}]', lambda mo: xlat[mo.group()], s)
def latex_highlight(classified_text, title = 'python',
commands = default_latex_commands,
document = default_latex_document):
'Create a complete LaTeX document with colorized source code'
macros = '\n'.join(r'\newcommand{\py%s}[1]{%s}' % c for c in commands.items())
result = []
for kind, text in classified_text:
if kind:
result.append(r'\py%s{' % kind)
result.append(alltt_escape(text))
if kind:
result.append('}')
return default_latex_document % dict(title=title, macros=macros, body=''.join(result))
if __name__ == '__main__':
import argparse
import os.path
import sys
import textwrap
import webbrowser
parser = argparse.ArgumentParser(
description = 'Add syntax highlighting to Python source code',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog = textwrap.dedent('''
examples:
# Show syntax highlighted code in the terminal window
$ ./highlight.py myfile.py
# Colorize myfile.py and display in a browser
$ ./highlight.py -b myfile.py
# Create an HTML section to embed in an existing webpage
./highlight.py -s myfile.py
# Create a complete HTML file
$ ./highlight.py -c myfile.py > myfile.html
# Create a PDF using LaTeX
$ ./highlight.py -l myfile.py | pdflatex
'''))
parser.add_argument('sourcefile', metavar = 'SOURCEFILE',
help = 'file containing Python sourcecode')
parser.add_argument('-b', '--browser', action = 'store_true',
help = 'launch a browser to show results')
parser.add_argument('-c', '--complete', action = 'store_true',
help = 'build a complete html webpage')
parser.add_argument('-l', '--latex', action = 'store_true',
help = 'build a LaTeX document')
parser.add_argument('-r', '--raw', action = 'store_true',
help = 'raw parse of categorized text')
parser.add_argument('-s', '--section', action = 'store_true',
help = 'show an HTML section rather than a complete webpage')
args = parser.parse_args()
if args.section and (args.browser or args.complete):
parser.error('The -s/--section option is incompatible with '
'the -b/--browser or -c/--complete options')
sourcefile = args.sourcefile
with open(sourcefile) as f:
source = f.read()
classified_text = analyze_python(source)
if args.raw:
encoded = raw_highlight(classified_text)
elif args.complete or args.browser:
encoded = build_html_page(classified_text, title=sourcefile)
elif args.section:
encoded = html_highlight(classified_text)
elif args.latex:
encoded = latex_highlight(classified_text, title=sourcefile)
else:
encoded = ansi_highlight(classified_text)
if args.browser:
htmlfile = os.path.splitext(os.path.basename(sourcefile))[0] + '.html'
with open(htmlfile, 'w') as f:
f.write(encoded)
webbrowser.open('file://' + os.path.abspath(htmlfile))
else:
sys.stdout.write(encoded)

111
Tools/scripts/ifdef.py
View file

@ -1,111 +0,0 @@
#! /usr/bin/env python3
# Selectively preprocess #ifdef / #ifndef statements.
# Usage:
# ifdef [-Dname] ... [-Uname] ... [file] ...
#
# This scans the file(s), looking for #ifdef and #ifndef preprocessor
# commands that test for one of the names mentioned in the -D and -U
# options. On standard output it writes a copy of the input file(s)
# minus those code sections that are suppressed by the selected
# combination of defined/undefined symbols. The #if(n)def/#else/#else
# lines themselves (if the #if(n)def tests for one of the mentioned
# names) are removed as well.
# Features: Arbitrary nesting of recognized and unrecognized
# preprocessor statements works correctly. Unrecognized #if* commands
# are left in place, so it will never remove too much, only too
# little. It does accept whitespace around the '#' character.
# Restrictions: There should be no comments or other symbols on the
# #if(n)def lines. The effect of #define/#undef commands in the input
# file or in included files is not taken into account. Tests using
# #if and the defined() pseudo function are not recognized. The #elif
# command is not recognized. Improperly nesting is not detected.
# Lines that look like preprocessor commands but which are actually
# part of comments or string literals will be mistaken for
# preprocessor commands.
import sys
import getopt
defs = []
undefs = []
def main():
opts, args = getopt.getopt(sys.argv[1:], 'D:U:')
for o, a in opts:
if o == '-D':
defs.append(a)
if o == '-U':
undefs.append(a)
if not args:
args = ['-']
for filename in args:
if filename == '-':
process(sys.stdin, sys.stdout)
else:
with open(filename) as f:
process(f, sys.stdout)
def process(fpi, fpo):
keywords = ('if', 'ifdef', 'ifndef', 'else', 'endif')
ok = 1
stack = []
while 1:
line = fpi.readline()
if not line: break
while line[-2:] == '\\\n':
nextline = fpi.readline()
if not nextline: break
line = line + nextline
tmp = line.strip()
if tmp[:1] != '#':
if ok: fpo.write(line)
continue
tmp = tmp[1:].strip()
words = tmp.split()
keyword = words[0]
if keyword not in keywords:
if ok: fpo.write(line)
continue
if keyword in ('ifdef', 'ifndef') and len(words) == 2:
if keyword == 'ifdef':
ko = 1
else:
ko = 0
word = words[1]
if word in defs:
stack.append((ok, ko, word))
if not ko: ok = 0
elif word in undefs:
stack.append((ok, not ko, word))
if ko: ok = 0
else:
stack.append((ok, -1, word))
if ok: fpo.write(line)
elif keyword == 'if':
stack.append((ok, -1, ''))
if ok: fpo.write(line)
elif keyword == 'else' and stack:
s_ok, s_ko, s_word = stack[-1]
if s_ko < 0:
if ok: fpo.write(line)
else:
s_ko = not s_ko
ok = s_ok
if not s_ko: ok = 0
stack[-1] = s_ok, s_ko, s_word
elif keyword == 'endif' and stack:
s_ok, s_ko, s_word = stack[-1]
if s_ko < 0:
if ok: fpo.write(line)
del stack[-1]
ok = s_ok
else:
sys.stderr.write('Unknown keyword %s\n' % keyword)
if stack:
sys.stderr.write('stack: %s\n' % stack)
if __name__ == '__main__':
main()

37
Tools/scripts/import_diagnostics.py
View file

@ -1,37 +0,0 @@
#!/usr/bin/env python3
"""Miscellaneous diagnostics for the import system"""
import sys
import argparse
from pprint import pprint
def _dump_state(args):
print(sys.version)
for name in args.attributes:
print("sys.{}:".format(name))
pprint(getattr(sys, name))
def _add_dump_args(cmd):
cmd.add_argument("attributes", metavar="ATTR", nargs="+",
help="sys module attribute to display")
COMMANDS = (
("dump", "Dump import state", _dump_state, _add_dump_args),
)
def _make_parser():
parser = argparse.ArgumentParser()
sub = parser.add_subparsers(title="Commands")
for name, description, implementation, add_args in COMMANDS:
cmd = sub.add_parser(name, help=description)
cmd.set_defaults(command=implementation)
add_args(cmd)
return parser
def main(args):
parser = _make_parser()
args = parser.parse_args(args)
return args.command(args)
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))

24
Tools/scripts/lfcr.py
View file

@ -1,24 +0,0 @@
#! /usr/bin/env python3
"Replace LF with CRLF in argument files. Print names of changed files."
import sys, re, os
def main():
for filename in sys.argv[1:]:
if os.path.isdir(filename):
print(filename, "Directory!")
continue
with open(filename, "rb") as f:
data = f.read()
if b'\0' in data:
print(filename, "Binary!")
continue
newdata = re.sub(b"\r?\n", b"\r\n", data)
if newdata != data:
print(filename)
with open(filename, "wb") as f:
f.write(newdata)
if __name__ == '__main__':
main()

80
Tools/scripts/linktree.py
View file

@ -1,80 +0,0 @@
#! /usr/bin/env python3
# linktree
#
# Make a copy of a directory tree with symbolic links to all files in the
# original tree.
# All symbolic links go to a special symbolic link at the top, so you
# can easily fix things if the original source tree moves.
# See also "mkreal".
#
# usage: mklinks oldtree newtree
import sys, os
LINK = '.LINK' # Name of special symlink at the top.
debug = 0
def main():
if not 3 <= len(sys.argv) <= 4:
print('usage:', sys.argv[0], 'oldtree newtree [linkto]')
return 2
oldtree, newtree = sys.argv[1], sys.argv[2]
if len(sys.argv) > 3:
link = sys.argv[3]
link_may_fail = 1
else:
link = LINK
link_may_fail = 0
if not os.path.isdir(oldtree):
print(oldtree + ': not a directory')
return 1
try:
os.mkdir(newtree, 0o777)
except OSError as msg:
print(newtree + ': cannot mkdir:', msg)
return 1
linkname = os.path.join(newtree, link)
try:
os.symlink(os.path.join(os.pardir, oldtree), linkname)
except OSError as msg:
if not link_may_fail:
print(linkname + ': cannot symlink:', msg)
return 1
else:
print(linkname + ': warning: cannot symlink:', msg)
linknames(oldtree, newtree, link)
return 0
def linknames(old, new, link):
if debug: print('linknames', (old, new, link))
try:
names = os.listdir(old)
except OSError as msg:
print(old + ': warning: cannot listdir:', msg)
return
for name in names:
if name not in (os.curdir, os.pardir):
oldname = os.path.join(old, name)
linkname = os.path.join(link, name)
newname = os.path.join(new, name)
if debug > 1: print(oldname, newname, linkname)
if os.path.isdir(oldname) and \
not os.path.islink(oldname):
try:
os.mkdir(newname, 0o777)
ok = 1
except:
print(newname + \
': warning: cannot mkdir:', msg)
ok = 0
if ok:
linkname = os.path.join(os.pardir,
linkname)
linknames(oldname, newname, linkname)
else:
os.symlink(linkname, newname)
if __name__ == '__main__':
sys.exit(main())

27
Tools/scripts/lll.py
View file

@ -1,27 +0,0 @@
#! /usr/bin/env python3
# Find symbolic links and show where they point to.
# Arguments are directories to search; default is current directory.
# No recursion.
# (This is a totally different program from "findsymlinks.py"!)
import sys, os
def lll(dirname):
for name in os.listdir(dirname):
if name not in (os.curdir, os.pardir):
full = os.path.join(dirname, name)
if os.path.islink(full):
print(name, '->', os.readlink(full))
def main(args):
if not args: args = [os.curdir]
first = 1
for arg in args:
if len(args) > 1:
if not first: print()
first = 0
print(arg + ':')
lll(arg)
if __name__ == '__main__':
main(sys.argv[1:])

246
Tools/scripts/mailerdaemon.py
View file

@ -1,246 +0,0 @@
#!/usr/bin/env python3
"""Classes to parse mailer-daemon messages."""
import calendar
import email.message
import re
import os
import sys
class Unparseable(Exception):
pass
class ErrorMessage(email.message.Message):
def __init__(self):
email.message.Message.__init__(self)
self.sub = ''
def is_warning(self):
sub = self.get('Subject')
if not sub:
return 0
sub = sub.lower()
if sub.startswith('waiting mail'):
return 1
if 'warning' in sub:
return 1
self.sub = sub
return 0
def get_errors(self):
for p in EMPARSERS:
self.rewindbody()
try:
return p(self.fp, self.sub)
except Unparseable:
pass
raise Unparseable
# List of re's or tuples of re's.
# If a re, it should contain at least a group (?P<email>...) which
# should refer to the email address. The re can also contain a group
# (?P<reason>...) which should refer to the reason (error message).
# If no reason is present, the emparse_list_reason list is used to
# find a reason.
# If a tuple, the tuple should contain 2 re's. The first re finds a
# location, the second re is repeated one or more times to find
# multiple email addresses. The second re is matched (not searched)
# where the previous match ended.
# The re's are compiled using the re module.
emparse_list_list = [
'error: (?P<reason>unresolvable): (?P<email>.+)',
('----- The following addresses had permanent fatal errors -----\n',
'(?P<email>[^ \n].*)\n( .*\n)?'),
'remote execution.*\n.*rmail (?P<email>.+)',
('The following recipients did not receive your message:\n\n',
' +(?P<email>.*)\n(The following recipients did not receive your message:\n\n)?'),
'------- Failure Reasons --------\n\n(?P<reason>.*)\n(?P<email>.*)',
'^<(?P<email>.*)>:\n(?P<reason>.*)',
'^(?P<reason>User mailbox exceeds allowed size): (?P<email>.+)',
'^5\\d{2} <(?P<email>[^\n>]+)>\\.\\.\\. (?P<reason>.+)',
'^Original-Recipient: rfc822;(?P<email>.*)',
'^did not reach the following recipient\\(s\\):\n\n(?P<email>.*) on .*\n +(?P<reason>.*)',
'^ <(?P<email>[^\n>]+)> \\.\\.\\. (?P<reason>.*)',
'^Report on your message to: (?P<email>.*)\nReason: (?P<reason>.*)',
'^Your message was not delivered to +(?P<email>.*)\n +for the following reason:\n +(?P<reason>.*)',
'^ was not +(?P<email>[^ \n].*?) *\n.*\n.*\n.*\n because:.*\n +(?P<reason>[^ \n].*?) *\n',
]
# compile the re's in the list and store them in-place.
for i in range(len(emparse_list_list)):
x = emparse_list_list[i]
if isinstance(x, str):
x = re.compile(x, re.MULTILINE)
else:
xl = []
for x in x:
xl.append(re.compile(x, re.MULTILINE))
x = tuple(xl)
del xl
emparse_list_list[i] = x
del x
del i
# list of re's used to find reasons (error messages).
# if a string, "<>" is replaced by a copy of the email address.
# The expressions are searched for in order. After the first match,
# no more expressions are searched for. So, order is important.
emparse_list_reason = [
r'^5\d{2} <>\.\.\. (?P<reason>.*)',
r'<>\.\.\. (?P<reason>.*)',
re.compile(r'^<<< 5\d{2} (?P<reason>.*)', re.MULTILINE),
re.compile('===== stderr was =====\nrmail: (?P<reason>.*)'),
re.compile('^Diagnostic-Code: (?P<reason>.*)', re.MULTILINE),
]
emparse_list_from = re.compile('^From:', re.IGNORECASE|re.MULTILINE)
def emparse_list(fp, sub):
data = fp.read()
res = emparse_list_from.search(data)
if res is None:
from_index = len(data)
else:
from_index = res.start(0)
errors = []
emails = []
reason = None
for regexp in emparse_list_list:
if isinstance(regexp, tuple):
res = regexp[0].search(data, 0, from_index)
if res is not None:
try:
reason = res.group('reason')
except IndexError:
pass
while 1:
res = regexp[1].match(data, res.end(0), from_index)
if res is None:
break
emails.append(res.group('email'))
break
else:
res = regexp.search(data, 0, from_index)
if res is not None:
emails.append(res.group('email'))
try:
reason = res.group('reason')
except IndexError:
pass
break
if not emails:
raise Unparseable
if not reason:
reason = sub
if reason[:15] == 'returned mail: ':
reason = reason[15:]
for regexp in emparse_list_reason:
if isinstance(regexp, str):
for i in range(len(emails)-1,-1,-1):
email = emails[i]
exp = re.compile(re.escape(email).join(regexp.split('<>')), re.MULTILINE)
res = exp.search(data)
if res is not None:
errors.append(' '.join((email.strip()+': '+res.group('reason')).split()))
del emails[i]
continue
res = regexp.search(data)
if res is not None:
reason = res.group('reason')
break
for email in emails:
errors.append(' '.join((email.strip()+': '+reason).split()))
return errors
EMPARSERS = [emparse_list]
def sort_numeric(a, b):
a = int(a)
b = int(b)
if a < b:
return -1
elif a > b:
return 1
else:
return 0
def parsedir(dir, modify):
os.chdir(dir)
pat = re.compile('^[0-9]*$')
errordict = {}
errorfirst = {}
errorlast = {}
nok = nwarn = nbad = 0
# find all numeric file names and sort them
files = list(filter(lambda fn, pat=pat: pat.match(fn) is not None, os.listdir('.')))
files.sort(sort_numeric)
for fn in files:
# Lets try to parse the file.
fp = open(fn)
m = email.message_from_file(fp, _class=ErrorMessage)
sender = m.getaddr('From')
print('%s\t%-40s\t'%(fn, sender[1]), end=' ')
if m.is_warning():
fp.close()
print('warning only')
nwarn = nwarn + 1
if modify:
os.rename(fn, ','+fn)
## os.unlink(fn)
continue
try:
errors = m.get_errors()
except Unparseable:
print('** Not parseable')
nbad = nbad + 1
fp.close()
continue
print(len(errors), 'errors')
# Remember them
for e in errors:
try:
mm, dd = m.getdate('date')[1:1+2]
date = '%s %02d' % (calendar.month_abbr[mm], dd)
except:
date = '??????'
if e not in errordict:
errordict[e] = 1
errorfirst[e] = '%s (%s)' % (fn, date)
else:
errordict[e] = errordict[e] + 1
errorlast[e] = '%s (%s)' % (fn, date)
fp.close()
nok = nok + 1
if modify:
os.rename(fn, ','+fn)
## os.unlink(fn)
print('--------------')
print(nok, 'files parsed,',nwarn,'files warning-only,', end=' ')
print(nbad,'files unparseable')
print('--------------')
list = []
for e in errordict.keys():
list.append((errordict[e], errorfirst[e], errorlast[e], e))
list.sort()
for num, first, last, e in list:
print('%d %s - %s\t%s' % (num, first, last, e))
def main():
modify = 0
if len(sys.argv) > 1 and sys.argv[1] == '-d':
modify = 1
del sys.argv[1]
if len(sys.argv) > 1:
for folder in sys.argv[1:]:
parsedir(folder, modify)
else:
parsedir('/ufs/jack/Mail/errorsinbox', modify)
if __name__ == '__main__' or sys.argv[0] == __name__:
main()

94
Tools/scripts/make_ctype.py
View file

@ -1,94 +0,0 @@
#!/usr/bin/env python3
"""Script that generates the ctype.h-replacement in stringobject.c."""
NAMES = ("LOWER", "UPPER", "ALPHA", "DIGIT", "XDIGIT", "ALNUM", "SPACE")
print("""
#define FLAG_LOWER 0x01
#define FLAG_UPPER 0x02
#define FLAG_ALPHA (FLAG_LOWER|FLAG_UPPER)
#define FLAG_DIGIT 0x04
#define FLAG_ALNUM (FLAG_ALPHA|FLAG_DIGIT)
#define FLAG_SPACE 0x08
#define FLAG_XDIGIT 0x10
static unsigned int ctype_table[256] = {""")
for i in range(128):
c = chr(i)
flags = []
for name in NAMES:
if name in ("ALPHA", "ALNUM"):
continue
if name == "XDIGIT":
method = lambda: c.isdigit() or c.upper() in "ABCDEF"
else:
method = getattr(c, "is" + name.lower())
if method():
flags.append("FLAG_" + name)
rc = repr(c)
if c == '\v':
rc = "'\\v'"
elif c == '\f':
rc = "'\\f'"
if not flags:
print(" 0, /* 0x%x %s */" % (i, rc))
else:
print(" %s, /* 0x%x %s */" % ("|".join(flags), i, rc))
for i in range(128, 256, 16):
print(" %s," % ", ".join(16*["0"]))
print("};")
print("")
for name in NAMES:
print("#define IS%s(c) (ctype_table[Py_CHARMASK(c)] & FLAG_%s)" %
(name, name))
print("")
for name in NAMES:
name = "is" + name.lower()
print("#undef %s" % name)
print("#define %s(c) undefined_%s(c)" % (name, name))
print("""
static unsigned char ctype_tolower[256] = {""")
for i in range(0, 256, 8):
values = []
for i in range(i, i+8):
if i < 128:
c = chr(i)
if c.isupper():
i = ord(c.lower())
values.append("0x%02x" % i)
print(" %s," % ", ".join(values))
print("};")
print("""
static unsigned char ctype_toupper[256] = {""")
for i in range(0, 256, 8):
values = []
for i in range(i, i+8):
if i < 128:
c = chr(i)
if c.islower():
i = ord(c.upper())
values.append("0x%02x" % i)
print(" %s," % ", ".join(values))
print("};")
print("""
#define TOLOWER(c) (ctype_tolower[Py_CHARMASK(c)])
#define TOUPPER(c) (ctype_toupper[Py_CHARMASK(c)])
#undef tolower
#define tolower(c) undefined_tolower(c)
#undef toupper
#define toupper(c) undefined_toupper(c)
""")

65
Tools/scripts/mkreal.py
View file

@ -1,65 +0,0 @@
#! /usr/bin/env python3
# mkreal
#
# turn a symlink to a directory into a real directory
import sys
import os
from stat import *
join = os.path.join
error = 'mkreal error'
BUFSIZE = 32*1024
def mkrealfile(name):
st = os.stat(name) # Get the mode
mode = S_IMODE(st[ST_MODE])
linkto = os.readlink(name) # Make sure again it's a symlink
with open(name, 'rb') as f_in: # This ensures it's a file
os.unlink(name)
with open(name, 'wb') as f_out:
while 1:
buf = f_in.read(BUFSIZE)
if not buf: break
f_out.write(buf)
os.chmod(name, mode)
def mkrealdir(name):
st = os.stat(name) # Get the mode
mode = S_IMODE(st[ST_MODE])
linkto = os.readlink(name)
files = os.listdir(name)
os.unlink(name)
os.mkdir(name, mode)
os.chmod(name, mode)
linkto = join(os.pardir, linkto)
#
for filename in files:
if filename not in (os.curdir, os.pardir):
os.symlink(join(linkto, filename), join(name, filename))
def main():
sys.stdout = sys.stderr
progname = os.path.basename(sys.argv[0])
if progname == '-c': progname = 'mkreal'
args = sys.argv[1:]
if not args:
print('usage:', progname, 'path ...')
sys.exit(2)
status = 0
for name in args:
if not os.path.islink(name):
print(progname+':', name+':', 'not a symlink')
status = 1
else:
if os.path.isdir(name):
mkrealdir(name)
else:
mkrealfile(name)
sys.exit(status)
if __name__ == '__main__':
main()

211
Tools/scripts/objgraph.py
View file

@ -1,211 +0,0 @@
#! /usr/bin/env python3
# objgraph
#
# Read "nm -o" input of a set of libraries or modules and print various
# interesting listings, such as:
#
# - which names are used but not defined in the set (and used where),
# - which names are defined in the set (and where),
# - which modules use which other modules,
# - which modules are used by which other modules.
#
# Usage: objgraph [-cdu] [file] ...
# -c: print callers per objectfile
# -d: print callees per objectfile
# -u: print usage of undefined symbols
# If none of -cdu is specified, all are assumed.
# Use "nm -o" to generate the input
# e.g.: nm -o /lib/libc.a | objgraph
import sys
import os
import getopt
import re
# Types of symbols.
#
definitions = 'TRGDSBAEC'
externals = 'UV'
ignore = 'Nntrgdsbavuc'
# Regular expression to parse "nm -o" output.
#
matcher = re.compile('(.*):\t?........ (.) (.*)$')
# Store "item" in "dict" under "key".
# The dictionary maps keys to lists of items.
# If there is no list for the key yet, it is created.
#
def store(dict, key, item):
if key in dict:
dict[key].append(item)
else:
dict[key] = [item]
# Return a flattened version of a list of strings: the concatenation
# of its elements with intervening spaces.
#
def flat(list):
s = ''
for item in list:
s = s + ' ' + item
return s[1:]
# Global variables mapping defined/undefined names to files and back.
#
file2undef = {}
def2file = {}
file2def = {}
undef2file = {}
# Read one input file and merge the data into the tables.
# Argument is an open file.
#
def readinput(fp):
while 1:
s = fp.readline()
if not s:
break
# If you get any output from this line,
# it is probably caused by an unexpected input line:
if matcher.search(s) < 0: s; continue # Shouldn't happen
(ra, rb), (r1a, r1b), (r2a, r2b), (r3a, r3b) = matcher.regs[:4]
fn, name, type = s[r1a:r1b], s[r3a:r3b], s[r2a:r2b]
if type in definitions:
store(def2file, name, fn)
store(file2def, fn, name)
elif type in externals:
store(file2undef, fn, name)
store(undef2file, name, fn)
elif not type in ignore:
print(fn + ':' + name + ': unknown type ' + type)
# Print all names that were undefined in some module and where they are
# defined.
#
def printcallee():
flist = sorted(file2undef.keys())
for filename in flist:
print(filename + ':')
elist = file2undef[filename]
elist.sort()
for ext in elist:
if len(ext) >= 8:
tabs = '\t'
else:
tabs = '\t\t'
if ext not in def2file:
print('\t' + ext + tabs + ' *undefined')
else:
print('\t' + ext + tabs + flat(def2file[ext]))
# Print for each module the names of the other modules that use it.
#
def printcaller():
files = sorted(file2def.keys())
for filename in files:
callers = []
for label in file2def[filename]:
if label in undef2file:
callers = callers + undef2file[label]
if callers:
callers.sort()
print(filename + ':')
lastfn = ''
for fn in callers:
if fn != lastfn:
print('\t' + fn)
lastfn = fn
else:
print(filename + ': unused')
# Print undefined names and where they are used.
#
def printundef():
undefs = {}
for filename in list(file2undef.keys()):
for ext in file2undef[filename]:
if ext not in def2file:
store(undefs, ext, filename)
elist = sorted(undefs.keys())
for ext in elist:
print(ext + ':')
flist = sorted(undefs[ext])
for filename in flist:
print('\t' + filename)
# Print warning messages about names defined in more than one file.
#
def warndups():
savestdout = sys.stdout
sys.stdout = sys.stderr
names = sorted(def2file.keys())
for name in names:
if len(def2file[name]) > 1:
print('warning:', name, 'multiply defined:', end=' ')
print(flat(def2file[name]))
sys.stdout = savestdout
# Main program
#
def main():
try:
optlist, args = getopt.getopt(sys.argv[1:], 'cdu')
except getopt.error:
sys.stdout = sys.stderr
print('Usage:', os.path.basename(sys.argv[0]), end=' ')
print('[-cdu] [file] ...')
print('-c: print callers per objectfile')
print('-d: print callees per objectfile')
print('-u: print usage of undefined symbols')
print('If none of -cdu is specified, all are assumed.')
print('Use "nm -o" to generate the input')
print('e.g.: nm -o /lib/libc.a | objgraph')
return 1
optu = optc = optd = 0
for opt, void in optlist:
if opt == '-u':
optu = 1
elif opt == '-c':
optc = 1
elif opt == '-d':
optd = 1
if optu == optc == optd == 0:
optu = optc = optd = 1
if not args:
args = ['-']
for filename in args:
if filename == '-':
readinput(sys.stdin)
else:
with open(filename) as f:
readinput(f)
#
warndups()
#
more = (optu + optc + optd > 1)
if optd:
if more:
print('---------------All callees------------------')
printcallee()
if optu:
if more:
print('---------------Undefined callees------------')
printundef()
if optc:
if more:
print('---------------All Callers------------------')
printcaller()
return 0
# Call the main program.
# Use its return value as exit status.
# Catch interrupts to avoid stack trace.
#
if __name__ == '__main__':
try:
sys.exit(main())
except KeyboardInterrupt:
sys.exit(1)

164
Tools/scripts/pdeps.py
View file

@ -1,164 +0,0 @@
#! /usr/bin/env python3
# pdeps
#
# Find dependencies between a bunch of Python modules.
#
# Usage:
# pdeps file1.py file2.py ...
#
# Output:
# Four tables separated by lines like '--- Closure ---':
# 1) Direct dependencies, listing which module imports which other modules
# 2) The inverse of (1)
# 3) Indirect dependencies, or the closure of the above
# 4) The inverse of (3)
#
# To do:
# - command line options to select output type
# - option to automatically scan the Python library for referenced modules
# - option to limit output to particular modules
import sys
import re
import os
# Main program
#
def main():
args = sys.argv[1:]
if not args:
print('usage: pdeps file.py file.py ...')
return 2
#
table = {}
for arg in args:
process(arg, table)
#
print('--- Uses ---')
printresults(table)
#
print('--- Used By ---')
inv = inverse(table)
printresults(inv)
#
print('--- Closure of Uses ---')
reach = closure(table)
printresults(reach)
#
print('--- Closure of Used By ---')
invreach = inverse(reach)
printresults(invreach)
#
return 0
# Compiled regular expressions to search for import statements
#
m_import = re.compile('^[ \t]*from[ \t]+([^ \t]+)[ \t]+')
m_from = re.compile('^[ \t]*import[ \t]+([^#]+)')
# Collect data from one file
#
def process(filename, table):
with open(filename, encoding='utf-8') as fp:
mod = os.path.basename(filename)
if mod[-3:] == '.py':
mod = mod[:-3]
table[mod] = list = []
while 1:
line = fp.readline()
if not line: break
while line[-1:] == '\\':
nextline = fp.readline()
if not nextline: break
line = line[:-1] + nextline
m_found = m_import.match(line) or m_from.match(line)
if m_found:
(a, b), (a1, b1) = m_found.regs[:2]
else: continue
words = line[a1:b1].split(',')
# print '#', line, words
for word in words:
word = word.strip()
if word not in list:
list.append(word)
# Compute closure (this is in fact totally general)
#
def closure(table):
modules = list(table.keys())
#
# Initialize reach with a copy of table
#
reach = {}
for mod in modules:
reach[mod] = table[mod][:]
#
# Iterate until no more change
#
change = 1
while change:
change = 0
for mod in modules:
for mo in reach[mod]:
if mo in modules:
for m in reach[mo]:
if m not in reach[mod]:
reach[mod].append(m)
change = 1
#
return reach
# Invert a table (this is again totally general).
# All keys of the original table are made keys of the inverse,
# so there may be empty lists in the inverse.
#
def inverse(table):
inv = {}
for key in table.keys():
if key not in inv:
inv[key] = []
for item in table[key]:
store(inv, item, key)
return inv
# Store "item" in "dict" under "key".
# The dictionary maps keys to lists of items.
# If there is no list for the key yet, it is created.
#
def store(dict, key, item):
if key in dict:
dict[key].append(item)
else:
dict[key] = [item]
# Tabulate results neatly
#
def printresults(table):
modules = sorted(table.keys())
maxlen = 0
for mod in modules: maxlen = max(maxlen, len(mod))
for mod in modules:
list = sorted(table[mod])
print(mod.ljust(maxlen), ':', end=' ')
if mod in list:
print('(*)', end=' ')
for ref in list:
print(ref, end=' ')
print()
# Call main and honor exit status
if __name__ == '__main__':
try:
sys.exit(main())
except KeyboardInterrupt:
sys.exit(1)

147
Tools/scripts/pickle2db.py
View file

@ -1,147 +0,0 @@
#!/usr/bin/env python3
"""
Synopsis: %(prog)s [-h|-b|-g|-r|-a|-d] [ picklefile ] dbfile
Read the given picklefile as a series of key/value pairs and write to a new
database. If the database already exists, any contents are deleted. The
optional flags indicate the type of the output database:
-a - open using dbm (open any supported format)
-b - open as bsddb btree file
-d - open as dbm.ndbm file
-g - open as dbm.gnu file
-h - open as bsddb hash file
-r - open as bsddb recno file
The default is hash. If a pickle file is named it is opened for read
access. If no pickle file is named, the pickle input is read from standard
input.
Note that recno databases can only contain integer keys, so you can't dump a
hash or btree database using db2pickle.py and reconstitute it to a recno
database with %(prog)s unless your keys are integers.
"""
import getopt
try:
import bsddb
except ImportError:
bsddb = None
try:
import dbm.ndbm as dbm
except ImportError:
dbm = None
try:
import dbm.gnu as gdbm
except ImportError:
gdbm = None
try:
import dbm.ndbm as anydbm
except ImportError:
anydbm = None
import sys
try:
import pickle as pickle
except ImportError:
import pickle
prog = sys.argv[0]
def usage():
sys.stderr.write(__doc__ % globals())
def main(args):
try:
opts, args = getopt.getopt(args, "hbrdag",
["hash", "btree", "recno", "dbm", "anydbm",
"gdbm"])
except getopt.error:
usage()
return 1
if len(args) == 0 or len(args) > 2:
usage()
return 1
elif len(args) == 1:
pfile = sys.stdin
dbfile = args[0]
else:
try:
pfile = open(args[0], 'rb')
except IOError:
sys.stderr.write("Unable to open %s\n" % args[0])
return 1
dbfile = args[1]
dbopen = None
for opt, arg in opts:
if opt in ("-h", "--hash"):
try:
dbopen = bsddb.hashopen
except AttributeError:
sys.stderr.write("bsddb module unavailable.\n")
return 1
elif opt in ("-b", "--btree"):
try:
dbopen = bsddb.btopen
except AttributeError:
sys.stderr.write("bsddb module unavailable.\n")
return 1
elif opt in ("-r", "--recno"):
try:
dbopen = bsddb.rnopen
except AttributeError:
sys.stderr.write("bsddb module unavailable.\n")
return 1
elif opt in ("-a", "--anydbm"):
try:
dbopen = anydbm.open
except AttributeError:
sys.stderr.write("dbm module unavailable.\n")
return 1
elif opt in ("-g", "--gdbm"):
try:
dbopen = gdbm.open
except AttributeError:
sys.stderr.write("dbm.gnu module unavailable.\n")
return 1
elif opt in ("-d", "--dbm"):
try:
dbopen = dbm.open
except AttributeError:
sys.stderr.write("dbm.ndbm module unavailable.\n")
return 1
if dbopen is None:
if bsddb is None:
sys.stderr.write("bsddb module unavailable - ")
sys.stderr.write("must specify dbtype.\n")
return 1
else:
dbopen = bsddb.hashopen
try:
db = dbopen(dbfile, 'c')
except bsddb.error:
sys.stderr.write("Unable to open %s. " % dbfile)
sys.stderr.write("Check for format or version mismatch.\n")
return 1
else:
for k in list(db.keys()):
del db[k]
while 1:
try:
(key, val) = pickle.load(pfile)
except EOFError:
break
db[key] = val
db.close()
pfile.close()
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))

506
Tools/scripts/pindent.py
View file

@ -1,506 +0,0 @@
#! /usr/bin/env python3
# This file contains a class and a main program that perform three
# related (though complimentary) formatting operations on Python
# programs. When called as "pindent -c", it takes a valid Python
# program as input and outputs a version augmented with block-closing
# comments. When called as "pindent -d", it assumes its input is a
# Python program with block-closing comments and outputs a commentless
# version. When called as "pindent -r" it assumes its input is a
# Python program with block-closing comments but with its indentation
# messed up, and outputs a properly indented version.
# A "block-closing comment" is a comment of the form '# end <keyword>'
# where <keyword> is the keyword that opened the block. If the
# opening keyword is 'def' or 'class', the function or class name may
# be repeated in the block-closing comment as well. Here is an
# example of a program fully augmented with block-closing comments:
# def foobar(a, b):
# if a == b:
# a = a+1
# elif a < b:
# b = b-1
# if b > a: a = a-1
# # end if
# else:
# print 'oops!'
# # end if
# # end def foobar
# Note that only the last part of an if...elif...else... block needs a
# block-closing comment; the same is true for other compound
# statements (e.g. try...except). Also note that "short-form" blocks
# like the second 'if' in the example must be closed as well;
# otherwise the 'else' in the example would be ambiguous (remember
# that indentation is not significant when interpreting block-closing
# comments).
# The operations are idempotent (i.e. applied to their own output
# they yield an identical result). Running first "pindent -c" and
# then "pindent -r" on a valid Python program produces a program that
# is semantically identical to the input (though its indentation may
# be different). Running "pindent -e" on that output produces a
# program that only differs from the original in indentation.
# Other options:
# -s stepsize: set the indentation step size (default 8)
# -t tabsize : set the number of spaces a tab character is worth (default 8)
# -e : expand TABs into spaces
# file ... : input file(s) (default standard input)
# The results always go to standard output
# Caveats:
# - comments ending in a backslash will be mistaken for continued lines
# - continuations using backslash are always left unchanged
# - continuations inside parentheses are not extra indented by -r
# but must be indented for -c to work correctly (this breaks
# idempotency!)
# - continued lines inside triple-quoted strings are totally garbled
# Secret feature:
# - On input, a block may also be closed with an "end statement" --
# this is a block-closing comment without the '#' sign.
# Possible improvements:
# - check syntax based on transitions in 'next' table
# - better error reporting
# - better error recovery
# - check identifier after class/def
# The following wishes need a more complete tokenization of the source:
# - Don't get fooled by comments ending in backslash
# - reindent continuation lines indicated by backslash
# - handle continuation lines inside parentheses/braces/brackets
# - handle triple quoted strings spanning lines
# - realign comments
# - optionally do much more thorough reformatting, a la C indent
# Defaults
STEPSIZE = 8
TABSIZE = 8
EXPANDTABS = False
import io
import re
import sys
next = {}
next['if'] = next['elif'] = 'elif', 'else', 'end'
next['while'] = next['for'] = 'else', 'end'
next['try'] = 'except', 'finally'
next['except'] = 'except', 'else', 'finally', 'end'
next['else'] = next['finally'] = next['with'] = \
next['def'] = next['class'] = 'end'
next['end'] = ()
start = 'if', 'while', 'for', 'try', 'with', 'def', 'class'
class PythonIndenter:
def __init__(self, fpi = sys.stdin, fpo = sys.stdout,
indentsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
self.fpi = fpi
self.fpo = fpo
self.indentsize = indentsize
self.tabsize = tabsize
self.lineno = 0
self.expandtabs = expandtabs
self._write = fpo.write
self.kwprog = re.compile(
r'^(?:\s|\\\n)*(?P<kw>[a-z]+)'
r'((?:\s|\\\n)+(?P<id>[a-zA-Z_]\w*))?'
r'[^\w]')
self.endprog = re.compile(
r'^(?:\s|\\\n)*#?\s*end\s+(?P<kw>[a-z]+)'
r'(\s+(?P<id>[a-zA-Z_]\w*))?'
r'[^\w]')
self.wsprog = re.compile(r'^[ \t]*')
# end def __init__
def write(self, line):
if self.expandtabs:
self._write(line.expandtabs(self.tabsize))
else:
self._write(line)
# end if
# end def write
def readline(self):
line = self.fpi.readline()
if line: self.lineno += 1
# end if
return line
# end def readline
def error(self, fmt, *args):
if args: fmt = fmt % args
# end if
sys.stderr.write('Error at line %d: %s\n' % (self.lineno, fmt))
self.write('### %s ###\n' % fmt)
# end def error
def getline(self):
line = self.readline()
while line[-2:] == '\\\n':
line2 = self.readline()
if not line2: break
# end if
line += line2
# end while
return line
# end def getline
def putline(self, line, indent):
tabs, spaces = divmod(indent*self.indentsize, self.tabsize)
i = self.wsprog.match(line).end()
line = line[i:]
if line[:1] not in ('\n', '\r', ''):
line = '\t'*tabs + ' '*spaces + line
# end if
self.write(line)
# end def putline
def reformat(self):
stack = []
while True:
line = self.getline()
if not line: break # EOF
# end if
m = self.endprog.match(line)
if m:
kw = 'end'
kw2 = m.group('kw')
if not stack:
self.error('unexpected end')
elif stack.pop()[0] != kw2:
self.error('unmatched end')
# end if
self.putline(line, len(stack))
continue
# end if
m = self.kwprog.match(line)
if m:
kw = m.group('kw')
if kw in start:
self.putline(line, len(stack))
stack.append((kw, kw))
continue
# end if
if kw in next and stack:
self.putline(line, len(stack)-1)
kwa, kwb = stack[-1]
stack[-1] = kwa, kw
continue
# end if
# end if
self.putline(line, len(stack))
# end while
if stack:
self.error('unterminated keywords')
for kwa, kwb in stack:
self.write('\t%s\n' % kwa)
# end for
# end if
# end def reformat
def delete(self):
begin_counter = 0
end_counter = 0
while True:
line = self.getline()
if not line: break # EOF
# end if
m = self.endprog.match(line)
if m:
end_counter += 1
continue
# end if
m = self.kwprog.match(line)
if m:
kw = m.group('kw')
if kw in start:
begin_counter += 1
# end if
# end if
self.write(line)
# end while
if begin_counter - end_counter < 0:
sys.stderr.write('Warning: input contained more end tags than expected\n')
elif begin_counter - end_counter > 0:
sys.stderr.write('Warning: input contained less end tags than expected\n')
# end if
# end def delete
def complete(self):
stack = []
todo = []
currentws = thisid = firstkw = lastkw = topid = ''
while True:
line = self.getline()
i = self.wsprog.match(line).end()
m = self.endprog.match(line)
if m:
thiskw = 'end'
endkw = m.group('kw')
thisid = m.group('id')
else:
m = self.kwprog.match(line)
if m:
thiskw = m.group('kw')
if thiskw not in next:
thiskw = ''
# end if
if thiskw in ('def', 'class'):
thisid = m.group('id')
else:
thisid = ''
# end if
elif line[i:i+1] in ('\n', '#'):
todo.append(line)
continue
else:
thiskw = ''
# end if
# end if
indentws = line[:i]
indent = len(indentws.expandtabs(self.tabsize))
current = len(currentws.expandtabs(self.tabsize))
while indent < current:
if firstkw:
if topid:
s = '# end %s %s\n' % (
firstkw, topid)
else:
s = '# end %s\n' % firstkw
# end if
self.write(currentws + s)
firstkw = lastkw = ''
# end if
currentws, firstkw, lastkw, topid = stack.pop()
current = len(currentws.expandtabs(self.tabsize))
# end while
if indent == current and firstkw:
if thiskw == 'end':
if endkw != firstkw:
self.error('mismatched end')
# end if
firstkw = lastkw = ''
elif not thiskw or thiskw in start:
if topid:
s = '# end %s %s\n' % (
firstkw, topid)
else:
s = '# end %s\n' % firstkw
# end if
self.write(currentws + s)
firstkw = lastkw = topid = ''
# end if
# end if
if indent > current:
stack.append((currentws, firstkw, lastkw, topid))
if thiskw and thiskw not in start:
# error
thiskw = ''
# end if
currentws, firstkw, lastkw, topid = \
indentws, thiskw, thiskw, thisid
# end if
if thiskw:
if thiskw in start:
firstkw = lastkw = thiskw
topid = thisid
else:
lastkw = thiskw
# end if
# end if
for l in todo: self.write(l)
# end for
todo = []
if not line: break
# end if
self.write(line)
# end while
# end def complete
# end class PythonIndenter
# Simplified user interface
# - xxx_filter(input, output): read and write file objects
# - xxx_string(s): take and return string object
# - xxx_file(filename): process file in place, return true iff changed
def complete_filter(input = sys.stdin, output = sys.stdout,
stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
pi.complete()
# end def complete_filter
def delete_filter(input= sys.stdin, output = sys.stdout,
stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
pi.delete()
# end def delete_filter
def reformat_filter(input = sys.stdin, output = sys.stdout,
stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
pi.reformat()
# end def reformat_filter
def complete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
input = io.StringIO(source)
output = io.StringIO()
pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
pi.complete()
return output.getvalue()
# end def complete_string
def delete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
input = io.StringIO(source)
output = io.StringIO()
pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
pi.delete()
return output.getvalue()
# end def delete_string
def reformat_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
input = io.StringIO(source)
output = io.StringIO()
pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
pi.reformat()
return output.getvalue()
# end def reformat_string
def make_backup(filename):
import os, os.path
backup = filename + '~'
if os.path.lexists(backup):
try:
os.remove(backup)
except OSError:
print("Can't remove backup %r" % (backup,), file=sys.stderr)
# end try
# end if
try:
os.rename(filename, backup)
except OSError:
print("Can't rename %r to %r" % (filename, backup), file=sys.stderr)
# end try
# end def make_backup
def complete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
with open(filename, 'r') as f:
source = f.read()
# end with
result = complete_string(source, stepsize, tabsize, expandtabs)
if source == result: return 0
# end if
make_backup(filename)
with open(filename, 'w') as f:
f.write(result)
# end with
return 1
# end def complete_file
def delete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
with open(filename, 'r') as f:
source = f.read()
# end with
result = delete_string(source, stepsize, tabsize, expandtabs)
if source == result: return 0
# end if
make_backup(filename)
with open(filename, 'w') as f:
f.write(result)
# end with
return 1
# end def delete_file
def reformat_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
with open(filename, 'r') as f:
source = f.read()
# end with
result = reformat_string(source, stepsize, tabsize, expandtabs)
if source == result: return 0
# end if
make_backup(filename)
with open(filename, 'w') as f:
f.write(result)
# end with
return 1
# end def reformat_file
# Test program when called as a script
usage = """
usage: pindent (-c|-d|-r) [-s stepsize] [-t tabsize] [-e] [file] ...
-c : complete a correctly indented program (add #end directives)
-d : delete #end directives
-r : reformat a completed program (use #end directives)
-s stepsize: indentation step (default %(STEPSIZE)d)
-t tabsize : the worth in spaces of a tab (default %(TABSIZE)d)
-e : expand TABs into spaces (default OFF)
[file] ... : files are changed in place, with backups in file~
If no files are specified or a single - is given,
the program acts as a filter (reads stdin, writes stdout).
""" % vars()
def error_both(op1, op2):
sys.stderr.write('Error: You can not specify both '+op1+' and -'+op2[0]+' at the same time\n')
sys.stderr.write(usage)
sys.exit(2)
# end def error_both
def test():
import getopt
try:
opts, args = getopt.getopt(sys.argv[1:], 'cdrs:t:e')
except getopt.error as msg:
sys.stderr.write('Error: %s\n' % msg)
sys.stderr.write(usage)
sys.exit(2)
# end try
action = None
stepsize = STEPSIZE
tabsize = TABSIZE
expandtabs = EXPANDTABS
for o, a in opts:
if o == '-c':
if action: error_both(o, action)
# end if
action = 'complete'
elif o == '-d':
if action: error_both(o, action)
# end if
action = 'delete'
elif o == '-r':
if action: error_both(o, action)
# end if
action = 'reformat'
elif o == '-s':
stepsize = int(a)
elif o == '-t':
tabsize = int(a)
elif o == '-e':
expandtabs = True
# end if
# end for
if not action:
sys.stderr.write(
'You must specify -c(omplete), -d(elete) or -r(eformat)\n')
sys.stderr.write(usage)
sys.exit(2)
# end if
if not args or args == ['-']:
action = eval(action + '_filter')
action(sys.stdin, sys.stdout, stepsize, tabsize, expandtabs)
else:
action = eval(action + '_file')
for filename in args:
action(filename, stepsize, tabsize, expandtabs)
# end for
# end if
# end def test
if __name__ == '__main__':
test()
# end if

130
Tools/scripts/pysource.py
View file

@ -1,130 +0,0 @@
#!/usr/bin/env python3
"""\
List python source files.
There are three functions to check whether a file is a Python source, listed
here with increasing complexity:
- has_python_ext() checks whether a file name ends in '.py[w]'.
- look_like_python() checks whether the file is not binary and either has
the '.py[w]' extension or the first line contains the word 'python'.
- can_be_compiled() checks whether the file can be compiled by compile().
The file also must be of appropriate size - not bigger than a megabyte.
walk_python_files() recursively lists all Python files under the given directories.
"""
__author__ = "Oleg Broytmann, Georg Brandl"
__all__ = ["has_python_ext", "looks_like_python", "can_be_compiled", "walk_python_files"]
import os, re
binary_re = re.compile(br'[\x00-\x08\x0E-\x1F\x7F]')
debug = False
def print_debug(msg):
if debug: print(msg)
def _open(fullpath):
try:
size = os.stat(fullpath).st_size
except OSError as err: # Permission denied - ignore the file
print_debug("%s: permission denied: %s" % (fullpath, err))
return None
if size > 1024*1024: # too big
print_debug("%s: the file is too big: %d bytes" % (fullpath, size))
return None
try:
return open(fullpath, "rb")
except IOError as err: # Access denied, or a special file - ignore it
print_debug("%s: access denied: %s" % (fullpath, err))
return None
def has_python_ext(fullpath):
return fullpath.endswith(".py") or fullpath.endswith(".pyw")
def looks_like_python(fullpath):
infile = _open(fullpath)
if infile is None:
return False
with infile:
line = infile.readline()
if binary_re.search(line):
# file appears to be binary
print_debug("%s: appears to be binary" % fullpath)
return False
if fullpath.endswith(".py") or fullpath.endswith(".pyw"):
return True
elif b"python" in line:
# disguised Python script (e.g. CGI)
return True
return False
def can_be_compiled(fullpath):
infile = _open(fullpath)
if infile is None:
return False
with infile:
code = infile.read()
try:
compile(code, fullpath, "exec")
except Exception as err:
print_debug("%s: cannot compile: %s" % (fullpath, err))
return False
return True
def walk_python_files(paths, is_python=looks_like_python, exclude_dirs=None):
"""\
Recursively yield all Python source files below the given paths.
paths: a list of files and/or directories to be checked.
is_python: a function that takes a file name and checks whether it is a
Python source file
exclude_dirs: a list of directory base names that should be excluded in
the search
"""
if exclude_dirs is None:
exclude_dirs=[]
for path in paths:
print_debug("testing: %s" % path)
if os.path.isfile(path):
if is_python(path):
yield path
elif os.path.isdir(path):
print_debug(" it is a directory")
for dirpath, dirnames, filenames in os.walk(path):
for exclude in exclude_dirs:
if exclude in dirnames:
dirnames.remove(exclude)
for filename in filenames:
fullpath = os.path.join(dirpath, filename)
print_debug("testing: %s" % fullpath)
if is_python(fullpath):
yield fullpath
else:
print_debug(" unknown type")
if __name__ == "__main__":
# Two simple examples/tests
for fullpath in walk_python_files(['.']):
print(fullpath)
print("----------")
for fullpath in walk_python_files(['.'], is_python=can_be_compiled):
print(fullpath)

14
Tools/scripts/reindent-rst.py
View file

@ -1,14 +0,0 @@
#!/usr/bin/env python3
# Make a reST file compliant to our pre-commit hook.
# Currently just remove trailing whitespace.
import sys
import patchcheck
def main(argv=sys.argv):
patchcheck.normalize_docs_whitespace(argv[1:])
if __name__ == '__main__':
sys.exit(main())

67
Tools/scripts/rgrep.py
View file

@ -1,67 +0,0 @@
#! /usr/bin/env python3
"""Reverse grep.
Usage: rgrep [-i] pattern file
"""
import sys
import re
import getopt
def main():
bufsize = 64 * 1024
reflags = 0
opts, args = getopt.getopt(sys.argv[1:], "i")
for o, a in opts:
if o == '-i':
reflags = reflags | re.IGNORECASE
if len(args) < 2:
usage("not enough arguments")
if len(args) > 2:
usage("exactly one file argument required")
pattern, filename = args
try:
prog = re.compile(pattern, reflags)
except re.error as msg:
usage("error in regular expression: %s" % msg)
try:
f = open(filename)
except IOError as msg:
usage("can't open %r: %s" % (filename, msg), 1)
with f:
f.seek(0, 2)
pos = f.tell()
leftover = None
while pos > 0:
size = min(pos, bufsize)
pos = pos - size
f.seek(pos)
buffer = f.read(size)
lines = buffer.split("\n")
del buffer
if leftover is None:
if not lines[-1]:
del lines[-1]
else:
lines[-1] = lines[-1] + leftover
if pos > 0:
leftover = lines[0]
del lines[0]
else:
leftover = None
for line in reversed(lines):
if prog.search(line):
print(line)
def usage(msg, code=2):
sys.stdout = sys.stderr
print(msg)
print(__doc__)
sys.exit(code)
if __name__ == '__main__':
main()

26
Tools/scripts/suff.py
View file

@ -1,26 +0,0 @@
#! /usr/bin/env python3
# suff
#
# show different suffixes amongst arguments
import sys
def main():
files = sys.argv[1:]
suffixes = {}
for filename in files:
suff = getsuffix(filename)
suffixes.setdefault(suff, []).append(filename)
for suff, filenames in sorted(suffixes.items()):
print(repr(suff), len(filenames))
def getsuffix(filename):
name, sep, suff = filename.rpartition('.')
return sep + suff if sep else ''
if __name__ == '__main__':
main()

2071
Tools/scripts/texi2html.py
View file

File diff suppressed because it is too large Load diff

61
Tools/scripts/which.py
View file

@ -1,61 +0,0 @@
#! /usr/bin/env python3
# Variant of "which".
# On stderr, near and total misses are reported.
# '-l<flags>' argument adds ls -l<flags> of each file found.
import sys
if sys.path[0] in (".", ""): del sys.path[0]
import sys, os
from stat import *
def msg(str):
sys.stderr.write(str + '\n')
def main():
pathlist = os.environ['PATH'].split(os.pathsep)
sts = 0
longlist = ''
if sys.argv[1:] and sys.argv[1][:2] == '-l':
longlist = sys.argv[1]
del sys.argv[1]
for prog in sys.argv[1:]:
ident = ()
for dir in pathlist:
filename = os.path.join(dir, prog)
try:
st = os.stat(filename)
except OSError:
continue
if not S_ISREG(st[ST_MODE]):
msg(filename + ': not a disk file')
else:
mode = S_IMODE(st[ST_MODE])
if mode & 0o111:
if not ident:
print(filename)
ident = st[:3]
else:
if st[:3] == ident:
s = 'same as: '
else:
s = 'also: '
msg(s + filename)
else:
msg(filename + ': not executable')
if longlist:
sts = os.system('ls ' + longlist + ' ' + filename)
sts = os.waitstatus_to_exitcode(sts)
if sts: msg('"ls -l" exit status: ' + repr(sts))
if not ident:
msg(prog + ': not found')
sts = 1
sys.exit(sts)
if __name__ == '__main__':
main()