mirror of
https://github.com/python/cpython.git
synced 2025-08-31 05:58:33 +00:00
Whitespace normalization.
This commit is contained in:
parent
384fd106e8
commit
dfc538acae
3 changed files with 12 additions and 13 deletions
|
@ -49,7 +49,7 @@ default is 1, i.e. aligned.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
__all__ = ["Chunk"]
|
__all__ = ["Chunk"]
|
||||||
|
|
||||||
class Chunk:
|
class Chunk:
|
||||||
def __init__(self, file, align = 1, bigendian = 1, inclheader = 0):
|
def __init__(self, file, align = 1, bigendian = 1, inclheader = 0):
|
||||||
import struct
|
import struct
|
||||||
|
|
|
@ -59,25 +59,25 @@ class RobotFileParser:
|
||||||
elif status>=400:
|
elif status>=400:
|
||||||
self.allow_all = 1
|
self.allow_all = 1
|
||||||
else:
|
else:
|
||||||
# status < 400
|
# status < 400
|
||||||
self.parse(connection.getfile().readlines())
|
self.parse(connection.getfile().readlines())
|
||||||
|
|
||||||
def parse(self, lines):
|
def parse(self, lines):
|
||||||
"""parse the input lines from a robot.txt file.
|
"""parse the input lines from a robot.txt file.
|
||||||
We allow that a user-agent: line is not preceded by
|
We allow that a user-agent: line is not preceded by
|
||||||
one or more blank lines."""
|
one or more blank lines."""
|
||||||
state = 0
|
state = 0
|
||||||
linenumber = 0
|
linenumber = 0
|
||||||
entry = Entry()
|
entry = Entry()
|
||||||
|
|
||||||
for line in lines:
|
for line in lines:
|
||||||
line = string.strip(line)
|
line = string.strip(line)
|
||||||
linenumber = linenumber + 1
|
linenumber = linenumber + 1
|
||||||
if not line:
|
if not line:
|
||||||
if state==1:
|
if state==1:
|
||||||
_debug("line %d: warning: you should insert"
|
_debug("line %d: warning: you should insert"
|
||||||
" allow: or disallow: directives below any"
|
" allow: or disallow: directives below any"
|
||||||
" user-agent: line" % linenumber)
|
" user-agent: line" % linenumber)
|
||||||
entry = Entry()
|
entry = Entry()
|
||||||
state = 0
|
state = 0
|
||||||
elif state==2:
|
elif state==2:
|
||||||
|
@ -98,7 +98,7 @@ class RobotFileParser:
|
||||||
if line[0] == "user-agent":
|
if line[0] == "user-agent":
|
||||||
if state==2:
|
if state==2:
|
||||||
_debug("line %d: warning: you should insert a blank"
|
_debug("line %d: warning: you should insert a blank"
|
||||||
" line before any user-agent"
|
" line before any user-agent"
|
||||||
" directive" % linenumber)
|
" directive" % linenumber)
|
||||||
self.entries.append(entry)
|
self.entries.append(entry)
|
||||||
entry = Entry()
|
entry = Entry()
|
||||||
|
@ -107,14 +107,14 @@ class RobotFileParser:
|
||||||
elif line[0] == "disallow":
|
elif line[0] == "disallow":
|
||||||
if state==0:
|
if state==0:
|
||||||
_debug("line %d: error: you must insert a user-agent:"
|
_debug("line %d: error: you must insert a user-agent:"
|
||||||
" directive before this line" % linenumber)
|
" directive before this line" % linenumber)
|
||||||
else:
|
else:
|
||||||
entry.rulelines.append(RuleLine(line[1], 0))
|
entry.rulelines.append(RuleLine(line[1], 0))
|
||||||
state = 2
|
state = 2
|
||||||
elif line[0] == "allow":
|
elif line[0] == "allow":
|
||||||
if state==0:
|
if state==0:
|
||||||
_debug("line %d: error: you must insert a user-agent:"
|
_debug("line %d: error: you must insert a user-agent:"
|
||||||
" directive before this line" % linenumber)
|
" directive before this line" % linenumber)
|
||||||
else:
|
else:
|
||||||
entry.rulelines.append(RuleLine(line[1], 1))
|
entry.rulelines.append(RuleLine(line[1], 1))
|
||||||
else:
|
else:
|
||||||
|
@ -212,7 +212,7 @@ def _test():
|
||||||
print rp.can_fetch('*', 'http://www.musi-cal.com/')
|
print rp.can_fetch('*', 'http://www.musi-cal.com/')
|
||||||
print rp.can_fetch('Musi-Cal-Robot/1.0',
|
print rp.can_fetch('Musi-Cal-Robot/1.0',
|
||||||
'http://www.musi-cal.com/cgi-bin/event-search'
|
'http://www.musi-cal.com/cgi-bin/event-search'
|
||||||
'?city=San+Francisco')
|
'?city=San+Francisco')
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
_test()
|
_test()
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
|
|
||||||
from test_support import verify, verbose, TestFailed
|
from test_support import verify, verbose, TestFailed
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
@ -8,7 +7,7 @@ def check_all(_modname):
|
||||||
"%s has no __all__ attribute" % _modname)
|
"%s has no __all__ attribute" % _modname)
|
||||||
exec "del %s" % _modname
|
exec "del %s" % _modname
|
||||||
exec "from %s import *" % _modname
|
exec "from %s import *" % _modname
|
||||||
|
|
||||||
_keys = locals().keys()
|
_keys = locals().keys()
|
||||||
_keys.remove("_modname")
|
_keys.remove("_modname")
|
||||||
_keys.sort()
|
_keys.sort()
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue