mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-20 07:31:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

String method conversion.

Browse source
This commit is contained in:
Eric S. Raymond 2001-02-09 08:40:40 +00:00
parent 630e69cd89
commit 141971f22a
1 changed files with 8 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

16
Lib/robotparser.py
View file

@ -9,7 +9,7 @@
The robots.txt Exclusion Protocol is implemented as specified in The robots.txt Exclusion Protocol is implemented as specified in
http://info.webcrawler.com/mak/projects/robots/norobots-rfc.html http://info.webcrawler.com/mak/projects/robots/norobots-rfc.html
""" """
import re,string,urlparse,urllib import re,urlparse,urllib
__all__ = ["RobotFileParser"] __all__ = ["RobotFileParser"]
@ -71,7 +71,7 @@ class RobotFileParser:
entry = Entry() entry = Entry()
for line in lines: for line in lines:
line = string.strip(line) line = line.strip()
linenumber = linenumber + 1 linenumber = linenumber + 1
if not line: if not line:
if state==1: if state==1:
@ -85,16 +85,16 @@ class RobotFileParser:
entry = Entry() entry = Entry()
state = 0 state = 0
# remove optional comment and strip line # remove optional comment and strip line
i = string.find(line, '#') i = line.find('#')
if i>=0: if i>=0:
line = line[:i] line = line[:i]
line = string.strip(line) line = line.strip()
if not line: if not line:
continue continue
line = string.split(line, ':', 1) line = line.split(':', 1)
if len(line) == 2: if len(line) == 2:
line[0] = string.lower(string.strip(line[0])) line[0] = line[0].strip().lower()
line[1] = string.strip(line[1]) line[1] = line[1].strip()
if line[0] == "user-agent": if line[0] == "user-agent":
if state==2: if state==2:
_debug("line %d: warning: you should insert a blank" _debug("line %d: warning: you should insert a blank"
@ -136,7 +136,7 @@ class RobotFileParser:
return 1 return 1
# search for given user agent matches # search for given user agent matches
# the first match counts # the first match counts
useragent = string.lower(useragent) useragent = useragent.lower()
url = urllib.quote(urlparse.urlparse(url)[2]) url = urllib.quote(urlparse.urlparse(url)[2])
for entry in self.entries: for entry in self.entries:
if entry.applies_to(useragent): if entry.applies_to(useragent):