mirror of https://github.com/python/cpython
286 lines
7.9 KiB
Python
286 lines
7.9 KiB
Python
import io
|
|
import unittest
|
|
import urllib.robotparser
|
|
from collections import namedtuple
|
|
from test import support
|
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
try:
|
|
import threading
|
|
except ImportError:
|
|
threading = None
|
|
|
|
|
|
class BaseRobotTest:
|
|
robots_txt = ''
|
|
agent = 'test_robotparser'
|
|
good = []
|
|
bad = []
|
|
|
|
def setUp(self):
|
|
lines = io.StringIO(self.robots_txt).readlines()
|
|
self.parser = urllib.robotparser.RobotFileParser()
|
|
self.parser.parse(lines)
|
|
|
|
def get_agent_and_url(self, url):
|
|
if isinstance(url, tuple):
|
|
agent, url = url
|
|
return agent, url
|
|
return self.agent, url
|
|
|
|
def test_good_urls(self):
|
|
for url in self.good:
|
|
agent, url = self.get_agent_and_url(url)
|
|
with self.subTest(url=url, agent=agent):
|
|
self.assertTrue(self.parser.can_fetch(agent, url))
|
|
|
|
def test_bad_urls(self):
|
|
for url in self.bad:
|
|
agent, url = self.get_agent_and_url(url)
|
|
with self.subTest(url=url, agent=agent):
|
|
self.assertFalse(self.parser.can_fetch(agent, url))
|
|
|
|
|
|
class UserAgentWildcardTest(BaseRobotTest, unittest.TestCase):
|
|
robots_txt = """\
|
|
User-agent: *
|
|
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
|
|
Disallow: /tmp/ # these will soon disappear
|
|
Disallow: /foo.html
|
|
"""
|
|
good = ['/', '/test.html']
|
|
bad = ['/cyberworld/map/index.html', '/tmp/xxx', '/foo.html']
|
|
|
|
|
|
class CrawlDelayAndCustomAgentTest(BaseRobotTest, unittest.TestCase):
|
|
robots_txt = """\
|
|
# robots.txt for http://www.example.com/
|
|
|
|
User-agent: *
|
|
Crawl-delay: 1
|
|
Request-rate: 3/15
|
|
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
|
|
|
|
# Cybermapper knows where to go.
|
|
User-agent: cybermapper
|
|
Disallow:
|
|
"""
|
|
good = ['/', '/test.html', ('cybermapper', '/cyberworld/map/index.html')]
|
|
bad = ['/cyberworld/map/index.html']
|
|
|
|
|
|
class RejectAllRobotsTest(BaseRobotTest, unittest.TestCase):
|
|
robots_txt = """\
|
|
# go away
|
|
User-agent: *
|
|
Disallow: /
|
|
"""
|
|
good = []
|
|
bad = ['/cyberworld/map/index.html', '/', '/tmp/']
|
|
|
|
|
|
class CrawlDelayAndRequestRateTest(BaseRobotTest, unittest.TestCase):
|
|
robots_txt = """\
|
|
User-agent: figtree
|
|
Crawl-delay: 3
|
|
Request-rate: 9/30
|
|
Disallow: /tmp
|
|
Disallow: /a%3cd.html
|
|
Disallow: /a%2fb.html
|
|
Disallow: /%7ejoe/index.html
|
|
"""
|
|
agent = 'figtree'
|
|
request_rate = namedtuple('req_rate', 'requests seconds')(9, 30)
|
|
crawl_delay = 3
|
|
good = [('figtree', '/foo.html')]
|
|
bad = ['/tmp', '/tmp.html', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html',
|
|
'/a%2fb.html', '/~joe/index.html']
|
|
|
|
def test_request_rate(self):
|
|
for url in self.good:
|
|
agent, url = self.get_agent_and_url(url)
|
|
with self.subTest(url=url, agent=agent):
|
|
if self.crawl_delay:
|
|
self.assertEqual(
|
|
self.parser.crawl_delay(agent), self.crawl_delay
|
|
)
|
|
if self.request_rate and self.parser.request_rate(agent):
|
|
self.assertEqual(
|
|
self.parser.request_rate(agent).requests,
|
|
self.request_rate.requests
|
|
)
|
|
self.assertEqual(
|
|
self.parser.request_rate(agent).seconds,
|
|
self.request_rate.seconds
|
|
)
|
|
|
|
|
|
class DifferentAgentTest(CrawlDelayAndRequestRateTest):
|
|
agent = 'FigTree Robot libwww-perl/5.04'
|
|
# these are not actually tested, but we still need to parse it
|
|
# in order to accommodate the input parameters
|
|
request_rate = None
|
|
crawl_delay = None
|
|
|
|
|
|
class InvalidRequestRateTest(BaseRobotTest, unittest.TestCase):
|
|
robots_txt = """\
|
|
User-agent: *
|
|
Disallow: /tmp/
|
|
Disallow: /a%3Cd.html
|
|
Disallow: /a/b.html
|
|
Disallow: /%7ejoe/index.html
|
|
Crawl-delay: 3
|
|
Request-rate: 9/banana
|
|
"""
|
|
good = ['/tmp']
|
|
bad = ['/tmp/', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html', '/a/b.html',
|
|
'/%7Ejoe/index.html']
|
|
crawl_delay = 3
|
|
|
|
|
|
class InvalidCrawlDelayTest(BaseRobotTest, unittest.TestCase):
|
|
# From bug report #523041
|
|
robots_txt = """\
|
|
User-Agent: *
|
|
Disallow: /.
|
|
Crawl-delay: pears
|
|
"""
|
|
good = ['/foo.html']
|
|
# bug report says "/" should be denied, but that is not in the RFC
|
|
bad = []
|
|
|
|
|
|
class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase):
|
|
# also test that Allow and Diasallow works well with each other
|
|
robots_txt = """\
|
|
User-agent: Googlebot
|
|
Allow: /folder1/myfile.html
|
|
Disallow: /folder1/
|
|
Request-rate: whale/banana
|
|
"""
|
|
agent = 'Googlebot'
|
|
good = ['/folder1/myfile.html']
|
|
bad = ['/folder1/anotherfile.html']
|
|
|
|
|
|
class UserAgentOrderingTest(BaseRobotTest, unittest.TestCase):
|
|
# the order of User-agent should be correct. note
|
|
# that this file is incorrect because "Googlebot" is a
|
|
# substring of "Googlebot-Mobile"
|
|
robots_txt = """\
|
|
User-agent: Googlebot
|
|
Disallow: /
|
|
|
|
User-agent: Googlebot-Mobile
|
|
Allow: /
|
|
"""
|
|
agent = 'Googlebot'
|
|
bad = ['/something.jpg']
|
|
|
|
|
|
class UserAgentGoogleMobileTest(UserAgentOrderingTest):
|
|
agent = 'Googlebot-Mobile'
|
|
|
|
|
|
class GoogleURLOrderingTest(BaseRobotTest, unittest.TestCase):
|
|
# Google also got the order wrong. You need
|
|
# to specify the URLs from more specific to more general
|
|
robots_txt = """\
|
|
User-agent: Googlebot
|
|
Allow: /folder1/myfile.html
|
|
Disallow: /folder1/
|
|
"""
|
|
agent = 'googlebot'
|
|
good = ['/folder1/myfile.html']
|
|
bad = ['/folder1/anotherfile.html']
|
|
|
|
|
|
class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase):
|
|
# see issue #6325 for details
|
|
robots_txt = """\
|
|
User-agent: *
|
|
Disallow: /some/path?name=value
|
|
"""
|
|
good = ['/some/path']
|
|
bad = ['/some/path?name=value']
|
|
|
|
|
|
class UseFirstUserAgentWildcardTest(BaseRobotTest, unittest.TestCase):
|
|
# obey first * entry (#4108)
|
|
robots_txt = """\
|
|
User-agent: *
|
|
Disallow: /some/path
|
|
|
|
User-agent: *
|
|
Disallow: /another/path
|
|
"""
|
|
good = ['/another/path']
|
|
bad = ['/some/path']
|
|
|
|
|
|
class EmptyQueryStringTest(BaseRobotTest, unittest.TestCase):
|
|
# normalize the URL first (#17403)
|
|
robots_txt = """\
|
|
User-agent: *
|
|
Allow: /some/path?
|
|
Disallow: /another/path?
|
|
"""
|
|
good = ['/some/path?']
|
|
bad = ['/another/path?']
|
|
|
|
|
|
class RobotHandler(BaseHTTPRequestHandler):
|
|
|
|
def do_GET(self):
|
|
self.send_error(403, "Forbidden access")
|
|
|
|
def log_message(self, format, *args):
|
|
pass
|
|
|
|
|
|
@unittest.skipUnless(threading, 'threading required for this test')
|
|
class PasswordProtectedSiteTestCase(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.server = HTTPServer((support.HOST, 0), RobotHandler)
|
|
|
|
self.t = threading.Thread(
|
|
name='HTTPServer serving',
|
|
target=self.server.serve_forever,
|
|
# Short poll interval to make the test finish quickly.
|
|
# Time between requests is short enough that we won't wake
|
|
# up spuriously too many times.
|
|
kwargs={'poll_interval':0.01})
|
|
self.t.daemon = True # In case this function raises.
|
|
self.t.start()
|
|
|
|
def tearDown(self):
|
|
self.server.shutdown()
|
|
self.t.join()
|
|
self.server.server_close()
|
|
|
|
@support.reap_threads
|
|
def testPasswordProtectedSite(self):
|
|
addr = self.server.server_address
|
|
url = 'http://' + support.HOST + ':' + str(addr[1])
|
|
robots_url = url + "/robots.txt"
|
|
parser = urllib.robotparser.RobotFileParser()
|
|
parser.set_url(url)
|
|
parser.read()
|
|
self.assertFalse(parser.can_fetch("*", robots_url))
|
|
|
|
|
|
class NetworkTestCase(unittest.TestCase):
|
|
|
|
def testPythonOrg(self):
|
|
support.requires('network')
|
|
with support.transient_internet('www.python.org'):
|
|
parser = urllib.robotparser.RobotFileParser(
|
|
"http://www.python.org/robots.txt")
|
|
parser.read()
|
|
self.assertTrue(
|
|
parser.can_fetch("*", "http://www.python.org/robots.txt"))
|
|
|
|
if __name__=='__main__':
|
|
unittest.main()
|