Python不使用scrapy框架而编写的网页爬虫程序

本文代码节选（略有改动）自《Python程序设计（第2版）》（董付国编著，清华大学出版社），没有使用scrapy爬虫框架，而是使用标准库urllib访问网页实现爬虫功能，如果网页包含感兴趣的关键词，就把这个网页保存成为本地文件，并且有效控制了爬取深度，避免爬遍互联网。

import sys
import re
import os
import urllib.request as lib
def craw_links(url, depth, keywords, processed):
    '''url:the url to craw
     depth:the current depth to craw
     keywords:the tuple of keywords to focus
     processed:the urls already crawled
    '''
    
    if url.startswith(('http://', 'https://')):        
        if url not in processed:
            # mark this url as processed
            processed.append(url)
        else:
            # avoid processing the same url again
            return
        
        print('Crawing '+url+'...')
        
        with lib.urlopen(url) as fp:
            # Python3 returns bytes
            # so need to decode
            contents = fp.read()
            contents_decoded = contents.decode('UTF-8')
        # form a regular expression
        pattern = '|'.join(keywords)
        # if this page contains certain keywords, save it to a file
        flag = False
        if pattern:
            searched = re.search(pattern, contents_decoded)
        else:
            # if the keywords to filter is not given, save current page
            flag = True
            
        if flag or searched:
            with open('craw\\'+url.replace(':','_').replace('/','_'), 'wb') as fp:
                fp.write(contents)
                
        # find all the links in the current page
        links = re.findall('href="(.*?)"', contents_decoded)
        # craw all links in the current page
        for link in links:
            # consider the relative path
            if not link.startswith(('http://','https://')):                
                try:
                    index = url.rindex('/')
                    link = url[0:index+1]+link
                except:
                    pass
                
            # control the crawl depth
            if depth>0 and link.endswith(('.htm','.html')):
                craw_links(link, depth-1, keywords, processed)
                
if __name__ == '__main__':   
    processed = []   
    keywords = ('datetime','KeyWord2')
    if not os.path.exists('craw') or not os.path.isdir('craw'):
        os.mkdir('craw')
    start_url = r'https://docs.python.org/3/library/index.html'
    craw_links(start_url, 1, keywords, processed)