异步爬取数据（追加保存在Excel中）

# 追加保存异步访问数据成功！
import urllib.request
import json
import re
import xlwt
import xlrd
from xlutils.copy import copy

def main():
# 这两行代码用于访问获取html网页，避免反爬，获取后保存在本地html文件中
#     url = 'https://movie.douban.com/j/search_subjects?type=movie&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=20&page_start=0'
#     html = askURL(url)

    file = open(r'豆瓣异步加载0.html','r',encoding='utf-8')   #打开爬虫获取的本地html文件
    dataList = getData(file)    # 获取数据列表
    path = r'豆瓣异步爬取数据.xls'    # 将爬取的数据置于该.xls文件中

#     savaData(path,dataList)    # 用于新建xls文件并保存数据
    xls_append(path,dataList)    # 用于在已有的xls文件末尾追加数据
def xls_append(path,value):
    index = len(value)
    workbook = xlrd.open_workbook(path)
    sheets = workbook.sheet_names()
#     print(sheets)
    worksheet = workbook.sheet_by_name(sheets[0])
    rows_old = worksheet.nrows
    
    new_workbook = copy(workbook)
    new_worksheet = new_workbook.get_sheet(0)
    for i in range(0,index):
        for j in range(0,len(value[i])):
            new_worksheet.write(i+rows_old,j,value[i][j])
    new_workbook.save(path)
    print("追加数据成功")
def savaData(path,dataList):
    book = xlwt.Workbook(encoding='utf-8')
    sheet = book.add_sheet('sheet2',cell_overwrite_ok=True)
    col = ('电影名','评分','链接')
    for i in range(len(col)):
        sheet.write(0,i,col[i])
        for j in range(len(dataList)):
            sheet.write(j+1,i,dataList[j][i])
    book.save(path)
    print("异步数据爬取保存成功！")
def getData(file):
    re_data = re.findall(r'{"subjects":(.+)}',str(file.readlines()))
#     print(re_data)
    dataList = []
    jsonObj = json.loads(re_data[0])
    
    for item in jsonObj:
        list = []
        list.append(item['title'])
        list.append(item['rate'])
        list.append(item['url'])
#         print(list,end='\n*******************************\n')
        dataList.append(list)
    return dataList


# *************************************************************************
def askURL(url):
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'
    }
    req = urllib.request.Request(url=url,headers=headers)
    response = urllib.request.urlopen(req)
    html = response.read().decode('utf-8')
#     print(html)
    return html
if __name__ == '__main__':
    main()