系列文章目录

文章目录

• 系列文章目录
• 前言
• 一、爬取结果展示
• • 英雄名称《李白》《貂蝉》《小乔》等
• 二、爬虫源码
• • 1.python+chromedriver
• 总结

前言

一、爬取结果展示

英雄名称《李白》《貂蝉》《小乔》等

1、李白
2、貂蝉
3、小乔

二、爬虫源码

1.python+chromedriver

1.1、chromedriver配置链接
1.2、库安装

pip install jsonpath
pip install selenium
pip --default-timeout=100 install wheel -i http://pypi.douban.com/simple/ --trusted-host pypi.douban.com
pip --default-timeout=100 install lxml -i http://pypi.douban.com/simple/ --trusted-host pypi.douban.com

代码如下（示例）：

# !/usr/bin/nev python
# -*-coding:utf8-*-

import requests, os, jsonpath, re
from selenium import webdriver
from pprint import pprint
from lxml import etree

def main():

    start_url = r'https://pvp.qq.com/web201605/js/herolist.json'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
                      'Chrome/87.0.4280.88 Safari/537.36',
        'Referer': 'https://pvp.qq.com/web201605/herolist.shtml'
    }

    driver = webdriver.Chrome(executable_path=r'C:\Users\Administrator\AppData\Local\Google\Chrome\Application\chromedriver.exe')

    # 第一次请求，获取hero_id hero_name hero_skin_names
    response = requests.get(start_url, headers=headers).json()
    # pprint(response)
    hero_ids = jsonpath.jsonpath(response, '$..ename')
    # pprint(hero_ids)
    hero_names = jsonpath.jsonpath(response, '$..cname')
    # pprint(hero_names)

    for hero_name, hero_id in zip(hero_names, hero_ids):
        hero_info_url = r'https://pvp.qq.com/web201605/herodetail/{}.shtml'.format(hero_id)


        # 发送英雄详情页请求得到 hero_info_content

        driver.get(hero_info_url)
        # 获取页面源码
        hero_info_content = driver.page_source
        # lxml解析
        hero_info_content_str = etree.HTML(hero_info_content)

        # 提取 hero_skin_names hero_skin_urls
        hero_skin_names = hero_info_content_str.xpath(r'//ul[@class="pic-pf-list pic-pf-list3"]/@data-imgname')[
            0].split('|')

        hero_skin_urls = hero_info_content_str.xpath(r'//ul[@class="pic-pf-list pic-pf-list3"]//img/@data-imgname')

        # hero_skin_name进行替换不必要的信息
        for hero_skin_name, hero_skin_url in zip(hero_skin_names, hero_skin_urls):
            suffix_notation = re.findall(r'&\d.?', hero_skin_name)[0]
            hero_skin_name = hero_skin_name.replace(suffix_notation, '')
            # 补全hero_skin_url地址
            hero_skin_url = r'https:'+hero_skin_url
            # 获取图片的二进制信息
            img_content = requests.get(hero_skin_url, headers=headers).content
            try:
                # 创建文件夹
                if not os.path.exists('./{}'.format(hero_name)):
                    os.mkdir(r'./{}'.format(hero_name))
                with open(r'./{}/{}.jpg'.format(hero_name, hero_skin_name), 'wb')as f:
                    f.write(img_content)
                    print('图片正在下载：{}/{}.jpg'.format(hero_name, hero_skin_name))

            except Exception as e:
                continue

if __name__ == '__main__':
    main()

总结

分享：
没有任何问题可以向无穷那样深深的触动人的情感，很少有别的观念能像无穷那样激励理智产生富有成果的思想，然而也没有任何其他的概念能向无穷那样需要加以阐明。——希尔伯特（Hilbert）