python爬虫爬取百度指数-CFANZ编程社区

1.环境：python版本+pyCharm

python3.7安装教程_PtaQ的博客-CSDN博客_python3.7

2.获取百度指数的cookie

1.先登录百度账号

2.在此页面点击键盘F12

3.点击NetWork、XHR

4.百度指数搜索“网贷”

5.复制cookie

点击右侧字符右键出现copy value复制

3.代码

# coding=gbk
# -*- coding:uft-8 -*-
# 百度指数

import requests
import urllib.parse
import execjs
import time


def collect(city, area):
    # name 想要搜索的关键字
    name = '网贷'
    wordType = 1
    # startDate 开始时间
    startDate = '2020-01-01'
    # endDate 结束时间
    endDate = '2022-01-09'
    baseUrl = 'https://index.baidu.com/api/SearchApi/index'
    addUrl = f'?area={area}&word=[[%7B%22name%22:%22{urllib.parse.quote(name)}%22,%22wordType%22:{wordType}%7D]]&startDate={startDate}&endDate={endDate}'
    headers = {
        'Referer': 'https://index.baidu.com/v2/main/index.html',
        'Cookie': '',

        'User-Agent': 'Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/91.0.4472.106Safari/537.36'
    }
    res = requests.get(url=baseUrl + addUrl, headers=headers).json()
    allData = res['data']['userIndexes'][0]['all']['data']
    uniqid = res['data']['uniqid']
    print(f'allData: {allData}')
    print(f'uniqid: {uniqid}')
    url = 'https://index.baidu.com/Interface/ptbk?uniqid=' + uniqid
    res = requests.get(url=url, headers=headers).json()
    data = res['data']
    print(f'data: {data}')
    ctx = execjs.compile('var decrypt=function(t,e){if(t){for(var n=t.split(""),i=e.split(""),a={},r=[],o=0;o<n.length/2;o++){a[n[o]]=n[n.length/2+o]}for(var s=0;s<e.length;s++){r.push(a[i[s]])}return r.join("")}};')
    resData = ctx.call('decrypt', data, allData)
    print(f'resData: {resData}')
    resLs = []
    for i in resData.split(','):
        if i:
            resLs.append(eval(i))
        else:
            resLs.append(0)
    star = int(time.mktime(time.strptime(startDate, '%Y-%m-%d')))
    end = int(time.mktime(time.strptime(endDate, '%Y-%m-%d')))
    # span想要的时间间隔 换算成秒数(如7天=604800秒)
    span = 604800
    dateLs = []
    stamp = star
    while stamp <= end:
        today = time.strftime('%Y-%m-%d', time.localtime(stamp))
        stamp += span
        tomorrow = time.strftime('%Y-%m-%d', time.localtime(stamp))
        dateLi = today + '~' + tomorrow
        dateLs.append(dateLi)
    for i in range(len(resLs)):
        try:
            open(f'./{name}.txt', 'a+').write(f'{dateLs[i]},{resLs[i]}\n')
        except IndexError:
            break


if __name__ == '__main__':
    print('=' * 150)
    ls = [
        {'label': '全国', 'value': '0'},
        # {'label': '深圳', 'value': '94'},
        # {'label': '东莞', 'value': '133'},
        # {'label': '云浮', 'value': '195'},
        # {'label': '佛山', 'value': '196'},
        # {'label': '湛江', 'value': '197'},
        # {'label': '江门', 'value': '198'},
        # {'label': '惠州', 'value': '199'},
        # {'label': '珠海', 'value': '200'},
        # {'label': '韶关', 'value': '201'},
        # {'label': '阳江', 'value': '202'},
        # {'label': '茂名', 'value': '203'},
        # {'label': '潮州', 'value': '204'},
        # {'label': '揭阳', 'value': '205'},
        # {'label': '中山', 'value': '207'},
        # {'label': '清远', 'value': '208'},
        # {'label': '肇庆', 'value': '209'},
        # {'label': '河源', 'value': '210'},
        # {'label': '梅州', 'value': '211'},
        # {'label': '汕头', 'value': '212'},
        # {'label': '汕尾', 'value': '213'}
    ]
    for li in ls:
        print(li)
        collect(li['label'], li['value'])
        print('=' * 150)
        time.sleep(5)

把复制的cookie粘贴到下图位置