针对数据提取的项目实战:
# -*- coding: utf-8 -*-
from lxml import etree
text = '''
<div>
<ul>
<li class="item-1">
<a href="link1.html">first item</a>
</li>
<li class="item-1">
<a href="link2.html">second item</a>
</li>
<li class="item-inactive">
<a href="link3.html">third item</a>
</li>
<li class="item-1">
<a href="link4.html">fourth item</a>
</li>
<li class="item-0">
<a href="link5.html">fifth item</a>
</li>
</ul>
</div>
'''
html = etree.HTML(text)
print(etree.tostring(html))
运行结果,可以看出body等标签自动已补全。
实现对百度贴吧爬取:
注意点1:
这里提供一些老版本的User-Agent :
注意点2:
注意点3:
# -*- coding: utf-8 -*-
import requests
from lxml import etree
# url
# headers
# 发送请求获取响应
# 从响应中提取数据
# 判断结束
class Tieba(object):
def __init__(self, name):
self.url = "https://tieba.baidu.com/f?kw={}".format(name)
print(self.url)
self.headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
# "User-Agent": "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; T132461)"
}
def get_data(self, url):
response = requests.get(url, headers=self.headers)
with open("temp.html", "wb") as f:
f.write(response.content)
return response.content
def parse_data(self, data):
# 创建element对象
data = data.decode().replace("<!--", "").replace("-->", "")
html = etree.HTML(data)
el_list = html.xpath('//*[@id="thread_list"]/li/div/div[2]/div[1]/div[1]/a')
# print(len(el_list))
data_list = []
for el in el_list:
temp = {}
temp['title'] = el.xpath('./text()')[0]
temp['link'] = 'https://tieba.baidu.com' + el.xpath('./@href')[0]
data_list.append(temp)
# 获取下一页url
try:
next_url = 'https:' + html.xpath('//a[contains(text(),"下一页>")]/@href')[0]
except:
next_url = None
return data_list, next_url
def save_data(self, data_list):
for data in data_list:
print(data)
def run(self):
next_url = self.url
while True:
# 发送请求获取响应
data = self.get_data(next_url)
# 从响应中提取数据,数据和翻页用的url
data_list, next_url = self.parse_data(data)
self.save_data(data_list)
print(next_url)
# 判断是否结束
if next_url == None:
break
if __name__ == '__main__':
tieba = Tieba("美食天下")
tieba.run()