import requests,os,re,threading,time
def download(url,fname):
with open(fname,mode="wb") as fw:
fw.write(requests.get(url).content)
def get_url(fname,re_str):
result_url=[] #保存图片url地址
re_obj=re.compile(re_str) #字符串编译正则对象
with open(fname,mode='r') as fr:
for line in fr.readlines():
data=re_obj.search(line)
if data != None:
result_url.append(data.group())
return result_url
if __name__ == '__main__':
fname="/opt/ruirui/mysina.html"
if os.path.exists("/opt/ruirui/mysina.html")==False:
download("https://www.sina.com.cn/","/opt/ruirui/mysina.html")
print("download ok")
if os.path.exists("/opt/bee")==False:
os.mkdir("/opt/bee")
print("mkdir ok")
re_str="(http|https)://[\w\.\/-]+\.(jpg|png|jpeg|gif|bmp|wmf)"
picture_list=get_url(fname,re_str)
start=time.time()
for url in picture_list:
path=os.path.join("/opt/bee",os.path.basename(url))
# path=os.path.join("/opt/bee",url.split('/')[-1])
# download(url,path) #单线程
#多线程
t=threading.Thread(target=download,args=(url,path,))
t.start()
end=time.time()
print(start-end)
运行结果:
/usr/local/bin/python3 /root/PycharmProjects/NSD211/test/demo3.py
-0.06506824493408203
Process finished with exit code 0
查看文件：
[root@server1 bee]# ls
08f9-4998c08393e575de378525f8fdaf1a16.png 5496-22e11033da5965443c51964efbecab9f.jpg b376-6a49ecb73b329f7940efb385e923357f.jpg ec1d-45d91047f61f2848a5ce63373a695ad5.jpg
1202-c52da3691f0287d1bfdfd100c408849c.jpg 62c6-7c2fc1180981678f31fac2a48afbc610.jpg b4bb-41e4f7802cbe6ca8f581dcae104a4e27.png f309-d5ba42d4fa379ce4b4d9de37054da14f.jpg
12377app.png 6361-35a3fa7bf5771c88f1dffe499cb55406.jpg bbde-5f2122bc7abf638cd210eebe63d1a2e2.jpg f6fd-d4fff8b508a2cd832bc28a87fe6fb3a2.jpg
123.png 7966-b6897d4ce482effc58640e776b65f8e3.jpg cd67-dc16796697d1f8fd3838c3bd76fa8cb7.png f8b9-dae73a77654f3231dfa692bebbfa8989.jpg
14.png 7f69-e60738db3386efd9edf4258de5fa5372.jpg cedc-0413adff60220843fd82ea835fc77ce2.png LieRen.png
189a-ec1260b0890ffb25634f4825af4e971f.jpg 8a56-4b9060116388235dd03796545c1b87a1.jpg cfaa-ee005a71fd6958c195dfb37aac60b87f.png w105h70z1l50t1q60603.jpg
2690-0c05734feb3fe8bcc667f4b92d7af59d.jpg 9043-3f2bf3de39bd11a131d0914d072f0438.jpg d756-cdbb83c22ded7457f0844496d6ad7b1d.png w105h70z1l50t1q608a4.jpg
41b1-d98620448d47018550b383767c661a87.jpg 92bb-8c1afbced2859a77a0d8a1fb2117bd13.jpg df5f-1cdee3addd4b4d6e9958cee9f0f8d50f.jpg WeiBiaoTi-2.png
43d6-f6c617afb338e361344e3fa183e921dc.jpg 9338-631798b2040a1e3966ebde93ba37a346.jpg e54a-fc9e91ebbd080dfa34b9a55323ad74da.jpg
4781-3342b74e7517b08b56180134539199d3.jpg a258-4260392496e78cbb202a8da3ff15da33.jpg eaa2-96309fe332aedc6c8ac7450816a44c46.jpg
4d1d-b6520721c0ddf7d5961dbd69684b2754.jpg a7db-0bdaca7672ffd3521f1efbbaaef27ddd.jpg eaa5-685ee54a97b4d16bfaf85b04ea3cce85.png
