AI大语言模型之分布式训练概述

使用python手写了个简易的豆瓣影评爬虫代码。

__author__ = 'wsx'

import time
import requests
from bs4 import BeautifulSoup
import os
import re
import uuid


def clean_windows_filename(string_file_name):
    invalid_chars = r'[\\/:*?"<>|]'
    return re.sub(invalid_chars, '', string_file_name)
class Mz:



    def __init__(self):
        # self.url = 'http://www.mzitu.com'
        # self.headers = {
        #     'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
        #     'Referer': 'http://www.mzitu.com/'
        # }
        self.url = 'https://nj9.net/id/'
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
            'Referer': 'https://nj9.net'
        }
        self.proxy_list = {
                'http': 'http://47.56.110.204:8989',
                'https': 'https://65.109.204.150:80'
            }
        self.req = requests.session()
        self.all_a = 5000
        self.all_a_title = []
        self.all_a_max = []
        try:
            os.makedirs(os.path.join(os.getcwd(), 'douban'))
            os.chdir(os.path.join(os.getcwd(), 'douban'))
        except OSError as e:
            # 处理其他类型的错误
            print("已存在文件，跳过创建文件！")
        finally:
            self.initpwd = os.getcwd()

    def contains_unfound(s):
        return "未找到" in s


    def Downloadimg2(self):
            nurl = "https://movie.douban.com/subject/36328210/comments"
            html = self.req.get(nurl, headers=self.headers)
            title = BeautifulSoup(html.text, 'lxml').find('h1').text
            # 定义不允许的字符
            invalid_chars = '<>:"/\\|?*'
            # 使用正则表达式替换所有不允许的字符为空字符串
            sanitized = re.sub(f'[{re.escape(invalid_chars)}]', '', title)
            if "未找到" in title:
                print("未找到")
            else:
                try:
                    os.makedirs(os.path.join(os.getcwd(), sanitized))
                except OSError as e:
                    # 处理其他类型的错误
                    print(title + "已存在文件，跳过创建文件！")
                finally:
                    os.chdir(os.path.join(os.getcwd(), sanitized))
                shortUrl = BeautifulSoup(html.content, 'lxml')
                # 查找所有的<img>标签
                short_tags = shortUrl.find_all('span', class_='short')
                # 获取每个<img>标签的src属性
                all_short = [short.text for short in short_tags]
                # 打印获取到的所有短评
                for short in all_short:
                    print(short)
                    # 打开文件并写入文本
                    txt_url = os.getcwd() + "\\" + sanitized+str(".txt")
                    with open(txt_url, 'a', encoding='utf-8') as file:
                        file.write(short)
                        file.write("\n\n")
            os.chdir(self.initpwd)
            print('Dowmload completed!')


if __name__ == '__main__':
    test = Mz()
    test.Downloadimg2()