0
点赞
收藏
分享

微信扫一扫

【雕爷学编程】MicroPython动手做(27)——物联网之掌控板小程序3

科牛 2023-08-06 阅读 70

golang爬虫下载的代码:

下载程序的借鉴内容:

这个是关于gbk,utf8等相互转换的包

 一、标准下载代码

package downloader

import (
	"log"
	"net/http"
	"io"

	"github.com/axgle/mahonia"

	"github.com/go-crawler/car-prices/fake"
)

func Get(url string) io.Reader {
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Printf("http.NewRequest err: %v", err)
	}

	req.Header.Add("User-Agent", fake.GetUserAgent())
	req.Header.Add("Referer", "https://car.autohome.com.cn")

	resp, err := client.Do(req)
	if err != nil {
		log.Printf("client.Do err: %v", err)
	}

	mah := mahonia.NewDecoder("gbk")
	return mah.NewReader(resp.Body)
}
举报

相关推荐

0 条评论