golang爬取免费的代理IP，并验证代理IP是否可用
这里选择爬取西刺的免费代理Ip，并且只爬取了一页，爬取的时候不设置useAgent西刺不会给你数据，西刺也做反爬虫处理了，所以小心你的IP被封掉
代码：
package main import ( "fmt" "github.com/PuerkitoBio/goquery" "net/http" "net/url" "strings" ) var total, useful int = 0, 0 var status = make(chan int) func main() { xcurl := "" request, _ := http.NewRequest("GET", xcurl, nil) request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:50.0) Gecko/20100101 Firefox/50.0") cli1 := &http.Client{} response, err := cli1.Do(request) if err != nil { fmt.Println(err) } dom, _ := goquery.NewDocumentFromResponse(response) dom.Find("#ip_list tbody tr").Each(func(i int, context *goquery.Selection) { ip := context.Find("td").Eq(1).Text() port := context.Find("td").Eq(2).Text() httpType := context.Find("td").Eq(5).Text() proxyIp := strings.ToLower(httpType) + "://" + ip + ":" + port // nim := context.Find("td").Eq(4).Text() //是否是高匿,高匿的可以隐藏你的原始IP if ip != "" && port != "" { total++ go checkProxyIP(proxyIp, i) } }) for i := 0; i < total; i++ { <-status } fmt.Println("num=", total, "\nuseful=", useful) fmt.Println("END!") } func checkProxyIP(proxyIp string, i int) { req, _ := http.NewRequest("GET", "", nil) //这里自己搭个web服务验证代理是否可用 proxy, _ := url.Parse(proxyIp) cli2 := &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyURL(proxy), }, } resp, _ := cli2.Do(req) if resp != nil && resp.StatusCode == 200 { buf := make([]byte, 1024) n, _ := resp.Body.Read(buf) reStr := string(buf[:n]) if reStr == "Hello World" { //验证代理有没有做手脚，可能给你返回一堆广告 useful++ fmt.Println(proxyIp) } } status <- i }