【Java爬虫】HttpClient样例

HttpClient简介

<dependency>
  <groupId>org.apache.httpcomponents</groupId>
  <artifactId>httpclient</artifactId>
  <version>4.5.13</version>
</dependency>

一般使用步骤

发送Get请求

public void httpClientTest() throws Exception{
        CloseableHttpClient httpClient= HttpClients.createDefault();
        //不带参数
        //HttpGet get=new HttpGet("http://www.it.cn");
        //带参数
        HttpGet get=new HttpGet("http://yun.com/course/936.html?capid=1$czpc$jingjiaczpz-PC-1");
        CloseableHttpResponse response=httpClient.execute(get);
        StatusLine statusLine=response.getStatusLine();
        System.out.println(statusLine);
        int statusCode=statusLine.getStatusCode();
        System.out.println(statusCode);
        HttpEntity entity= response.getEntity();
        String html= EntityUtils.toString(entity,"utf-8");
        System.out.println(html);
        response.close();
        httpClient.close();
    }

发送Post请求

public void testPost() throws Exception{
        CloseableHttpClient httpClient=HttpClients.createDefault();
        HttpPost post=new HttpPost("http://yun.com/course/936.html");
        //创建list存放请求表单
        List<NameValuePair>form=new ArrayList<>();
        //较多的提交数据可以增元素
        form.add(new BasicNameValuePair("capid","1$czpc$jingjiaczpz-PC-1"));
       // form.add(new BasicNameValuePair("capid","1$czpc$jingjiaczpz-PC-1"));
        //form.add(new BasicNameValuePair("capid","1$czpc$jingjiaczpz-PC-1"));
        HttpEntity entity=new UrlEncodedFormEntity(form);
        post.setEntity(entity);
        CloseableHttpResponse response= httpClient.execute(post);
        String s = EntityUtils.toString(response.getEntity(), "utf-8");
        System.out.println(s);
        response.close();
        httpClient.close();
    }

请求连接池

public void testPoolingConnectManager() throws Exception{
        PoolingHttpClientConnectionManager cm=new PoolingHttpClientConnectionManager();
        CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();
        HttpGet get=new HttpGet("http://www.it.cn");
        CloseableHttpResponse execute = httpClient.execute(get);
        String s = EntityUtils.toString(execute.getEntity(), "utf-8");
        System.out.println(s);
        execute.close();
        //此处不需要关闭连接池 会直接被连接池回收
    }