关于java爬虫的一些学习记录_从简去繁的博客

一.创建HttpClient工具类(底层代码)

@Component
public class HttpUtils {
    private PoolingHttpClientConnectionManager cm;

    public HttpUtils() {
        this.cm = new PoolingHttpClientConnectionManager();
        cm.setMaxTotal(100);
        cm.setDefaultMaxPerRoute(10);
    }

    public String doGetForElement(String url){
         CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(this.cm).build();

         HttpGet httpGet = new HttpGet(url);
         
         httpGet.setConfig(this.getRequestConfig());
         
         
         CloseableHttpResponse httpResponse = null;

         try {
             httpResponse = httpClient.execute(httpGet);

             if (httpResponse.getStatusLine().getStatusCode()==200){
                 if(httpResponse.getEntity()!=null){
                     String element = EntityUtils.toString(httpResponse.getEntity(),"gb2312");
                     return element;
                 }
             }
         } catch (IOException e) {
            e.printStackTrace();
         }finally {
             try {
                 httpResponse.close();
             } catch (IOException e) {
                 e.printStackTrace();
             }
         }
         return "";
    }
    
    public RequestConfig getRequestConfig(){
        RequestConfig requestConfig = RequestConfig.custom()
                .setConnectTimeout(1000)
                .setConnectionRequestTimeout(1000)
                .setSocketTimeout(60*1000)
                .build();
        return requestConfig;
    }
}@Component
public cla