java 数据采集，采集各个网站的小说文章示例

1、首先要有 jsoup-1.7.1.jar jar包 引入。

2、jsoup 的工具类：

package com.wp.util;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

public class JsoupUtil {

	public static Document getDocument(String url) {
		int error_count = 0;
		Document doc = null;
		while (true) {
			if (error_count > Constants.url_error_count) {
				break;
			}
			try {
				doc = Jsoup
						.connect(url)
						.timeout(Constants.url_ConnectTimeout)
						.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
						.header("Accept-Encoding", "gzip,deflate,sdch")
						.header("Connection", "keep-alive")
						.followRedirects(true)
						.userAgent("Mozilla/4.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)")
						.get();
			} catch (Exception e) {
				error_count++;
			}
			if (doc != null) {
				break;
			}
		}
		return doc;
	}

	public static Document parseHtml(String html) {
		return Jsoup.parse(html);
	}

}
p