一个简单的网络爬虫-从网上爬取美女图片

CrawlerPicture.java 文件

package com.lym.crawlerDemo;

import java.io.DataInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import com.lym.mode.Picture;

/**
 * 从  https://m.qqba.com/ 爬取美女图片
 * @author Administrator
 *
 */
public class CrawlerPicture {

	public final static int STARTPAGE = 301;
	public final static int ENDPAGE = 500;//爬取的页面数量
	
	/**
	 * 获取图片的src和alt属性值
	 * @return
	 * @throws IOException
	 */
	public static List<Picture> getPictureUrl() throws IOException{
		int number = 1;
		List<Picture> pics = new ArrayList<Picture>();//存储获取到的所有图片的URL地址
		for (int i = STARTPAGE; i < ENDPAGE; i++) {
			String url = "https://m.qqba.com/people/list/"+i+".htm";
			Document doc = null;
			doc = Jsoup.connect(url).get();		//获取页面文档
			Elements divList = doc.body().select("div.image-cell");
			for (int j = 0; j < divList.size(); j++) {
				Elements imgList = divList.get(j).select("img");//一个网页内所有的img标签
				for (int k = 0; k < imgList.size(); k++) {
					Picture pic = new Picture();
					pic.setId(number++);
					pic.setSrc(imgList.get(k).attr("src"));
					pic.setAlt(imgList.get(k).attr("alt"));
					
					pics.add(pic);
				}
			}
		}
		return pics;
	}
	
	
	/**
	 * 获取图片输入流
	 * @param picUrl  图片的URL地址
	 * @return
	 * @throws IOException 
	 */
	public static InputStream getPictureInputStream(String picUrl) throws IOException{
		URL url = new URL(picUrl);
		DataInputStream dis = new DataInputStream(url.openStream());//获取图片的输入流
		return dis;
	}
	
	/**
	 * 保存图片到本地磁盘中
	 * @param number 图片编号
	 * @throws IOException 
	 */
	public static void savePicture(InputStream in, Picture pic) throws IOException{
		String newImgUrl = "D:/picture/"+pic.getAlt()+"--"+pic.getId()+".jpg";//图片在磁盘上的存储路径
		FileOutputStream fos = new FileOutputStream(new File(newImgUrl));
		byte[] buf = new byte[1024];
		int len = -1;
		while( (len = in.read(buf)) >0){
			fos.write(buf, 0, len);
		}
		fos.close();
	}
	
	
	
	
	
	/**
	 * 测试
	 * @param args
	 */
	public static void main(String[] args) {
		try {
			List<Picture> pics = getPictureUrl();
			System.out.println("图片正在下载...");
			for (int i = 0; i < pics.size(); i++) {
				Picture pic = pics.get(i);
				String picUrl = pic.getSrc();
				InputStream in = getPictureInputStream(picUrl);
				savePicture(in, pic);
				in.close();
			}
			System.out.println("下载完成！");
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
}package com.lym.cra