Python网络爬虫：User Agent和代理IP

# 一：
# 异常处理，及设置请求次数
# 可添加time时间间隔
import urllib2
def download(url,num_retries=2):
	print("Downloading:",url)
	try:
		html = urllib2.urlopen(url).read()
	except urllib2.URLError as e:
		print("Download error:",e.reason)
		html = None
		if num_retries>0:
			if hasattr(e,"code") and 500 <= e.code <600:
				# 当服务器响应结果为5XX时重新请求
				return download(url,num_retries-1)
	return html

# 二：在一的基础上添加用户代理
import urllib2
def download(url,uesr_agent="wswp",num_retries=2):
	print("Downloading:",url)
	headers = {"User_agent":uesr_agent}
	requset = urllib2.Request(url,headers=headers)
	try:
		html = urllib2.urlopen(requset).read()
	except urllib2.URLError as e:
		print("Download error:",e.reason)
		html = None
		if num_retries>0:
			if hasattr(e,"code") and 500 <= e.code <600:
				# 当服务器响应结果为5XX时重新请求
				return download(url,num_retries-1)
	return html

# 三、支持代理
import urllib2
def download(url,uesr_agent="wswp",proxy=None,num_retries=2):
	print("Downloading:",url)
	headers = {"User_agent":uesr_agent}
	requset = urllib2.Request(url,headers=headers)
	opener = urllib2.build_opener()
	if proxy:
		proxy_params = {urlparse.urlparse(url).scheme:proxy}
		opener.add_handler(urllib2.ProxyHandler(proxy_params))
	try:
		html = urllib2.urlopen(requset).read()
	except urllib2.URLError as e:
		print("Download error:",e.reason)
		html = None
		if num_retries>0:
			if hasattr(e,"code") and 500 <= e.code <600:
				# 当服务器响应结果为5XX时重新请求
				return download(url,num_retries-1)
	return html# 一：
# 异常处理，及设置请求次数
# 可添加time