心血来潮写了个多线程抓妹子图，虽然代码还是有一些瑕疵，但是还是记录下来，分享给大家。
Pic_downloader.py
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
# -*- coding: utf-8 -*-"""Created on Fri Aug 07 17:30:58 2015 @author: Dreace"""import urllib2import sysimport timeimport osimport randomfrom multiprocessing.dummy import Pool as ThreadPool type_ = sys.getfilesystemencoding()def rename(): return time.strftime("%Y%m%d%H%M%S")def rename_2(name): if len(name) == 2: name = '0' + name + '.jpg' elif len(name) == 1: name = '00' + name + '.jpg' else: name = name + '.jpg' return namedef download_pic(i): global count global time_out if Filter(i): try: content = urllib2.urlopen(i,timeout = time_out) url_content = content.read() f = open(repr(random.randint(10000,999999999)) + "_" + rename_2(repr(count)),"wb") f.write(url_content) f.close() count += 1 except Exception, e: print i + "下载超时，跳过！".decode("utf-8").encode(type_)def Filter(content): for line in Filter_list: line=line.strip('\n') if content.find(line) == -1: return Truedef get_pic(url_address): global pic_list try: str_ = urllib2.urlopen(url_address, timeout = time_out).read() url_content = str_.split("\"") for i in url_content: if i.find(".jpg") != -1: pic_list.append(i) except Exception, e: print "获取图片超时，跳过！".decode("utf-8").encode(type_)MAX = 2count = 0time_out = 60thread_num = 30pic_list = []page_list = []Filter_list = ["imgsize.ph.126.net","img.ph.126.net","img2.ph.126.net"]dir_name = "C:\Photos\\"+rename()os.makedirs(dir_name)os.chdir(dir_name)start_time = time.time()url_address = "http://sexy.faceks.com/?page="for i in range(1,MAX + 1): page_list.append(url_address + repr(i))page_pool = ThreadPool(thread_num)page_pool.map(get_pic,page_list)print "获取到".decode("utf-8").encode(type_),len(pic_list),"张图片，开始下载！".decode("utf-8").encode(type_)pool = ThreadPool(thread_num) pool.map(download_pic,pic_list)pool.close() pool.join()print count,"张图片保存在".decode("utf-8").encode(type_) + dir_nameprint "共耗时".decode("utf-8").encode(type_),time.time() - start_time,"s" |
我们来看下一个网友的作品
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
|
#coding: utf-8 ############################################################## File Name: main.py# Author: mylonly# mail: mylonly@gmail.com# Created Time: Wed 11 Jun 2014 08:22:12 PM CST##########################################################################!/usr/bin/pythonimport re,urllib2,HTMLParser,threading,Queue,time#各图集入口链接htmlDoorList = []#包含图片的Hmtl链接htmlUrlList = []#图片Url链接QueueimageUrlList = Queue.Queue(0)#捕获图片数量imageGetCount = 0#已下载图片数量imageDownloadCount = 0#每个图集的起始地址，用于判断终止nextHtmlUrl = ''#本地保存路径localSavePath = '/data/1920x1080/'#如果你想下你需要的分辨率的，请修改replace_str,有如下分辨率可供选择1920x1200，1980x1920,1680x1050,1600x900,1440x900,1366x768,1280x1024,1024x768,1280x800replace_str = '1920x1080'replaced_str = '960x600'#内页分析处理类class ImageHtmlParser(HTMLParser.HTMLParser):def __init__(self):self.nextUrl = ''HTMLParser.HTMLParser.__init__(self)def handle_starttag(self,tag,attrs):global imageUrlListif(tag == 'img' and len(attrs) > 2 ):if(attrs[0] == ('id','bigImg')):url = attrs[1][1]url = url.replace(replaced_str,replace_str)imageUrlList.put(url)global imageGetCountimageGetCount = imageGetCount + 1print urlelif(tag == 'a' and len(attrs) == 4):if(attrs[0] == ('id','pageNext') and attrs[1] == ('class','next')):global nextHtmlUrlnextHtmlUrl = attrs[2][1];#首页分析类class IndexHtmlParser(HTMLParser.HTMLParser):def __init__(self):self.urlList = []self.index = 0self.nextUrl = ''self.tagList = ['li','a']self.classList = ['photo-list-padding','pic']HTMLParser.HTMLParser.__init__(self)def handle_starttag(self,tag,attrs):if(tag == self.tagList[self.index]):for attr in attrs:if (attr[1] == self.classList[self.index]):if(self.index == 0):#第一层找到了self.index = 1else:#第二层找到了self.index = 0print attrs[1][1]self.urlList.append(attrs[1][1])breakelif(tag == 'a'):for attr in attrs:if (attr[0] == 'id' and attr[1] == 'pageNext'):self.nextUrl = attrs[1][1]print 'nextUrl:',self.nextUrlbreak#首页Hmtl解析器indexParser = IndexHtmlParser()#内页Html解析器imageParser = ImageHtmlParser()#根据首页得到所有入口链接print '开始扫描首页...'host = 'http://desk.zol.com.cn'indexUrl = '/meinv/'while (indexUrl != ''):print '正在抓取网页:',host+indexUrlrequest = urllib2.Request(host+indexUrl)try:m = urllib2.urlopen(request)con = m.read()indexParser.feed(con)if (indexUrl == indexParser.nextUrl):breakelse:indexUrl = indexParser.nextUrlexcept urllib2.URLError,e:print e.reasonprint '首页扫描完成，所有图集链接已获得：'htmlDoorList = indexParser.urlList#根据入口链接得到所有图片的urlclass getImageUrl(threading.Thread):def __init__(self):threading.Thread.__init__(self)def run(self):for door in htmlDoorList:print '开始获取图片地址,入口地址为:',doorglobal nextHtmlUrlnextHtmlUrl = ''while(door != ''):print '开始从网页%s获取图片...'% (host+door)if(nextHtmlUrl != ''):request = urllib2.Request(host+nextHtmlUrl)else:request = urllib2.Request(host+door)try:m = urllib2.urlopen(request)con = m.read()imageParser.feed(con)print '下一个页面地址为:',nextHtmlUrlif(door == nextHtmlUrl):breakexcept urllib2.URLError,e:print e.reasonprint '所有图片地址均已获得:',imageUrlListclass getImage(threading.Thread):def __init__(self):threading.Thread.__init__(self)def run(self):global imageUrlListprint '开始下载图片...'while(True):print '目前捕获图片数量:',imageGetCountprint '已下载图片数量:',imageDownloadCountimage = imageUrlList.get()print '下载文件路径:',imagetry:cont = urllib2.urlopen(image).read()patter = '[0-9]*\.jpg';match = re.search(patter,image);if match:print '正在下载文件：',match.group()filename = localSavePath+match.group()f = open(filename,'wb')f.write(cont)f.close()global imageDownloadCountimageDownloadCount = imageDownloadCount + 1else:print 'no match'if(imageUrlList.empty()):breakexcept urllib2.URLError,e:print e.reasonprint '文件全部下载完成...'get = getImageUrl()get.start()print '获取图片链接线程启动:'time.sleep(2)download = getImage()download.start()print '下载图片链接线程启动:' |
批量抓取指定网页上的所有图片
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
|
# -*- coding:utf-8 -*-# coding=UTF-8 import os,urllib,urllib2,re url = u"http://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=index&fr=&sf=1&fmq=&pv=&ic=0&nc=1&z=&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&word=python&oq=python&rsp=-1"outpath = "t:\\" def getHtml(url): webfile = urllib.urlopen(url) outhtml = webfile.read() print outhtml return outhtml def getImageList(html): restr=ur'(' restr+=ur'http:\/\/[^\s,"]*\.jpg' restr+=ur'|http:\/\/[^\s,"]*\.jpeg' restr+=ur'|http:\/\/[^\s,"]*\.png' restr+=ur'|http:\/\/[^\s,"]*\.gif' restr+=ur'|http:\/\/[^\s,"]*\.bmp' restr+=ur'|https:\/\/[^\s,"]*\.jpeg' restr+=ur'|https:\/\/[^\s,"]*\.jpeg' restr+=ur'|https:\/\/[^\s,"]*\.png' restr+=ur'|https:\/\/[^\s,"]*\.gif' restr+=ur'|https:\/\/[^\s,"]*\.bmp' restr+=ur')' htmlurl = re.compile(restr) imgList = re.findall(htmlurl,html) print imgList return imgList def download(imgList, page): x = 1 for imgurl in imgList: filepathname=str(outpath+'pic_%09d_%010d'%(page,x)+str(os.path.splitext(urllib2.unquote(imgurl).decode('utf8').split('/')[-1])[1])).lower() print '[Debug] Download file :'+ imgurl+' >> '+filepathname urllib.urlretrieve(imgurl,filepathname) x+=1 def downImageNum(pagenum): page = 1 pageNumber = pagenum while(page <= pageNumber): html = getHtml(url)#获得url指向的html内容 imageList = getImageList(html)#获得所有图片的地址，返回列表 download(imageList,page)#下载所有的图片 page = page+1 if __name__ == '__main__': downImageNum(1) |
以上就是给大家汇总的3款Python实现的批量抓取妹纸图片的代码了，希望对大家学习Python爬虫能够有所帮助。
