煎蛋网妹子图爬虫总结

这次是只用字符串查找的方式来找网页中图片链接的

 1 #!/usr/bin/python
 2 #coding:utf-8
 3 import urllib.request
 4 import os
 5 import time
 6 import random
 7 
 8 def url_open(url):    
 9     # header = {}
10     # header['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1;WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0'
11     req=urllib.request.Request(url)
12     req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1;WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0')
13 
14     proxylist=['111.192.44.204:9000','222.82.222.242:9999','124.202.247.110:8080']
15     proxy=random.choice(proxylist)
16     proxyhandler=urllib.request.ProxyHandler({'htttp':proxy})
17     opener=urllib.request.build_opener(proxyhandler)
18     urllib.request.install_opener(opener)   request模块中建立opener
19     response = urllib.request.urlopen(url)
20     html=response.read()
21     return html
22 
23 def find_img(url):
24     html=url_open(url).decode('utf-8')    从utf-8解码成unicode   反之encode（utf-8）  从unicode编码成utf-8
25     img_addrs=[]
26 
27     a=html.find('img src=')   找链接起始位置a，结束位置b，然后b-a  切片
28     while a !=-1:          
29         b=html.find('.jpg',a,a+100)
30         if b!=-1:
31             img_addrs.append('http:'+html[a+9:b+4])    将链接放到list中
32         else:
33             b=a+9
34             print('A')
35         a=html.find('img src=',b)
36     for each in img_addrs:    对list进行itrate
37         print (each)
38         return img_addrs
39 
40 def save_img(folder,img_addrs):  保存图片
41     print(img_addrs)
42     for each in img_addrs:
43         # filename=each.split('/')[-1]
44         global j
45         j += 1
46         with open(str(j)+'.jpg','wb') as f:
47             img=url_open(each)
48             f.write(img)
49 
50 def download_mm(folder="d://xx22"):   主函数
51     os.mkdir(folder)
52     os.chdir(folder)
53     global j
54     j=0
55 
56     url="http://jiandan.net/ooxx/"
57     page_num=26
58     for i in range(page_num):
59         page_num-=1
60         page_url=url+'page-'+str(page_num)+'#comments'
61         print(page_num)
62 
63         img_addrs=find_img(page_url)
64         save_img(folder,img_addrs)
65         time.sleep(1)
66 
67 if __name__=="__main__":
68     download_mm()