下载糗事百科的内容_python版

1.  #coding:utf-8
3. import urllib.request
4. import xml.dom.minidom
5. import sqlite3
6. import threading
7. import time
9. class logger(object):
10.     def log(self,*msg):
11.         for i in msg:
12.             print(i)
14. Log = logger()
15. Log.log('测试下')
17. class downloader(object):
18.     
19.     def __init__(self,url):
20.         self.url = url
21.     
22.     def download(self):
23.         Log.log('开始下载',self.url)
24.         try:
25.             content = urllib.request.urlopen(self.url).read()
26.             #req = urllib.request.Request(url)
27.             #response = urllib.request.urlopen(req)
28.             #content = response.read()
29.             Log.log('下载完毕')
30.             return(content)   
31.         except:
32.             Log.log('下载出错')
33.             return(None)
34.        
35.             
36. class parser(object):
37.     
38.     def __init__(self,content):
39.         #获得根节点
40.         self.html = xml.dom.minidom.parseString(content)
41.     
42.     def parse(self):
43.         Log.log('开始提取数据')
44.         contents = {'content':'','url':[]}
45.         #获得div节点
46.         divs = self.html.getElementsByTagName('div')
47.         #获得content节点
48.         for div in divs:
49.             if div.hasAttribute('class') and /
50.                div.getAttribute('class') == 'content':
51.                #获得糗事百科的内容
52.                textNode = div.childNodes[0]
53.                qContent = textNode.data
54.                #数据填充
55.                contents['content'] = qContent
56.         
57.         #获得上一糗事、下一糗事节点
58.         spans = self.html.getElementsByTagName('span')
59.         for span in spans:
60.             pspan = span.parentNode
61.             if pspan.tagName == 'a':
62.                 #pspan为对应的链接,此时需要将对应的地址加入数据库
63.                 url = pspan.getAttribute('href')
64.                 qid = url[10:][:-4]
65.                 #数据填充
66.                 contents['url'].append(qid)
67.         Log.log('提取数据完毕')
68.         return(contents)
70. def downloadPage(qid,db):
71.     url = 'http://www.qiushibaike.com/articles/'+str(qid)+'.htm'
72.     content = downloader(url).download()
73.     if content:
74.         contents = parser(content).parse()
75.         if contents['content']:
76.             db.updateContent(qid,contents['content'])
77.         for i in contents['url']:
78.             db.addQID(i)
79.         if len(contents['url']) == 2:
80.             db.updateStatus(qid,2)
82. #下载池，表示同时允许下载的链接个数
83. class downloaderPool(object):
84.     def __init__(self,maxLength=15):
85.         self.downloaders = [None]*maxLength
86.         self.downloadList = []
87.         self.db = None
88.     
89.     def setDownloadList(self,downloadList):
90.         self.downloadList = list(set(self.downloadList+downloadList))
91.     
92.     def setdb(self,db):
93.         self.db = db
94.     
95.     def daemon(self):
96.         #每隔一秒查询线程的状态，为非活动线程则设置为None
97.         Log.log('设置守护进程')
98.         for index,downloader in enumerate(self.downloaders):
99.             if downloader:
100.                 if not downloader.isAlive():
101.                     Log.log('将下载器置空',index)
102.                     self.downloaders[index] = None
103.         
104.         #检查线程池状态
105.         for index,downloader in enumerate(self.downloaders):
106.             if not downloader:
107.                 qid = self.getQID()
108.                 if qid:
109.                     #创建线程
110.                     t = threading.Thread(target=downloadPage,args=(qid,self.db))   
111.                     self.downloaders[index] = t
112.                     t.start()
113.                     t.join()
114.                     Log.log('设置下载器',index)
115.         #间隔一秒执行一次
116.         time.sleep(1)            
117.     
118.     def getQID(self):
119.         try:
120.             tmp = self.downloadList[0]
121.             del self.downloadList[0]
122.             return(tmp)
123.         except:
124.             return(None)
125.         
126.     def beginDownload(self):
127.         #创建守护线程
128.         daemon = threading.Thread(target=self.daemon)
129.         daemon.setDaemon(True)
130.         daemon.start()
131.         daemon.join()          
132.                     
133.     def getDownloader(self):
134.         for index,downloader in enumerate(self.downloaders):
135.             if not downloader:
136.                 return(index)
137.         return(None)
140. ADD_Q_ID = 'insert into qiushibaike(id,success) values(?,?)'
141. UPDATE_Q_CONTENT = 'update qiushibaike set content=? where id=?'
142. UPDATE_Q_STATUS = 'update qiushibaike set success=? where id=?'
143. Q_LIST = 'select id from qiushibaike where success=?'
144. Q_LIST_BY_ID = 'select count(*) from qiushibaike where id=?'
145. class dbConnect(object):
146.     """
147.     create table qiushibaike(
148.     id,Integer
149.     content,Varchar
150.     success,Interger
151.     )
152.     #id表示糗事的ID
153.     #content表示糗事的内容
154.     #success表示是否下载成功，当该糗事内容下载完成，且获得上一页、下一页ID时表示下载完成
155.             1表示未完成
156.             2表示完成
157.     """
158.     def __init__(self,dbpath='db.sqlite'):
159.         self.dbpath = dbpath
160.     
161.     def addQID(self,qid):
162.         Log.log('插入糗事百科',qid)
163.         #获得连接
164.         cn = sqlite3.connect(self.dbpath)
165.         c = cn.cursor()
166.         
167.         try:
168.             #添加内容并提交
169.             c.execute(ADD_Q_ID,(qid,1))
170.             cn.commit()
171.         except:
172.             Log.log('添加ID出错',qid)
173.         
174.         #关闭连接            
175.         c.close()
176.             
177.         cn.close()
178.         Log.log('插入成功')
179.     
180.     def updateContent(self,qid,content):
181.         Log.log('更新糗事百科',qid,content)
182.         #获得连接
183.         cn = sqlite3.connect(self.dbpath)
184.         c = cn.cursor()
185.         #添加内容并提交
186.         c.execute(UPDATE_Q_CONTENT,(content,qid))
187.         cn.commit()
188.         #关闭连接
189.         c.close()
190.         cn.close()
191.         Log.log('更新成功')
192.     
193.     def updateStatus(self,qid,flag):
194.         Log.log('更新状态',qid,flag)
195.         #获得连接
196.         cn = sqlite3.connect(self.dbpath)
197.         c = cn.cursor()
198.         #添加内容并提交
199.         c.execute(UPDATE_Q_STATUS,(flag,qid))
200.         cn.commit()
201.         #关闭连接
202.         c.close()
203.         cn.close()
204.         Log.log('更新状态成功')
205.     
206.     def getList(self,unDonloaded=1):
207.         Log.log('获得列表')
208.         l = []
209.         #获得连接
210.         cn = sqlite3.connect(self.dbpath)
211.         c = cn.cursor()
212.         #获得数据
213.         c.execute(Q_LIST,(unDonloaded,))
214.         rows = c.fetchall()
215.         
216.         for i in rows:
217.             l.append(i[0])
218.         #关闭连接
219.         c.close()
220.         cn.close()
221.         
222.         Log.log('获得列表成功')
223.         return(l)
225. class singleDownloader(object):
226.     def __init__(self):
227.         self.downloadList = []
228.     
229.     def setdb(self,db):
230.         self.db = db
231.     
232.     def setDownloadList(self,downloadList):
233.         self.downloadList = list(set(self.downloadList+downloadList))
234.         
235.     def beginDownload(self):
236.         for i in self.downloadList:
237.             downloadPage(i,self.db)
238.     
239. def main():
240.     db = dbConnect('db.sqlite')
241.     #dp = downloaderPool()
242.     #dp.setdb(db)
243.     sp = singleDownloader()
244.     sp.setdb(db)
245.     
246.     dp=sp
247.     
248.     unDownloadedList = db.getList()
249.     #当还有未下载的糗事时就要继续下载
250.     while(len(unDownloadedList)):
251.         #使用该列表填充下载池
252.         dp.setDownloadList(unDownloadedList)
253.         
254.         dp.beginDownload()
255.         
256.         time.sleep(1)
257.         #重置参数
258.         unDownloadedList = db.getList()
260. if __name__ == '__main__':
261.     main()

代码是没问题的，可以正常运行，但是希望做到以下2方面：
1、多线程下载
2、代码分离度更高，跟面向对象

各位看家有什么好想法，贴出来看看。