基于python的网络爬虫系统_WebCollector-Python
WebCollector-Python
WebCollector-Python 是一个无须配置、便于二次开发的 Python 爬虫框架（内核），它提供精简的的 API，只需少量代码即可实现一个功能强大的爬虫。
WebCollector Java版本
WebCollector Java版相比WebCollector-Python具有更高的效率: https://github.com/CrawlScript/WebCollector
安装
pip安装命令
pip install https://github.com/CrawlScript/WebCollector-Python/archive/master.zip
示例
Basic
快速入门
自动探测URL
# coding=utf-8
import webcollector as wc
class NewsCrawler(wc.RamCrawler):
def __init__(self):
super().__init__(auto_detect=True)
self.num_threads = 10
self.add_seed("https://github.blog/")
self.add_regex("+https://github.blog/[0-9]+.*")
self.add_regex("-.*#.*") # do not detect urls that contain "#"
def visit(self, page, detected):
if page.match_url("https://github.blog/[0-9]+.*"):
title = page.select("h1.lh-condensed")[0].text.strip()
content = page.select("div.markdown-body")[0].text.replace("\n", " ").strip()
print("\nURL: ", page.url)
print("TITLE: ", title)
print("CONTENT: ", content[:50], "...")
crawler = NewsCrawler()
crawler.start(10)
手动探测URL
# coding=utf-8
import webcollector as wc
class NewsCrawler(wc.RamCrawler):
def __init__(self):
super().__init__(auto_detect=False)
self.num_threads = 10
self.add_seed("https://github.blog/")
def visit(self, page, detected):
detected.extend(page.links("https://github.blog/[0-9]+.*"))
if page.match_url("https://github.blog/[0-9]+.*"):
title = page.select("h1.lh-condensed")[0].text.strip()
content = page.select("div.markdown-body")[0].text.replace("\n", " ").strip()
print("\nURL: ", page.url)
print("TITLE: ", title)
print("CONTENT: ", content[:50], "...")
crawler = NewsCrawler()
crawler.start(10)
用detected_filter插件过滤探测到的URL
# coding=utf-8
import webcollector as wc
from webcollector.filter import Filter
import re
class RegexDetectedFilter(Filter):
def filter(self, crawl_datum):
if re.fullmatch("https://github.blog/2019-02.*", crawl_datum.url):
return crawl_datum
else:
print("filtered by detected_filter: {}".format(crawl_datum.brief_info()))
return None
class NewsCrawler(wc.RamCrawler):
def __init__(self):
super().__init__(auto_detect=True, detected_filter=RegexDetectedFilter())
self.num_threads = 10
self.add_seed("https://github.blog/")
def visit(self, page, detected):
detected.extend(page.links("https://github.blog/[0-9]+.*"))
if page.match_url("https://github.blog/[0-9]+.*"):
title = page.select("h1.lh-condensed")[0].text.strip()
content = page.select("div.markdown-body")[0].text.replace("\n", " ").strip()
print("\nURL: ", page.url)
print("TITLE: ", title)
print("CONTENT: ", content[:50], "...")
crawler = NewsCrawler()
crawler.start(10)
用RedisCrawler进行可断点的采集（可在关闭后恢复）
# coding=utf-8
from redis import StrictRedis
import webcollector as wc
class NewsCrawler(wc.RedisCrawler):
def __init__(self):
super().__init__(redis_client=StrictRedis("127.0.0.1"),
db_prefix="news",
auto_detect=True)
self.num_threads = 10
self.resumable = True # you can resume crawling after shutdown
self.add_seed("https://github.blog/")
self.add_regex("+https://github.blog/[0-9]+.*")
self.add_regex("-.*#.*") # do not detect urls that contain "#"
def visit(self, page, detected):
if page.match_url("https://github.blog/[0-9]+.*"):
title = page.select("h1.lh-condensed")[0].text.strip()
content = page.select("div.markdown-body")[0].text.replace("\n", " ").strip()
print("\nURL: ", page.url)
print("TITLE: ", title)
print("CONTENT: ", content[:50], "...")
crawler = NewsCrawler()
crawler.start(10)
用Requests定制Http请求
# coding=utf-8
import webcollector as wc
from webcollector.model import Page
from webcollector.plugin.net import HttpRequester
import requests
class MyRequester(HttpRequester):
def get_response(self, crawl_datum):
# custom http request
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36"
}
print("sending request with MyRequester")
# send request and get response
response = requests.get(crawl_datum.url, headers=headers)
# update code
crawl_datum.code = response.status_code
# wrap http response as a Page object
page = Page(crawl_datum,
response.content,
content_type=response.headers["Content-Type"],
http_charset=response.encoding)
return page
class NewsCrawler(wc.RamCrawler):
def __init__(self):
super().__init__(auto_detect=True)
self.num_threads = 10
# set requester to enable MyRequester
self.requester = MyRequester()
self.add_seed("https://github.blog/")
self.add_regex("+https://github.blog/[0-9]+.*")
self.add_regex("-.*#.*") # do not detect urls that contain "#"
def visit(self, page, detected):
if page.match_url("https://github.blog/[0-9]+.*"):
title = page.select("h1.lh-condensed")[0].text.strip()
content = page.select("div.markdown-body")[0].text.replace("\n", " ").strip()
print("\nURL: ", page.url)
print("TITLE: ", title)
print("CONTENT: ", content[:50], "...")
crawler = NewsCrawler()
crawler.start(10)