1、创建Scrapy项目
scapy startproject dongguan2
2.进入项目目录,使用命令genspider创建Spider
scrapy genspider xixi "wz.sun0769.com"
3、定义要抓取的数据(处理items.py文件)
# -*- coding: utf-8 -*-
import scrapy
class Dongguan2Item(scrapy.Item):
# 贴子编号
number = scrapy.Field()
# 贴子标题
title = scrapy.Field()
# 贴子内容
content = scrapy.Field()
# 贴子url
url = scrapy.Field()
# 有图贴子的链接地址
image_link = scrapy.Field()
4、编写提取item数据的Spider(在spiders文件夹下:xixi.py)
# -*- coding: utf-8 -*-
import scrapy
import re
# 如果下面在pycharm中有红色波浪线,参照这个设置:https://blog.csdn.net/z564359805/article/details/80650843
from dongguan2.items import Dongguan2Item
class XixiSpider(scrapy.Spider):
name = 'xixi'
allowed_domains = ['wz.sun0769.com']
url = "http://wz.sun0769.com/index.php/question/report?page="
offset = 0
start_urls = [url + str(offset)]
def parse(self, response):
# 获取贴子的链接地址列表
links = response.xpath('//a[@class="news14"]/@href').extract()
for link in links:
yield scrapy.Request(link,callback=self.parse_item)
# 获取到最后一页的链接,记得是列表要加[0]
end_link = response.xpath('//div[@class="pagination"]/a[last()]/@href').extract()[0]
# 利用正则取出最后的数字也就是offset的最大值
offset_max = re.search(r'page=(\d+)', end_link).group(1)
if self.offset <= int(offset_max):
self.offset += 30
yield scrapy.Request(self.url + str(self.offset),callback=self.parse)
def parse_item(self,response):
item = Dongguan2Item()
# 每个贴子中的标题,strip()去除首尾空格
item['title'] = response.xpath('//div[@class="pagecenter p3"]//strong/text()').extract()[0].strip()
# 根据标题提取出编号
item['number'] = item['title'].split(":")[-1].strip()
# 先取出贴子内容有图的情况
content = response.xpath('//div[@class="contentext"]/text()').extract()
if len(content) == 0:
# 代表没有图
content = response.xpath('//div[@class="c1 text14_2"]/text()').extract()
# content为列表,通过join方法拼接为字符串,并去除首尾空格,以及将内容中的空格去掉
item['content'] = "".join(content).strip().replace("\xa0", "")
# 如果贴子没有图要给item['image_link']赋默认值,否则写入的时候会报NoneType错
item['image_link'] = "此贴无图"
else:
# 贴子有图情况,将图的链接地址取出来放在item['image_link']
item['image_link'] = response.xpath('//div[@class="textpic"]/img/@src').extract()[0]
# content为列表,通过join方法拼接为字符串,并去除首尾空格,以及将内容中的空格去掉
item['content'] = "".join(content).strip().replace("\xa0", "")
# 贴子的链接地址
item['url'] = response.url
yield item
5.处理pipelines管道文件保存数据,可将结果保存到文件中(pipelines.py)
# -*- coding: utf-8 -*-
import scrapy
import json
import os,os.path
from scrapy.utils.project import get_project_settings
from scrapy.pipelines.images import ImagesPipeline
# 继承ImagesPipeline()的子类处理图片并保存,参考:https://blog.csdn.net/z564359805/article/details/80693578
class ImagePipeline(ImagesPipeline):
# 获取settings文件中设置的图片保存地址IMAGES_STORE
IMAGES_STORE =get_project_settings().get("IMAGES_STORE")
def get_media_requests(self, item, info):
# 如果贴子有图,处理贴子图片地址
if item['image_link'] != '此贴无图':
image_url = item['image_link']
full_image_url = "http://wz.sun0769.com"+str(image_url)
yield scrapy.Request(full_image_url)
def item_completed(self, results, item, info):
if item['image_link'] != '此贴无图':
image_url = item['image_link']
# 拼接成图片链接地址并打印提示
full_image_url = "http://wz.sun0769.com" + str(image_url)
print('\n' + "此贴子有图片,链接地址为:%s" % full_image_url)
image_path = [x['path'] for ok, x in results if ok]
os.rename(self.IMAGES_STORE + '/' + image_path[0],self.IMAGES_STORE + '/' + item['number'] + '.jpg')
# 如果贴子有图,修改item['image_link']为本地图片地址,这里不修改注释掉了
# item['image_link'] = self.IMAGES_STORE + '/' + item['number']
return item
# 转码操作,继承json.JSONEncoder的子类,在json目录下的encoder.py中
class MyEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o,bytes):
return str(o,encoding='utf-8')
return JSONEncoder.default(self, o)
class Dongguan2Pipeline(object):
def __init__(self):
self.file = open("dongguan.json",'w',encoding='utf-8')
def process_item(self, item, spider):
text = json.dumps(dict(item),ensure_ascii=False,cls=MyEncoder)+ '\n'
self.file.write(text)
return item
def close_spider(self,spider):
print("数据处理完毕,谢谢使用!")
self.file.close()
6.配置settings文件(settings.py)
# Obey robots.txt rules,具体含义参照:https://blog.csdn.net/z564359805/article/details/80691677
ROBOTSTXT_OBEY = False
# Override the default request headers:添加User-Agent信息
DEFAULT_REQUEST_HEADERS = {
'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0);',
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
}
# 图片保存地址,这样会在当前执行的目录下创建images文件夹,也可以写具体地址
IMAGES_STORE = "./images"
# Configure item pipelines
ITEM_PIPELINES = {
'dongguan2.pipelines.Dongguan2Pipeline': 300,
'dongguan2.pipelines.ImagePipeline': 3,
}
# 还可以将日志存到本地文件中(可选添加设置)
LOG_FILE = "dongguanlog.log"
LOG_LEVEL = "DEBUG"
7.以上设置完毕,进行爬取:执行项目命令crawl,启动Spider:
scrapy crawl xixi