python3.x - 怎么解决python 爬虫运行多进程报错:TypeError: cannot pickle '_thread.lock' object?

Question

python 爬虫运行多进程报错:TypeError: cannot pickle '_thread.lock' object

# coding=utf-8"""    @project: 15python_spider    @Author：frank    @file： 01_xiaomi_app.py    @date：2024/3/7 19:52"""import jsonimport timefrom multiprocessing import Processfrom queue import Queueimport requestsclass XiaomiSpider(object):    def __init__(self):        self.url = 'http://app.mi.com/categotyAllListApi?page={}&categoryId=2&pageSize=30'        self.headers = {'User-Agent': 'Mozilla/5.0'}        # url队列        self.url_queue = Queue()        self.n = 0        self.app_list = []    # URL入队列    def url_in(self):        for i in range(6):            url = self.url.format(i)            # 入队列            self.url_queue.put(url)    # 线程事件函数    def get_data(self):        while True:            # self.url_queue.empty() 为空，则退出执行            if self.url_queue.empty():                break            # get地址，请求+解析+保存            url = self.url_queue.get()            html = requests.get(                url=url,                headers=self.headers            ).content.decode('utf-8')            html = json.loads(html)            # 解析数据            for app in html['data']:                # 应用名称                app_name = app['displayName']                app_link = 'https://app.mi.com/details?id={}'.format(app['packageName'])                app_info = {                    'app_name': app_name,                    'app_link': app_link                }                self.app_list.append(app_info)                self.n += 1            print(url)    # 主函数    def main(self):        # url 入队列        self.url_in()        t_list = []        for i in range(5):            t = Process(target=self.get_data)            t_list.append(t)            t.start()        for i in t_list:            i.join()        with open('app_list.json', 'w') as f:            json.dump(self.app_list, f, ensure_ascii=False)        print('应用数量:', self.n)if __name__ == "__main__":    start = time.time()    spider = XiaomiSpider()    spider.main()    end = time.time()    print('执行时间:%.2f' % (end - start))

怎么解决python 爬虫运行多进程报错:TypeError: cannot pickle '_thread.lock' object

Answer 1

你需要 from multiprocessing import Queue