用漂亮的汤刮多页

Question

我已经获得了刮取第一页的代码，但是url从：

https://www.expansion.com/empresas-de/ganaderia/granjas-en-general/index.html --

import requests<br>
from bs4 import BeautifulSoup<br>
url = "https://www.expansion.com/empresas-de/ganaderia/granjas-en-general/index.html"<br>
page = requests.get(url)<br>
soup = BeautifulSoup(page.content, "html.parser")<br>
lists = soup.select("div#simulacion_tabla ul")<br>

for lis in lists:<br>
    title = lis.find('li', class_="col1").text<br>
    location = lis.find('li', class_="col2").text<br>
    province = lis.find('li', class_="col3").text<br>
    info = [title, location, province]<br>

如何创建从第2页到第65页的循环？非常感谢！

Answer 1

首先，请确保以正确的方式格式化代码，以便每个人都能阅读。在这里你可以找到更多。

这可能是一个潜在的解决方案。远不是优化代码，但您可以从中获得一些灵感。

import requests
from bs4 import BeautifulSoup

def scrape_page(url):
    """ Scrape the give url and return the bs4 ResultSet """
    page = requests.get(url)
    soup = BeautifulSoup(page.content, "html.parser")
    table = soup.select("div#simulacion_tabla ul")
    print(type(table))
    return table

def extract_rows(table):
    """ Extract rows """
    rows = []
    for row in table:
        title = row.find('li', class_="col1").text
        location = row.find('li', class_="col2").text
        province = row.find('li', class_="col3").text
        rows.append([title, location, province])
    return rows
        
big_table = []
index = scrape_page("https://www.expansion.com/empresas-de/ganaderia/granjas-en-general/index.html")
for row in extract_rows(index):
    big_table.append(row)

for x in range(2, 66):
    index = scrape_page("https://www.expansion.com/empresas-de/ganaderia/granjas-en-general/" + str(x) + ".html")
    for row in extract_rows(index):
        big_table.append(row)

print(big_table)

Answer 2

以下是工作解决方案：

import requests
from bs4 import BeautifulSoup
for page in range(1,65):
    url = "https://www.expansion.com/empresas-de/ganaderia/granjas-en-general/{page}.html".format(page =page)
    #print(url)
    page = requests.get(url)
    soup = BeautifulSoup(page.content, "html.parser")
    lists = soup.select("div#simulacion_tabla ul")

    for lis in lists:
        title = lis.find('li', class_="col1").text
        location = lis.find('li', class_="col2").text
        province = lis.find('li', class_="col3").text
        info = [title, location, province]
        print(info)