web抓取webdriver异常时出错

Question

!pip install selenium
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from time import sleep
from datetime import datetime
import pandas as pd
errors = []
season = []

for id in range(46605, 46985):
my_url = f'https://www.premierleague.com/match/{id}'
option = Options()
#option.headless = True
driver = webdriver.Chrome(options=option)
driver.get(my_url)

代码在这里运行正常。

date = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '//*[@id="mainContent"]/div/section/div[2]/section/div[1]/div/div[1]/div[1]'))).text
date = datetime.strptime(date, '%a %d %b %Y').strftime('%m/%d/%Y')
home_team = driver.find_element_by_xpath('//*[@id="mainContent"]/div/section/div[2]/section/div[3]/div/div/div[1]/div[1]/a[2]/span[1]').text
away_team = driver.find_element_by_xpath('//*[@id="mainContent"]/div/section/div[2]/section/div[3]/div/div/div[1]/div[3]/a[2]/span[1]').text

执行这些行时弹出错误。错误截图1

错误截图2

Answer 1

@Kaustav在回答时说，这是一个代码块，不知道如何把它放在评论部分。虽然我赞同@Joe的想法，但因为你想要一个精确的语法，我想我可以把一段代码放在一起展示给你。因此，这里的代码打开浏览器，获取详细信息（当您输入代码时），然后将其存储在列表中（用于显示此任务，但实际上可能不是必需的，这取决于您的要求）。

我要再次重申，打开、使用和关闭浏览器（即使是无头浏览器）进行如此多次的迭代，可能会在循环中的某个时间点影响性能，所有的时间都会被浪费。我强烈建议你找到本网站的API（如果有的话），并使用它成功。

话虽如此，下面是供您使用的代码。

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from time import sleep
from datetime import datetime
import pandas as p
import time
errors = []
season = []

url_ls=[] # used to show to query creator a demo
for id in range(46605, 46985):
    my_url = f'https://www.premierleague.com/match/{id}'
    option = Options()
    option.add_argument('--headless')
    option.add_argument('--disable-gpu')
    driver = webdriver.Chrome(options=option)
    driver.get(my_url)
    title = driver.current_url
    try:
        WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.XPATH, "//*[text()='Accept All Cookies']"))).click()
    except:
        print("cookie modal not found")
        continue
    # time.sleep(10)
    match_date = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[@class='matchInfo']//div[contains(@class, 'matchDate')]"))).text
    match_date_f = datetime.strptime(match_date, '%a %d %b %Y').strftime('%m/%d/%Y')
    home_team = driver.find_element(By.XPATH, "//div[@class='team home']").text
    away_team = driver.find_element(By.XPATH, "//div[@class='team away']").text
    tup = (title + "|" + match_date + "|" + home_team + "|" + away_team)
    url_ls.append(tup)
    driver.close()
print(url_ls)  # used to show to query creator a demo

这是输出（我只是附加了当前的url，以及这个代码中元素的文本，但是你可以进一步扩展这个块并添加你的元素——你可以像导入熊猫一样，将数据发送到数据框，然后发送到excel或csv——随你喜欢）

['https://www.premierleague.com/match/46605|Sat 10 Aug 2019|Liverpool|Norwich City', 'https://www.premierleague.com/match/46606|Sat 10 Aug 2019|AFC Bournemouth|Sheffield United', 'https://www.premierleague.com/match/46607|Sat 10 Aug 2019|Burnley|Southampton', 'https://www.premierleague.com/match/46608|Sat 10 Aug 2019|Crystal Palace|Everton', 'https://www.premierleague.com/match/46609|Sun 11 Aug 2019|Leicester City|Wolverhampton Wanderers']

Process finished with exit code 0

Answer 2

为什么不使用英超联赛网站使用的api？

import requests

fixture = 66553

headers =   {
    'accept':'*/*',
    'accept-encoding':"gzip;q=1.0, identity; q=0.5",
    'accept-language':'en-ZA,en;q=0.9,en-GB;q=0.8,en-US;q=0.7,de;q=0.6',
    'content-type':'application/x-www-form-urlencoded; charset=UTF-8',
    'origin':'https://www.premierleague.com',
    'referer':'https://www.premierleague.com/',
    'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'
    }

url = f'https://footballapi.pulselive.com/football/broadcasting-schedule/fixtures/{fixture}'
data = requests.get(url,headers=headers).json()

print(data['fixture']['attendance'])
print(data['fixture']['kickoff']['label'])
print(data['fixture']['teams'])

url = f'https://footballapi.pulselive.com/football/fixtures/{fixture}/textstream/EN?pageSize=1000&sort=desc'
data = requests.get(url,headers=headers).json()

for message in data['events']['content']:
    print(message['text'])