由于未知错误导致页面崩溃，Python Selenium会话已被删除：无法从崩溃的选项卡确定加载状态

Question

问题可能是内存使用。页面开始变得非常慢，在某个时候出现以下错误消息

from bs4 import BeautifulSoup
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import ElementClickInterceptedException
from selenium.common.exceptions import TimeoutException
from selenium.webdriver import ActionChains


# Set some Selenium Options
options = webdriver.ChromeOptions()
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')

# Webdriver
wd = webdriver.Chrome(executable_path='/usr/bin/chromedriver', options=options)
# URL
url = 'https://www.techpilot.de/zulieferer-suchen?laserschneiden'

# Load URL
wd.get(url)

# Get HTML
soup = BeautifulSoup(wd.page_source, 'html.parser')
wd.fullscreen_window()


wait = WebDriverWait(wd, 15)
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#bodyJSP #CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll"))).click()
wait.until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR, "#efficientSearchIframe")))
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".hideFunctionalScrollbar #CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll"))).click()
#wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".fancyCompLabel")))
roaster=wd.find_element_by_xpath('//*[@id="resultTypeRaster"]')
ActionChains(wd).click(roaster).perform()

#use keys to get where the button is
html = wd.find_element_by_tag_name('html')

c=2
for i in range(100):
    html.send_keys(Keys.END)
    time.sleep(1)
    html.send_keys(Keys.END)
    time.sleep(1)
    html.send_keys(Keys.ARROW_UP)
    try:
        wait.until(EC.presence_of_all_elements_located((By.XPATH, "//*[@id='resultPane']/div["+str(c)+"]/span")))
        loadButton=wd.find_element_by_xpath("//*[@id='resultPane']/div["+str(c)+"]/span")
        loadButton.click()
    except TimeoutException or ElementClickInterceptedException:
        break
    time.sleep(1)
    c+=1
wd.close

下面是一些我浏览过的类似问题的链接，我尝试添加选项，但不起作用。其他一些技巧真的让我困惑，所以我希望有人能在这里帮助我（我对编码很陌生）

以下是我浏览的链接

未知错误：会话被删除，因为选项卡崩溃导致页面崩溃

python linux selenium:无法访问chrome

未知错误：由于未知错误导致页面崩溃，会话被删除：无法从ChromeDriver Selenium崩溃的选项卡中确定加载状态

只是为了澄清程序的目标是获取所有配置文件的列表，并从中获取内容，这就是为什么程序的这一部分首先加载整个页面以获取所有这些链接（因为javascript，我不能用bsoup获取它们），所以我没有太多解决方法！

Answer 1

解决方案是从dom树中删除元素，就像@pcalkins在dom树上面说的那样，否则“过载”

Answer 2

就像我在评论中提到的。对于初学者来说，这不是一项容易的任务。不过，这段代码应该给你一个开始。

这里最大的问题是，结果是通过iframe加载的，所以你需要先得到它。

看看这段代码，它将获得配置文件的基本信息，并将它们作为json返回。如果你需要更多的解释，请随时在评论中询问。

import re
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait


def get_profile_info(profile_url):
    # gets info of a profile page // Adjust here to get more info
    wd.get(profile_url)
    label_element = WebDriverWait(wd, 5).until(
        EC.presence_of_element_located((By.ID, "labelAddress"))
    )
    label = label_element.find_element_by_tag_name("h1").text

    street = label_element.find_element_by_css_selector(
        "span[itemprop='streetAddress']"
    ).text

    postal_code = label_element.find_element_by_css_selector(
        "span[itemprop='postalCode']"
    ).text

    city = label_element.find_element_by_css_selector(
        "span[itemprop='addressLocality']"
    ).text

    address_region = label_element.find_element_by_css_selector(
        "span[itemprop='addressRegion']"
    ).text

    country = label_element.find_element_by_css_selector(
        "span[itemprop='addressCountry']"
    ).text

    return {
        "label": label,
        "street": street,
        "postal_code": postal_code,
        "city": city,
        "address_region": address_region,
        "country": country,
    }


def get_profile_url(label_element):
    # get the url from a result element
    onlick = label_element.get_attribute("onclick")
    # some regex magic
    return re.search(r"(?<=open\(\')(.*?)(?=\')", onlick).group()


def load_more_results():
    # load more results if needed // use only on the search page!
    button_wrapper = wd.find_element_by_class_name("loadNextBtn")
    button_wrapper.find_element_by_tag_name("span").click()


#### Script starts here ####

# Set some Selenium Options
options = webdriver.ChromeOptions()
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")

# Webdriver
wd = webdriver.Chrome(options=options)
# Load URL
wd.get("https://www.techpilot.de/zulieferer-suchen?laserschneiden")


# lets first wait for the timeframe
iframe = WebDriverWait(wd, 5).until(
    EC.frame_to_be_available_and_switch_to_it("efficientSearchIframe")
)

# the result parent
result_pane = WebDriverWait(wd, 5).until(
    EC.presence_of_element_located((By.ID, "resultPane"))
)


result_elements = wd.find_elements_by_class_name("fancyCompLabel")

# lets first collect all the links visible
href_list = []
for element in result_elements:
    url = get_profile_url(element)
    href_list.append(url)

# lets collect all the data now
result = []
for href in href_list:
    result.append(get_profile_info(href))

wd.close

# lets see what we've got
print(result)