Python Selenium挂在某些页面上

Question

当我试图刮下面的url时，selenium在大约30秒内没有响应，然后在带有美丽汤的行上发出错误，因为没有html代码要解析。我的selenium+chrome设置对大多数网站都很好，但这个不是:http://www.11st.co.kr/product/sellerproductdetail.tmall？method=getsellerproductdetail&prdno=2609814501&trtypecd=22&trctgrno=895019

def GetSelenium():
    options = webdriver.ChromeOptions()
    options.add_argument("start-maximized")
    options.add_argument("--disable-infobars")
    options.add_argument('headless')
    options.add_argument('lang=ko_KR')
    options.add_argument('--no-sandbox')
    options.add_argument('--disable-dev-shm-usage')
    options.add_argument('user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36')
    
    driver = webdriver.Chrome('/usr/bin/chromedriver',chrome_options=options)
    return driver

@api_view()
def crawlOthers(request, crawl_url):
    crawl_url = request.data["requestUrl"]
    print("URL:" + crawl_url)
    driver = GetSelenium()
    driver.get(crawl_url)
    html = driver.page_source
    driver.quit()
    bsObj = BeautifulSoup(html,'html.parser')

    #parsing deleted

Answer 1

我认为GetSelenium（）函数是错误的。我认为我们应该声明选项值并将内容放入chrome驱动程序并返回chrome驱动程序值。

我将在下面附上代码。

from selenium import webdriver
from bs4 import BeautifulSoup


def GetSelenium():
    options = webdriver.ChromeOptions()
    options.add_argument("start-maximized")
    options.add_argument("--disable-infobars")
    options.add_argument('headless')
    options.add_argument('lang=ko_KR')
    options.add_argument('--no-sandbox')
    options.add_argument('--disable-dev-shm-usage')
    options.add_argument('user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36')
    driver = webdriver.Chrome(chrome_options=options)

    return driver

def crawlOthers():
    url = 'http://www.11st.co.kr/product/SellerProductDetail.tmall?method=getSellerProductDetail&prdNo=2609814501&trTypeCd=22&trCtgrNo=895019'
    print("URL:" + url)
    driver = GetSelenium()
    driver.get(url)
    html = driver.page_source
    driver.quit()
    bsObj = BeautifulSoup(html,'html.parser')
    print(bsObj)

    #parsing deleted

if __name__ == '__main__':
    crawlOthers()