Selenium Python在获取谷歌评论时无法向下滚动

Question

self.driver = webdriver.Chrome(executable_path="./chromedriver.exe",chrome_options=webdriver.ChromeOptions())

company_name = self.driver.find_element_by_name("q")
company_name.send_keys("STANLEY BRIDGE CYCLES AND SPORTS LIMITED ")
time.sleep(2)

self.driver.find_element_by_name("btnK").click()
time.sleep(2)

self.driver.find_elements_by_link_text("View all Google reviews")[0].click()
time.sleep(2)

即使使用上面的代码向下滚动页面，我仍然只得到10个评论。不过我没有得到任何错误。

需要帮助如何向下滚动页面，以获得至少20个评论。到目前为止，我只能得到10个评论。根据我对此问题的在线搜索，人们大多使用：“driver.execute_script（”window.scrollto(0,document.body.scrollheight）；“）”若要随时向下滚动页面，请执行以下操作。但对我来说这不起作用。我检查了页面前后的高度（“driver.execute_script（”window.scrollto(0,document.body.scrollheight）；“））是一样的。

Answer 1

使用Javascript滚动到最后一次审查，这将触发额外的审查负载。

last_review = self.driver.find_element_by_css_selector('div.gws-localreviews__google-review:last-of-type')
self.driver.execute_script('arguments[0].scrollIntoView(true);', last_review)

编辑：

下面的例子对我来说在Firefox和Chrome上是正确的，你可以根据你的需要重用提取谷歌评论功能

import time

from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait


def extract_google_reviews(driver, query):
    driver.get('https://www.google.com/?hl=en')
    driver.find_element_by_name('q').send_keys(query)
    WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.NAME, 'btnK'))).click()

    reviews_header = driver.find_element_by_css_selector('div.kp-header')
    reviews_link = reviews_header.find_element_by_partial_link_text('Google reviews')
    number_of_reviews = int(reviews_link.text.split()[0])
    reviews_link.click()

    all_reviews = WebDriverWait(driver, 3).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'div.gws-localreviews__google-review')))
    while len(all_reviews) < number_of_reviews:
        driver.execute_script('arguments[0].scrollIntoView(true);', all_reviews[-1])
        WebDriverWait(driver, 5, 0.25).until_not(EC.presence_of_element_located((By.CSS_SELECTOR, 'div[class$="activityIndicator"]')))
        all_reviews = driver.find_elements_by_css_selector('div.gws-localreviews__google-review')

    reviews = []
    for review in all_reviews:
        try:
            full_text_element = review.find_element_by_css_selector('span.review-full-text')
        except NoSuchElementException:
            full_text_element = review.find_element_by_css_selector('span[class^="r-"]')
        reviews.append(full_text_element.get_attribute('textContent'))

    return reviews

if __name__ == '__main__':
    try:
        driver = webdriver.Firefox()
        reviews = extract_google_reviews(driver, 'STANLEY BRIDGE CYCLES AND SPORTS LIMITED')
    finally:
        driver.quit()

    print(reviews)