python - 这段代码为什么不能获取到数据？

Question

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
import time

# 设置 ChromeDriver
options = webdriver.ChromeOptions()
options.add_argument('--headless')  # 如果不想看到浏览器界面，启用headless模式
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)

# 打开目标网站
url = 'https://www.us168168.com/#/houseRentingDetails?id=1860134671530127362'
driver.get(url)

# 等待动态内容加载
time.sleep(15)  # 可以根据具体的页面加载时间调整

# 查找需要的元素
content = driver.find_element(By.TAG_NAME, "body").text  # 获取页面的文本内容
print(content)

# 关闭浏览器
driver.quit()

Answer 1

前提安装这些模块
1.selenium： 用于控制浏览器。
2.webdriver_manager： 用于自动下载和管理 ChromeDriver。

目前我重新写的代码

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import logging

# 配置日志
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")

def initialize_driver():
    """初始化Chrome浏览器驱动并返回实例。"""
    options = webdriver.ChromeOptions()
    options.add_argument('--disable-gpu')
    options.add_argument('--disable-blink-features=AutomationControlled')
    options.add_experimental_option("excludeSwitches", ["enable-automation"])
    options.add_experimental_option('useAutomationExtension', False)
    options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
    return webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)

def wait_for_page_load(driver, timeout=60):
    """等待页面完全加载"""
    WebDriverWait(driver, timeout).until(
        lambda d: d.execute_script('return document.readyState') == 'complete'
    )

def scrape_content(driver, url, css_selector, timeout=20):
    """访问目标URL并抓取指定CSS选择器的内容。"""
    try:
        logging.info(f"正在访问 {url}")
        driver.get(url)

        # 等待页面加载完成
        wait_for_page_load(driver)
        logging.info(f"页面加载完成，当前页面 URL: {driver.current_url}")

        # 检查是否重定向到错误的 URL
        if driver.current_url == "data:,":
            raise Exception("页面加载失败，URL 显示为 data:,")

        # 等待目标元素加载并获取文本内容
        element = WebDriverWait(driver, timeout).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, css_selector))
        )
        return element.text
    except Exception as e:
        logging.error(f"发生错误: {e}", exc_info=True)
        return None

if __name__ == "__main__":
    driver = None
    try:
        # 初始化浏览器驱动
        driver = initialize_driver()

        # 目标URL和CSS选择器
        url = 'https://www.us168168.com/#/houseRentingDetails?id=1491098005497819137'
        css_selector = "div.title"

        # 抓取内容
        content = scrape_content(driver, url, css_selector)
        if content:
            print("抓取到的内容:", content)
        else:
            print("未能抓取到内容")

    finally:
        if driver:
            driver.quit()
            logging.info("浏览器已关闭")

运行结果
会打开一个浏览器窗口，然后正常加载页面，到获取数据那一步就空了，是不是目标网站做了防爬虫机制