谷歌地图评论数据抓取：Playwright 问题解析与Selenium方案优化

pip install selenium

from selenium import webdriver from selenium.webdriver.chrome.service import Service from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC import re import time  # 定义一个类来存储商家信息 class Business:     def __init__(self):         self.name = "N/A"         self.address = "N/A"         self.website = "N/A"         self.phone_number = "N/A"         self.category = "N/A"         self.reviews_average = None         self.reviews_count = None  # 配置WebDriver # 假设ChromeDriver在系统PATH中，或者指定路径 # service = Service(executable_path='/path/to/chromedriver') driver = webdriver.Chrome() # service=service  try:     # 1. 导航到谷歌地图搜索结果页     driver.get("https://www.google.com/maps/search/restaurants+near+me") # 示例URL      # 2. 智能等待页面加载并处理可能的Cookie同意弹窗     # 尝试查找并点击同意按钮 (根据实际页面结构调整XPath/Selector)     try:         WebDriverWait(driver, 10).until(             EC.element_to_be_clickable((By.XPATH, "//button[contains(@aria-label, 'Accept all')]"))         ).click()         print("Accepted cookies.")         time.sleep(2) # 给页面一些时间来处理     except:         print("No cookie consent popup found or already handled.")      # 3. 滚动加载更多商家列表（如果需要）     # 谷歌地图通常是无限滚动，需要模拟滚动操作     # 示例：滚动几次以加载更多列表项     scrollable_div_xpath = '//*[@id="QA0Szd"]/div/div/div[1]/div[2]/div/div[1]/div/div/div[1]/div[1]' # 示例滚动容器XPath     try:         scrollable_div = WebDriverWait(driver, 10).until(             EC.presence_of_element_located((By.XPATH, scrollable_div_xpath))         )         for _ in range(3): # 滚动3次，每次滚动到底部             driver.execute_script("arguments[0].scrollTop = arguments[0].scrollHeight", scrollable_div)             time.sleep(3) # 等待新内容加载             print(f"Scrolled {_ + 1} times.")     except Exception as e:         print(f"Could not find scrollable div or error during scroll: {e}")       # 4. 获取所有商家列表项     # 使用更稳定的css选择器或XPath，例如查找所有带有特定数据属性的列表项     # 注意：这里的XPath/CSS选择器需要根据实际的谷歌地图HTML结构来确定     # 示例：查找所有列表中的商家卡片     listing_elements_xpath = '//div[@role="article" and @aria-label]'     listings = WebDriverWait(driver, 15).until(         EC.presence_of_all_elements_located((By.XPATH, listing_elements_xpath))     )     print(f"Found {len(listings)} listings.")      scraped_businesses = []     total_scraped_reviews = 0      for index, listing in enumerate(listings):         try:             print(f"n--- Processing listing {index + 1} ---")             # 5. 点击单个商家列表项             # 确保元素是可点击的             WebDriverWait(driver, 10).until(EC.element_to_be_clickable(listing)).click()             print(f"Clicked listing {index + 1}.")              # 6. 智能等待详细信息面板加载             # 找到详细信息面板中的一个稳定元素，例如商家名称或评论区             # 注意：这里的XPath需要针对详细信息面板的结构进行调整             # 商家名称在详细信息面板中的XPath             detail_name_xpath = '//div[contains(@class, "qBF1Pd fontHeadlineSmall ")]'             WebDriverWait(driver, 15).until(                 EC.presence_of_element_located((By.XPATH, detail_name_xpath))             )             print("Detail panel loaded.")              business = Business()              # 7. 从详细信息面板中提取数据             # 这里的XPath不再依赖于原始列表的索引，而是针对当前打开的详细信息面板             try:                 business.name = driver.find_element(By.XPATH, detail_name_xpath).text             except:                 business.name = "N/A"              try:                 # 地址：data-item-id="address"                 business.address = driver.find_element(By.XPATH, '//button[@data-item-id="address"]//div[contains(@class, "fontBodyMedium")]').text             except:                 business.address = "N/A"              try:                 # 网站：data-item-id="authority"                 business.website = driver.find_element(By.XPATH, '//a[@data-item-id="authority"]//div[contains(@class, "fontBodyMedium")]').text             except:                 business.website = "N/A"              try:                 # 电话：data-item-id="phone:tel:"                 business.phone_number = driver.find_element(By.XPATH, '//button[contains(@data-item-id, "phone:tel:")]//div[contains(@class, "fontBodyMedium")]').text             except:                 business.phone_number = "N/A"              try:                 # 类别：通常在名称下方，可能有特定的类或父元素                 # 这是一个示例XPath，可能需要根据实际页面结构调整                 business.category = driver.find_element(By.XPATH, '//*[@id="QA0Szd"]/div/div/div[1]/div[3]/div/div[1]/div/div/div[2]/div[2]/div/div[1]/div[2]/div/div[2]/span/span/button').text             except:                 business.category = "N/A"              # 提取评论数据             # 评论元素通常有一个role="img"和aria-label属性包含星级和数量             # 这个XPath是针对详细信息面板中的评论元素             reviews_span_xpath = '//span[@role="img" and contains(@aria-label, "stars")]'             try:                 # 确保评论元素可见                 reviews_element = WebDriverWait(driver, 5).until(                     EC.visibility_of_element_located((By.XPATH, reviews_span_xpath))                 )                 reviews_label = reviews_element.get_attribute("aria-label")                 print(f"Reviews Label: {reviews_label}")                  match = re.match(r'([d.]+) stars ([d,]+) Reviews', reviews_label)                 if match:                     business.reviews_average = float(match.group(1))                     business.reviews_count = int(re.sub(',', '', match.group(2)))                     total_scraped_reviews += 1                 else:                     business.reviews_average = None                     business.reviews_count = None             except Exception as e:                 print(f"Could not find reviews for this listing or error processing label: {e}")                 business.reviews_average = None                 business.reviews_count = None              scraped_businesses.append(business)              # 8. 返回列表视图             # 查找并点击返回按钮，通常是一个左箭头图标             # 这个XPath也需要根据实际页面结构调整             back_button_xpath = '//button[@aria-label="Back"]'             try:                 WebDriverWait(driver, 5).until(                     EC.element_to_be_clickable((By.XPATH, back_button_xpath))                 ).click()                 print("Clicked back button.")                 time.sleep(2) # 给页面一些时间返回列表             except Exception as e:                 print(f"Could not click back button or no back button found: {e}")                 # 如果没有返回按钮，可能需要刷新页面或重新导航，但会影响效率                 # 更好的做法是确保找到并点击返回按钮          except Exception as e:             print(f"Error processing listing {index + 1}: {e}")             # 如果某个商家处理失败，尝试继续下一个             # 确保返回列表视图，以防卡在详情页             try:                 back_button_xpath = '//button[@aria-label="Back"]'                 WebDriverWait(driver, 3).until(                     EC.element_to_be_clickable((By.XPATH, back_button_xpath))                 ).click()                 time.sleep(1)             except:                 pass # 无法返回，跳过此商家      print(f"nTotal businesses scraped: {len(scraped_businesses)}")     print(f"Total reviews labels processed: {total_scraped_reviews}")      # 打印抓取结果     for b in scraped_businesses:         print(f"Name: {b.name}, Reviews Average: {b.reviews_average}, Reviews Count: {b.reviews_count}")  finally:     # 关闭浏览器     driver.quit()     print("Browser closed.")