有3821个链接,但他们只给了我103个链接,我也对链接应用了window.scroll
的条件,但它们都不起作用
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import ssl
import undetected_chromedriver as uc
import requests
from bs4 import BeautifulSoup
import re
ssl._create_default_https_context = ssl._create_unverified_context
# ... Rest of your code ...
options = uc.ChromeOptions()
driver = uc.Chrome(options=options)
driver.get("http://www.servicealberta.gov.ab.ca/find-if-business-is-licenced.cfm")
# Click the button to load initial content
click_on_button = driver.find_element(By.CSS_SELECTOR, "td:nth-child(1) input:nth-child(1)")
click_on_button.click()
time.sleep(2)
base_url = "http://www.servicealberta.gov.ab.ca/"
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
data = BeautifulSoup(driver.page_source, "html.parser")
links = data.select("tbody td[colspan='4'] a")
for link in links:
url = base_url + link['href']
print(url)
print(len(url))