[FIXED] Accessing element in web page using Selenium with Python

Issue

I can't seem to be able to scrape the description on this page: https://www.centris.ca/fr/condo-appartement~a-louer~brossard/17307615?view=Summary&uc=6

With this selector (which I copied from Chrome's inspector), Idon't get an error but an empty string. Any idea what "by" method I could use to access it?

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium_stealth import stealth

# Selenium setup
service = Service(ChromeDriverManager().install())
options = webdriver.ChromeOptions()
options.add_argument("--headless")
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)

browser = webdriver.Chrome(service=service, options=options)
browser.set_page_load_timeout(300)

stealth(browser,
        languages=["en-US", "en"],
        vendor="Google Inc.",
        platform="Win32",
        webgl_vendor="Intel Inc.",
        renderer="Intel Iris OpenGL Engine",
        fix_hairline=True)

url = 'https://www.centris.ca/fr/condo-appartement~a-louer~brossard/17307615?view=Summary&uc=6'
browser.get(url)
browser.implicitly_wait(3)
browser.maximize_window()

# Extract necessary information
url = browser.current_url
print("URL:", url)

descr = browser.find_element(By.CSS_SELECTOR, '#overview > div.grid_3 > div.row.description-row > div.property-description.col-md-6 > div:nth-child(2)').text

print(descr)

Solution

I guess it happened because consent popup is shown on load. When consent is shown, you description element is not visible (overlapped), so you can't get it's text.

Also I suggest to use shorter css selector [itemprop=description] to locate your element.

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

url = 'https://www.centris.ca/fr/condo-appartement~a-louer~brossard/17307615?view=Summary&uc=6'
driver = webdriver.Chrome()
driver.get(url)
driver.maximize_window()
wait = WebDriverWait(driver, 10)

url = driver.current_url
print("URL:", url)

consent = wait.until(EC.presence_of_element_located((By.ID, 'didomi-notice-agree-button')))
consent.click()
wait.until(EC.invisibility_of_element(consent))

description = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, '[itemprop=description]')))

print(description.text)

Answered By - Yaroslavm