[FIXED] Web scraping from cardboard.market and want to pull the cheapest price value for any given search

Issue

I am developing a website using Django for sports cards and my goal is to be able to display the lowest price of any given card using Selenium with ChromeDriver. For example I am using the soccer player David Seaman from the card set 1990 Pro Set. I created a function that should go to https://cardboard.market/, search for the card, sort by the lowest price first, then pull the value of the first card in the results (this is the cheapest card). Edit: Currently, without sorting by lowest price first I am able to pull the first card price from the results. My issue is that it is not working to click the sort by lowest price button before it pulls the value.

Any ideas of what the problem could be or if there are any other ideas of accomplishing this goal please let me know!

This is what I currently have:

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

def get_lowest_card_price(card_name):
    base_url = "https://cardboard.market/"
    search_url = f"{base_url}search?q={card_name}"

    try:
        # Set up ChromeDriver
        driver_path = "C:/Webdrivers/chromedriver.exe"
        service = ChromeService(driver_path)
        driver = webdriver.Chrome(service=service)

        # Navigate to the search URL
        driver.get(search_url)

        # Wait for the sorting button to be clickable
        sorting_button = WebDriverWait(driver, 10).until(
            EC.element_to_be_clickable((By.CLASS_NAME, 'bubble-element.Button.cmgaaC'))
        )

        # Click on the sorting button
        sorting_button.click()

        # Find the target element using the provided hierarchy of tags
        page_tag = driver.find_element(By.CLASS_NAME, 'bubble-element.Page.cmfxaQ.bubble-r-container.flex.column.main-page')
        group1_tag = page_tag.find_element(By.CLASS_NAME, 'bubble-element.Group.cmfxt.bubble-r-container.flex.column')
        group2_tag = group1_tag.find_element(By.CLASS_NAME, 'bubble-element.Group.cmfxp.bubble-r-container.flex.column')
        group3_tag = group2_tag.find_element(By.CLASS_NAME, 'bubble-element.Group.cmgaCaU.bubble-r-container.flex.column')
        group4_tag = group3_tag.find_element(By.CLASS_NAME, 'bubble-element.RepeatingGroup.cmgaCaV.bubble-rg')
        group5_tag = group4_tag.find_element(By.CLASS_NAME, 'bubble-element.group-item.bubble-r-container.flex.column.entry-1')
        group6_tag = group5_tag.find_element(By.CLASS_NAME, 'bubble-element.Group.cmgaCaZ.bubble-r-container.flex.column')
        group7_tag = group6_tag.find_element(By.CLASS_NAME, 'bubble-element.Group.cmgaCm.bubble-r-container.relative')
        target_element = group7_tag.find_element(By.CLASS_NAME, 'bubble-element.Text.cmgaCs')

        # Extract the text content
        if target_element:
            price_text = target_element.text.strip()
            return price_text
        else:
            print(f"Could not find the specified element for card '{card_name}'.")
    except Exception as e:
        print(f"Error: {e}")
    finally:
        # Close the browser
        if driver:
            driver.quit()

    return None
#----------------------------------------------------------------
def ProSet1990_1(request):
    # Hardcoded card details for demonstration purposes
    card_name = "David Seaman"
    card_set = "1990 Pro Set"

    # Use the web scraping bot to get the lowest card price
    lowest_price = get_lowest_card_price(f'{card_set} {card_name}')

    # Pass the data to the template
    context = {
        'card_name': card_name,
        'lowest_price': lowest_price,
    }

    # Render the template with the data
    return render(request, 'Application/ProSet1990_1.html', context)

Solution

It looks like that the cmgaaC class of the div is dynamic and changes frequently.

Instead, try using Xpath to find a div.clickable-element whose child div contains the text "Sort". Something like //div[@class='clickable-element']//div[text()='Sort']//...

Answered By - MLpranav