[FIXED] BeautifulSoup4 returns an empty list

Issue

I'm trying to access the website "https://veri.bet/simulator", then the "Access Betting Simulator" button, then download the sports lines infos and show them in a JSON format. However, I'm getting a blank list on the console. What am I doing wrong? I'm new to web scraping.

import requests
from bs4 import BeautifulSoup
import dataclasses
import json

@dataclasses.dataclass
class Item:
    sport_league: str = ''     # sport as we classify it, e.g. baseball, basketball, football
    event_date_utc: str = ''   # date of the event, in UTC, ISO format
    team1: str = ''            # team 1 name
    team2: str = ''            # team 2 name
    pitcher: str = ''          # optional, pitcher for baseball
    period: str = ''           # full time, 1st half, 1st quarter and so on
    line_type: str = ''        # whatever site reports as line type, e.g. moneyline, spread, over/under
    price: str = ''            # price site reports, e.g. '-133' or '+105'
    side: str = ''             # side of the bet for over/under, e.g. 'over', 'under'
    team: str = ''             # team name, for over/under bets this will be either team name or total
    spread: float = 0.0        # for handicap and over/under bets, e.g. -1.5, +2.5

def parse_betting_line(row):
    item = Item()
    item.sport_league = row.find('span', {'class': 'sport-league'}).text.strip()
    item.event_date_utc = row.find('span', {'class': 'event-date-utc'}).text.strip()
    item.team1 = row.find('span', {'class': 'team1'}).text.strip()
    item.team2 = row.find('span', {'class': 'team2'}).text.strip()
    item.pitcher = row.find('span', {'class': 'pitcher'}).text.strip()
    item.period = row.find('span', {'class': 'period'}).text.strip()
    item.line_type = row.find('span', {'class': 'line-type'}).text.strip()
    item.price = row.find('span', {'class': 'price'}).text.strip()
    item.side = row.find('span', {'class': 'side'}).text.strip()
    item.team = row.find('span', {'class': 'team'}).text.strip()
    if item.line_type in ['spread', 'over/under']:
        item.spread = float(row.find('span', {'class': 'spread'}).text.strip())
    return item

def main():
    response = requests.get('https://veri.bet/simulator')
    soup = BeautifulSoup(response.content, 'html.parser')

    betting_lines = []
    for row in soup.find_all('div', {'class': 'betting-line'}):
        betting_lines.append(parse_betting_line(row))

    print(json.dumps(betting_lines, indent=2))

if __name__ == '__main__':
    main()

Solution

site hasn't "betting-line" class, Access Betting Simulator goes to https://veri.bet/odds-picks?filter=upcoming and get data from ajax https://veri.bet/x-ajax-oddspicks?filter=upcoming&showAll=yes . So u can scrap it like that:

import requests
from bs4 import BeautifulSoup


r = requests.get('https://veri.bet/x-ajax-oddspicks?filter=upcoming&showAll=yes')
soup = BeautifulSoup(r.text, 'lxml')
games = []
for container in soup.find('table', {'id': 'odds-picks'}).find_all('div', class_='container'):
    data = []
    game_time = container.find('span', class_='badge badge-light text-wrap text-left').get_text(strip=True)
    for tr in container.find_all('span', {'class': 'text-muted'}):
        data.append(tr.get_text(strip=True).replace('\r\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t', ' '))
    if 'TOTAL' not in data[4:]:
        game = {
            'Team1': data[4],
            'Team2': data[8],
            'ML': f"{data[5]}/{data[9]}",
            'SPREAD': f"{data[6]}/{data[10]}",
            'Total': f"{data[7]}/{data[11]}",
            'Time': game_time
        }
        games.append(game)
for x in games[1::3]:
    print(x)

OUTPUT:

{'Team1': 'Dallas Cowboys', 'Team2': 'Carolina Panthers', 'ML': '-513/+397', 'SPREAD': '-10.5(-108)/+10.5(-106)', 'Total': 'O 42(-109)/U 42(-107)', 'Time': '1:00 PM ET (11/19/2023)'}
{'Team1': 'Los Angeles Chargers', 'Team2': 'Green Bay Packers', 'ML': '-175/+152', 'SPREAD': '-3(-119)/+3(+104)', 'Total': 'O 44(-102)/U 44(-114)', 'Time': '1:00 PM ET (11/19/2023)'}
{'Team1': 'New York Giants', 'Team2': 'Washington Commanders', 'ML': '+355/-451', 'SPREAD': '+10(-112)/-10(-102)', 'Total': 'O 37.5(-104)/U 37.5(-113)', 'Time': '1:00 PM ET (11/19/2023)'}
{'Team1': 'Seattle Seahawks', 'Team2': 'Los Angeles Rams', 'ML': '-122/+102', 'SPREAD': '-1(-113)/+1(-105)', 'Total': 'O 46(-108)/U 46(-110)', 'Time': '4:25 PM ET (11/19/2023)'}
{'Team1': 'Philadelphia Eagles', 'Team2': 'Kansas City Chiefs', 'ML': '+134/-152', 'SPREAD': '+2.5(+108)/-2.5(-122)', 'Total': 'O 45.5(-109)/U 45.5(-107)', 'Time': '8:15 PM ET (11/20/2023)'}
{'Team1': 'Western Michigan', 'Team2': 'Northern Illinois', 'ML': '+174/-211', 'SPREAD': '+5.5(-114)/-5.5(-104)', 'Total': 'O 55(-108)/U 55(-112)', 'Time': '7:00 PM ET'}
{'Team1': 'Boston College', 'Team2': 'Pittsburgh', 'ML': '+120/-142', 'SPREAD': '+2.5(+102)/-2.5(-120)', 'Total': 'O 47(-109)/U 47(-111)', 'Time': '7:00 PM ET (11/16/2023)'}
{'Team1': 'Coastal Carolina', 'Team2': 'Army', 'ML': '-190/+158', 'SPREAD': '-4(-110)/+4(-108)', 'Total': 'O 45(-108)/U 45(-112)', 'Time': '12:00 PM ET (11/18/2023)'}
{'Team1': 'Louisville', 'Team2': 'Miami', 'ML': '-109/-109', 'SPREAD': '-1(-103)/+1(-115)', 'Total': 'O 47(+100)/U 47(-120)', 'Time': '12:00 PM ET (11/18/2023)'}
{'Team1': 'Southern Mississippi', 'Team2': 'Mississippi State', 'ML': '+404/-579', 'SPREAD': '+13.5(-104)/-13.5(-114)', 'Total': 'O 46.5(-109)/U 46.5(-111)', 'Time': '12:00 PM ET (11/18/2023)'}
{'Team1': 'Oklahoma', 'Team2': 'BYU', 'ML': '-2632/+1136', 'SPREAD': '-24(-115)/+24(-104)', 'Total': 'O 57(-110)/U 57(-110)', 'Time': '12:00 PM ET (11/18/2023)'}
{'Team1': 'Massachusetts', 'Team2': 'Liberty', 'ML': '+1613/-5000', 'SPREAD': '+27.5(-106)/-27.5(-112)', 'Total': 'O 62(-110)/U 62(-110)', 'Time': '1:00 PM ET (11/18/2023)'}
{'Team1': 'Rice', 'Team2': 'Charlotte', 'ML': '-142/+120', 'SPREAD': '-3(-104)/+3(-114)', 'Total': 'O 47.5(-110)/U 47.5(-110)', 'Time': '2:00 PM ET (11/18/2023)'}
{'Team1': 'Hawaii', 'Team2': 'Wyoming', 'ML': '+405/-585', 'SPREAD': '+13.5(-110)/-13.5(-108)', 'Total': 'O 45(-107)/U 45(-113)', 'Time': '2:00 PM ET (11/18/2023)'}
...

Answered By - Sergey K