[FIXED] How do I resolve list index out of range error?

Issue

I have a code that scrapes data into a dataframe

import os
import re
import threading
from multiprocessing.pool import ThreadPool
import pandas as pd
from bs4 import BeautifulSoup as bs
from selenium import webdriver



class Driver:
    def __init__(self):
        options = webdriver.ChromeOptions()
        options.add_argument("--headless")
        # Un-comment next line to supress logging:
        options.add_experimental_option('excludeSwitches', ['enable-logging'])
        self.driver = webdriver.Chrome(options=options)

    def __del__(self):
        self.driver.quit()  # clean up driver when we are cleaned up
        # print('The driver has been "quitted".')


threadLocal = threading.local()


def create_driver():
    the_driver = getattr(threadLocal, 'the_driver', None)
    if the_driver is None:
        the_driver = Driver()
        setattr(threadLocal, 'the_driver', the_driver)
    return the_driver.driver


class GameData:
    def __init__(self):
        self.date = []
        self.time = []
        self.game = []
        self.score = []
        self.home_odds = []
        self.draw_odds = []
        self.away_odds = []
        self.country = []
        self.league = []


def generate_matches(table):
    tr_tags = table.findAll('tr')
    for tr_tag in tr_tags:
        if 'class' in tr_tag.attrs and 'dark' in tr_tag['class']:
            th_tag = tr_tag.find('th', {'class': 'first2 tl'})
            a_tags = th_tag.findAll('a')
            country = a_tags[0].text
            league = a_tags[1].text
        else:
            td_tags = tr_tag.findAll('td')
            if len(td_tags) > 0:  # or just if td_tags
                yield [td_tags[0].text, td_tags[1].text, td_tags[2].text, td_tags[3].text,
                       td_tags[4].text, td_tags[5].text, country, league]


def parse_data(url, return_urls=False):
    browser = create_driver()
    browser.get(url)
    soup = bs(browser.page_source, "lxml")
    div = soup.find('div', {'id': 'col-content'})
    table = div.find('table', {'class': 'table-main'})
    h1 = soup.find('h1').text
    print(h1)
    m = re.search(r'\d+ \w+ \d{4}$', h1)
    game_date = m[0]
    game_data = GameData()
    for row in generate_matches(table):
        game_data.date.append(game_date)
        game_data.time.append(row[0])
        game_data.game.append(row[1])
        # Score present?
        if ':' not in row[2]:
            # No, shift a few columns right:
            row[5], row[4], row[3], row[2] = row[4], row[3], row[2], nan
        game_data.score.append(row[2])
        game_data.home_odds.append(nan if row[3] == '-' else row[3])
        game_data.draw_odds.append(nan if row[4] == '-' else row[4])
        game_data.away_odds.append(nan if row[5] == '-' else row[5])
        game_data.country.append(row[6])
        game_data.league.append(row[7])

    if return_urls:
        span = soup.find('span', {'class': 'next-games-date'})
        a_tags = span.findAll('a')
        urls = ['https://www.oddsportal.com' + a_tag['href'] for a_tag in a_tags]
        return game_data, urls
    return game_data


if __name__ == '__main__':
    results = None
    pool = ThreadPool(5)  # We will be getting, however, 7 URLs
    # Get today's data and the Urls for the other days:
    game_data_today, urls = pool.apply(parse_data, args=('https://www.oddsportal.com/matches/soccer', True))
    urls.pop(1)  # Remove url for today: We already have the data for that
    game_data_results = pool.imap(parse_data, urls)
    for i in range(8):
        try:
            game_data = game_data_today if i == 1 else next(game_data_results)
            result = pd.DataFrame(game_data.__dict__)
            if results is None:
                results = result
            else:
                results = results.append(result, ignore_index=True)
    print(results)
    # ensure all the drivers are "quitted":
    del threadLocal
    import gc

    gc.collect()  # a little extra insurance

However I get partial output at:

h1 = soup.find('h1').text
print(h1)

Next Soccer Matches: Today, 18 Dec 2021
Next Soccer Matches: Wednesday, 22 Dec 2021
Next Soccer Matches: Thursday, 23 Dec 2021
Next Soccer Matches: Friday, 24 Dec 2021
Next Soccer Matches: Tuesday, 21 Dec 2021
Next Soccer Matches: Monday, 20 Dec 2021
Next Soccer Matches: Yesterday, 17 Dec 2021
Next Soccer Matches: Tomorrow, 19 Dec 2021

While I checked, for the argument

When I explore further

        td_tags = tr_tag.findAll('td')
        print(td_tags)
        if len(td_tags) > 0:  # or just if td_tags
            print(len(td_tags))

I get 2 cases

[<td class="table-time datet t1639839600-1-1-0-0">15:00</td>, <td class="name table-participant"><a href="/soccer/england/npl-premier-division/south-shields-witton-albion-COXLb3wr/">South Shields - Witton</a><span class="ico-event-info" onmouseout="allowHideTootip(true);delayHideTip(200);" onmouseover="toolTip('Postponed due to Covid-19.', this, event, '4');allowHideTootip(false);delayHideTip(200);return false;"> </span></td>, <td class="center bold table-odds table-score">postp.</td>, <td class="odds-nowrp" xodd="1.36" xoid="E-4v5i6xv464x0xd4ur7"><a href="" onclick="globals.ch.togle(this , 'E-4v5i6xv464x0xd4ur7');return false;" xparam="odds_text">1.36</a></td>, <td class="odds-nowrp" xodd="4.84" xoid="E-4v5i6xv498x0x0"><a href="" onclick="globals.ch.togle(this , 'E-4v5i6xv498x0x0');return false;" xparam="odds_text">4.84</a></td>, <td class="odds-nowrp" xodd="7.05" xoid="E-4v5i6xv464x0xd4ur8"><a href="" onclick="globals.ch.togle(this , 'E-4v5i6xv464x0xd4ur8');return false;" xparam="odds_text">7.05</a></td>, <td class="center info-value">11</td>]
7
[<td class="table-time datet t1639839600-1-1-0-0">15:00</td>, <td class="name table-participant" colspan="2"><a href="/soccer/england/npl-premier-division/warrington-town-scarborough-athletic-ObyQcNhl/">Warrington - Scarborough</a></td>, <td class="odds-nowrp" xodd="1.8" xoid="E-4v5i7xv464x0xd4ur9"><a href="" onclick="globals.ch.togle(this , 'E-4v5i7xv464x0xd4ur9');return false;" xparam="odds_text">1.80</a></td>, <td class="odds-nowrp" xodd="3.59" xoid="E-4v5i7xv498x0x0"><a href="" onclick="globals.ch.togle(this , 'E-4v5i7xv498x0x0');return false;" xparam="odds_text">3.59</a></td>, <td class="odds-nowrp" xodd="3.91" xoid="E-4v5i7xv464x0xd4ura"><a href="" onclick="globals.ch.togle(this , 'E-4v5i7xv464x0xd4ura');return false;" xparam="odds_text">3.91</a></td>, <td class="center info-value">12</td>]
6

I am getting an IndexError at:

line 67, in generate_matches
    yield [td_tags[0].text, td_tags[1].text, td_tags[2].text, td_tags[3].text,
IndexError: list index out of range

How can I resolve this?

Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque pellentesque, ipsum vel tempor suscipit, turpis mauris venenatis leo, nec vestibulum arcu urna et quam

Solution

The fact that you get an IndexError when accessing td_tags implies that len(td_tags) is not always 6 or 7. It is <6 at least once.

Look at this bit of code:

if len(td_tags) > 0:  # or just if td_tags
    yield [td_tags[0].text, td_tags[1].text, td_tags[2].text, td_tags[3].text,
           td_tags[4].text, td_tags[5].text, country, league]

You are using list indices 0 up to and including 5. To not get an IndexError requires len(td_tags) of at least 6! So change the first line to:

if len(td_tags) > 5:

That should get rid of the IndexError.

Answered By - Roland Smith