[FIXED] what is NavigableString in this error refers to, and why that happened?

Issue

from bs4 import BeautifulSoup
import numpy as np
import requests 
from selenium import webdriver
from nltk.tokenize import sent_tokenize,word_tokenize

html = webdriver.Firefox(executable_path=r'D:\geckodriver.exe')
html.get("https://www.tsa.gov/coronavirus/passenger-throughput")

def TSA_travel_numbers(html):

    soup = BeautifulSoup(html,'lxml')
    for i,rows in enumerate(soup.find('div',class_='view-content'),1):
        # print(rows.content)
        for header in rows.find('tr'):
    

            number = rows.find_all('td',class_='views-field views-field field-2021-throughput views-align-center')

            print(number.text)


TSA_travel_numbers(html.page_source)

My error as follows :

Traceback (most recent call last):
  File "TSA_travel.py", line 23, in <module>
    TSA_travel_numbers(html.page_source)
  File "TSA_travel.py", line 15, in TSA_travel_numbers
    for header in rows.find('tr'):
TypeError: 'int' object is not iterable

What is happening here? I can't iter thru 'tr' tags, please help me to solve this problem. Sorry for your time and advance thanks!

Solution

As the error says, you can't iterate over an int, which is your rows.

Also, there's no need for a webdriver as data on the page is static.

Here's my take on it:

import requests
from bs4 import BeautifulSoup
from tabulate import tabulate


def get_page(url):
    return requests.get(url).text


def get_data(page):
    soup = BeautifulSoup(page, 'lxml')
    return [
        item.getText(strip=True) for item in soup.select(".views-align-center")
    ]


def build_table(table_rows):
    t = [table_rows[i:i + 4] for i in range(0, len(table_rows[1:]), 4)]
    h = t[0]
    return t[1:], h


if __name__ == '__main__':
    source = "https://www.tsa.gov/coronavirus/passenger-throughput"
    table, header = build_table(get_data(get_page(source)))
    print(tabulate(table, headers=header, tablefmt="pretty"))

Output:

+------------+--------------------------+--------------------------+--------------------------+
|    Date    | 2021 Traveler Throughput | 2020 Traveler Throughput | 2019 Traveler Throughput |
+------------+--------------------------+--------------------------+--------------------------+
|  5/9/2021  |        1,707,805         |         200,815          |        2,419,114         |
|  5/8/2021  |        1,429,657         |         169,580          |        1,985,942         |
|  5/7/2021  |        1,703,267         |         215,444          |        2,602,631         |
|  5/6/2021  |        1,644,050         |         190,863          |        2,555,342         |
|  5/5/2021  |        1,268,938         |         140,409          |        2,270,662         |
|  5/4/2021  |        1,134,103         |         130,601          |        2,106,597         |
|  5/3/2021  |        1,463,672         |         163,692          |        2,470,969         |
|  5/2/2021  |        1,626,962         |         170,254          |        2,512,598         |
|  5/1/2021  |        1,335,535         |         134,261          |        1,968,278         |
| 4/30/2021  |        1,558,553         |         171,563          |        2,546,029         |
| 4/29/2021  |        1,526,681         |         154,695          |        2,499,461         |
| 4/28/2021  |        1,184,326         |         119,629          |        2,256,442         |
| 4/27/2021  |        1,077,199         |         110,913          |        2,102,068         |
| 4/26/2021  |        1,369,410         |         119,854          |        2,412,770         |
| 4/25/2021  |        1,571,220         |         128,875          |        2,506,809         |
| 4/24/2021  |        1,259,724         |         114,459          |        1,990,464         |
| 4/23/2021  |        1,521,393         |         123,464          |        2,521,897         |
| 4/22/2021  |        1,509,649         |         111,627          |        2,526,961         |
| 4/21/2021  |        1,164,099         |          98,968          |        2,254,209         |
| 4/20/2021  |        1,082,443         |          92,859          |        2,227,475         |
| 4/19/2021  |        1,412,500         |          99,344          |        2,594,171         |
| 4/18/2021  |        1,572,383         |         105,382          |        2,356,802         |
| 4/17/2021  |        1,277,815         |          97,236          |        1,988,205         |
| 4/16/2021  |        1,468,218         |         106,385          |        2,457,133         |
| 4/15/2021  |        1,491,435         |          95,085          |        2,616,158         |
| 4/14/2021  |        1,152,703         |          90,784          |        2,317,381         |
| 4/13/2021  |        1,085,034         |          87,534          |        2,208,688         |
| 4/12/2021  |        1,468,972         |         102,184          |        2,484,580         |
| 4/11/2021  |        1,561,495         |          90,510          |        2,446,801         |

and so on ...

Or, an ever shorter approach, just use pandas:

import pandas as pd
import requests
from tabulate import tabulate

if __name__ == '__main__':
    source = "https://www.tsa.gov/coronavirus/passenger-throughput"
    df = pd.read_html(requests.get(source).text, flavor="bs4")[0]
    print(tabulate(df.head(10), tablefmt="pretty", showindex=False))

Output:

+-----------+-----------+--------+---------+
| 5/9/2021  | 1707805.0 | 200815 | 2419114 |
| 5/8/2021  | 1429657.0 | 169580 | 1985942 |
| 5/7/2021  | 1703267.0 | 215444 | 2602631 |
| 5/6/2021  | 1644050.0 | 190863 | 2555342 |
| 5/5/2021  | 1268938.0 | 140409 | 2270662 |
| 5/4/2021  | 1134103.0 | 130601 | 2106597 |
| 5/3/2021  | 1463672.0 | 163692 | 2470969 |
| 5/2/2021  | 1626962.0 | 170254 | 2512598 |
| 5/1/2021  | 1335535.0 | 134261 | 1968278 |
| 4/30/2021 | 1558553.0 | 171563 | 2546029 |
+-----------+-----------+--------+---------+

Answered By - baduker