Issue
I am currently creating a telegram chatbot that web scrapes poems. I am having issues 'cleaning up' my web scraping output, especially when I want to retrieve the audio recording from www.poetryarchive.org + text.
This is my current code:
# import relevant libraries for web scraping and asynchronous input/output
import asyncio
import telepot
import telepot.aio
from telepot.aio.loop import MessageLoop
from pprint import pprint
from bs4 import BeautifulSoup
import requests
# Telegram bot API token (and its configuration to aysnc version)
TOKEN = '<INSERT API TOKEN HERE>'
bot = telepot.aio.Bot(TOKEN)
# This function will be modified to accommodate our web scraper code afterwards
async def handle(msg):
# Creating a global variable
global chat_id
# These variables will allow us to glance at the received messages - extract the messages' 'headline info'
content_type, chat_type, chat_id = telepot.glance(msg)
# Log variables
print(content_type, chat_type, chat_id)
pprint(msg)
username = msg['chat']['first_name']
if content_type == 'text':
# map the message to the '/start' function
if msg['text'] == '/start':
await bot.sendMessage(chat_id, "Hi! I am PoeticScraper and I am able to scrape your desired poem, its credits, the poet's information, and the poem's glossary tags. Key in '/help' for more assistance or any keywords without the '/' to get the ball rolling!")
# map the message to the '/help' function
elif msg['text'] == '/help':
await bot.sendMessage(chat_id, "I am here to help! For starters, you can type any keywords in and I will retrieve the most relevant poem for you! Alternatively, if you have specific poems you want to search, you can do so too! But please note the for poem titles with multiple words, please use '-' between them!")
elif msg['text'] == '/more':
await bot.sendMessage(chat_id, "Can't get enough of poems? Fret not! There are so many other poetry websites for you to browse: www.poetryfoundation.org & www.poets.org")
elif msg['text'] != '/start' or '/help':
text = msg['text']
# it's better to strip and lower the input in order for the subsequent function to comprehend it
text = text.strip()
await getInformation(text.lower())
# if all else fails...
else:
await bot.sendMessage(chat_id, "404 not found!")
# the variable 'headers' is required so that the target website perceives our web-scraper as a browser
headers = {'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36'}
# defining a function - in charge of making the HTTP request, receiving an HTML response, and retrieving the poem
async def getInformation(text):
# specify the url - all the URLs on this website follow this convention: 'https://poetryarchive.org/poem/<insert poem name>/'
mainpage_url = 'https://poetryarchive.org/poem/' + text
# query the website and return the html to the variable 'page'
page = requests.get(mainpage_url, headers = headers)
# parse the html using BeautifulSoup and store in variable `soup`
soup = BeautifulSoup(page.text, 'html.parser')
pprint(soup)
# this part is crucial to web-scraping the required information from the webpage
try:
# poet's details
try:
for poet in soup.find_all('h3', {'style': 'margin-top:0px;'}):
for poet_name in poet.find_all('a'):
pprint(poet_name.get_text('href'))
await bot.sendMessage(chat_id, poet_name)
except:
await bot.sendMessage(chat_id, "The poet's bio is currently unavailable.")
# credits/copyright info about the poem
try:
credits = soup.find('div', {'class': 'source-box bg-grey pa-boxed small-text'}).text
await bot.sendMessage(chat_id, credits)
except:
await bot.sendMessage(chat_id, "No credits found!")
# audio version of the poem
try:
for audio_recording in soup.find_all('div', {'class': 'pa-player-wrap'}):
pprint(audio_recording.get('audiosrc'))
await bot.sendMessage(chat_id, audio_recording)
except:
await bot.sendMessage(chat_id, "Sorry. The audio recording is currently unavailable.")
# textual version of the poem
try:
poem = soup.find('div', {'class': 'poem-content'}).text
await bot.sendMessage(chat_id, poem)
except:
await bot.sendMessage(chat_id, "Sorry. The poem is currently unavailable.")
# genre types for this poem
try:
for meta_tags in soup.find_all('div', {'class': 'player-metas'}):
for tag in meta_tags.find_all('a'):
pprint(tag.get_text('href'))
await bot.sendMessage(chat_id, "These are the glossary tags associated with this poem: ")
await bot.sendMessage(chat_id, tag)
except:
await bot.sendMessage(chat_id, "Sorry. The glossary tags for this poem are currently unavailable.")
except:
await bot.sendMessage(chat_id, "Oh no! Something went wrong... Please enter '/help' for more information.")
# Program startup
loop = asyncio.get_event_loop()
loop.create_task(MessageLoop(bot, handle).run_forever())
print('Listening ...')
# Keep the program running
loop.run_forever()
This is the telegram output that I got for the poem 'Upstream':
<div class="pa-player-wrap">
<div class="df-close"></div>
<div class="df-title"><h6>Upstream - Jean Binta Breeze</h6></div> <div audiosrc="https://poetryarchive.org/wp-content/uploads/poems/full/Upstream.mp3" class="pa-player for-poem-3216 paused">
<a class="pa-poem-play pa-player-control"></a>
<div class="pa-poem-progress">
<div class="timer current"></div>
<div class="pa-progress-slider">
<div class="pa-loading-bar"></div>
<div class="pa-progress-bar"></div>
</div>
<div class="timer full"></div>
</div>
<div class="pa-poem-volume" title="Volume">
<a class="pa-volume pa-player-control"></a>
<div class="pa-volume-slider-wrap">
<div class="pa-volume-slider">
<div class="pa-volume-bar">
<div class="pa-volume-handlebar" style="width: 100%;"></div>
</div>
</div>
</div>
</div>
<a class="close-df"><i class="fal fa-fw fa-times c-pink"></i></a>
<a class="close-eyes pa-player-control" title="Distraction free">
<div class="eye eye-open"></div>
<div class="eye eye-shut"></div>
</a>
<div class="pa-poem-add">
<div class="member-fav nonmember-fav" title="Become a member to add to a collection"></div> </div>
<a class="pa-poem-drag" title="Reorder collection"></a>
</div>
</div>
What can I do so that I am able to just scrape for the URL in the tag? I've tried so many methods but it hasn't worked.
Solution
The following might work where you use css selectors to target the audiosrc
attribute of element with class pa-player
(part of multi-valued class)
audio = soup.select_one('.pa-player[audiosrc]')
message_link = 'Audio link not found'
if audio:
message_link = audio['audiosrc']
Answered By - QHarr
0 comments:
Post a Comment
Note: Only a member of this blog may post a comment.