Issue
I am trying to scrape this website https://99petshops.com.au/Search?brandName=Ziwi%20Peak&animalCode=DOG&storeId=89%2F&page=1 and there are couple of prices listed oer there in which I am interested to pair with the real price with each listing in a way that csv headers will look like this:
title, lowest_price, lowest_price_0, lowest_price_1, lowest_price_2, lowest_price_3....
title, lowest_price, lowest_price_0, lowest_price_1, lowest_price_2, lowest_price_3....
I am getting all the price data but couldn't make it organized as above so far my code for this is:
import requests
from bs4 import BeautifulSoup
import csv
import json
class ZiwiScraper:
results = []
headers = {
'authority': '99petshops.com.au',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'accept-language': 'en,ru;q=0.9',
'cache-control': 'max-age=0',
# Requests sorts cookies= alphabetically
# 'cookie': 'TrackerGuid=f5419f8d-632a-46b1-aa04-eed027d03e89; _ga=GA1.3.1385392550.1666770065; _gid=GA1.3.1560927430.1666770065',
'referer': 'https://www.upwork.com/',
'sec-ch-ua': '"Chromium";v="104", " Not A;Brand";v="99", "Yandex";v="22"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'cross-site',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1',
'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.114 YaBrowser/22.9.1.1110 (beta) Yowser/2.5 Safari/537.36',
}
def fetch(self, url):
print(f'HTTP GET request to URL: {url}', end='')
res = requests.get(url, headers=self.headers)
print(f' | Status Code: {res.status_code}')
return res
def parse(self, html):
master_dict = {}
soup = BeautifulSoup(html, 'lxml')
titles = [title.text.strip() for title in soup.find_all('h2')]
low_prices = [low_price.text.split(' ')[-1] for low_price in soup.find_all('span', {'class': 'hilighted'})]
store_names = []
stores = soup.find_all('p')
for store in stores:
store_name = store.find('img')
if store_name:
store_names.append(store_name['alt'])
shipping_prices = [shipping.text.strip() for shipping in soup.find_all('p', {'class': 'shipping'})]
price_per_hundered_kg = [unit_per_kg.text.strip() for unit_per_kg in soup.find_all('p', {'class': 'unit-price'})]
other_details = soup.find_all('div', {'class': 'pd-details'})
for index in range(0, len(titles)):
try:
price_per_100_kg = price_per_hundered_kg[index]
except:
price_per_100_kg = ''
try:
lowest_prices = low_prices[index]
except:
lowest_prices = ''
for detail in other_details:
detail_1 = [pr.text.strip() for pr in detail.find_all('span', {'class': 'sp-price'})]
for idx, price in enumerate(detail_1):
self.results.append({
'title': titles[index],
'lowest_prices': lowest_prices,
f'lowest_price_{idx}': detail_1[idx],
'store_names': store_names[index],
'shipping_prices': shipping_prices[index],
'price_per_100_kg': price_per_100_kg,
})
def to_csv(self):
key_list = list()
for key_list_for_one_element in [list(x.keys()) for x in self.results]:
key_list.extend(key_list_for_one_element)
key_list = set(key_list)
with open('ziwi_pets_2.csv', 'w') as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=key_list)
writer.writeheader()
for row in self.results:
writer.writerow(row)
print('Stored results to "ziwi_pets_2.csv"')
def run(self):
for page in range(1):
url = f'https://99petshops.com.au/Search?brandName=Ziwi%20Peak&animalCode=DOG&storeId=89%2F&page={page}'
response = self.fetch(url)
self.parse(response.text)
self.to_csv()
if __name__ == '__main__':
scraper = ZiwiScraper()
scraper.run()
This selector gives all the prices:
price_list = [pr.find_all('span', {'class': 'sp-price'}) for pr in soup.find_all('table', {'class': 'tbl-price top-10'})]
I just couldn't figure out how to pair all the prices with each associated listing.
Please can anyone help me out here? Thanks in advance!
Solution
If I understand you right, you want to get title, price and get the prices from the table when you expand the other prices. I recommend to use pandas for creating the dataframe and saving the CSV:
import requests
import pandas as pd
from bs4 import BeautifulSoup
url = "https://99petshops.com.au/Search?brandName=Ziwi%20Peak&animalCode=DOG&storeId=89%2F&page=1"
soup = BeautifulSoup(requests.get(url).content, "html.parser")
all_info = []
for item in soup.select(".pd-info"):
title = item.h2.get_text(strip=True)
price = item.select_one('span:-soup-contains("Price")').span.text
d = {"Title": title, "Price": price}
for i, p in enumerate(item.select(".sp-price"), 1):
d[f"price_{i:>02}"] = p.get_text(strip=True)
all_info.append(d)
df = pd.DataFrame(all_info).fillna("")
print(df)
df.to_csv("data.csv", index=False)
Prints:
Title Price price_01 price_02 price_03 price_04 price_05 price_06 price_07 price_08 price_09 price_10 price_11 price_12 price_13 price_14 price_15 price_16 price_17 price_18 price_19 price_20 price_21 price_22 price_23 price_24 price_25 price_26 price_27 price_28 price_29
0 Ziwi Peak Dog Air-Dried Free Range Chicken Recipe 1Kg $57.75 $64.60 $64.60 $64.95 $64.95 $64.99 $66.29 $67.32 $69.69 $69.69 $69.95 $70.19 $71.99 $72.99 $73.39 $75.15 $75.95 $76.99 $77.99 $77.99 $79.99 $81.54 $81.99 $83.49 $83.49 $83.50
1 Ziwi Peak Dog Air-Dried Free Range Chicken Recipe 2.5Kg $127.80 $138.95 $140.24 $140.65 $141.68 $144.99 $147.84 $148.49 $149.95 $152.14 $152.14 $155.95 $159.30 $159.95 $159.99 $162.90 $164.99 $164.99 $164.99 $166.99 $167.00 $169.99 $176.95 $176.99 $176.99 $176.99 $181.00
2 Ziwi Peak Dog Air-Dried Lamb Recipe 2.5Kg $127.80 $138.95 $140.24 $140.63 $140.65 $144.99 $147.84 $148.49 $149.95 $155.95 $155.99 $159.30 $159.99 $159.99 $159.99 $160.15 $162.90 $164.99 $164.99 $165.00 $169.99 $174.99 $176.95 $176.99 $176.99 $181.00
3 Ziwi Peak Dog Air-Dried Beef Recipe 454g $29.25 $32.95 $34.32 $34.99 $35.95 $35.99 $35.99 $36.13 $37.20 $37.95 $38.25 $38.95 $38.99 $39.09 $40.28 $40.95 $41.99 $42.49 $42.50 $42.50 $43.99 $44.75 $44.99 $45.49 $45.49 $45.50
4 Ziwi Peak Dog Air-Dried Free Range Chicken Recipe 454g $29.25 $32.95 $34.32 $34.95 $35.00 $35.95 $36.13 $36.99 $37.95 $37.99 $38.25 $38.95 $39.09 $39.09 $40.28 $40.95 $41.99 $42.49 $42.50 $42.50 $43.99 $44.99 $45.49 $45.49 $45.50
5 Ziwi Peak Dog Air-Dried Free Range Chicken Recipe 4Kg $216.22 $218.95 $219.95 $220.99 $222.75 $222.75 $223.61 $228.85 $230.23 $233.64 $233.99 $238.99 $239.00 $244.99 $249.99 $251.10 $257.40 $259.99 $259.99 $259.99 $259.99 $261.00 $264.29 $278.99 $279.99 $286.00
6 Ziwi Peak Dog Air-Dried Beef Recipe 2.5Kg $127.80 $138.65 $138.95 $139.99 $140.24 $141.82 $141.82 $144.99 $146.95 $147.84 $148.49 $152.14 $152.14 $155.99 $159.30 $159.95 $159.99 $160.15 $162.90 $164.99 $164.99 $165.00 $169.99 $174.99 $176.95 $176.99 $176.99 $181.00 $446.00
7 Ziwi Peak Dog Air-Dried Lamb Recipe 1Kg $57.75 $62.05 $63.95 $64.60 $64.60 $64.99 $66.29 $67.32 $67.99 $69.69 $69.69 $69.95 $70.19 $71.99 $72.95 $72.99 $73.39 $74.95 $75.15 $75.95 $77.99 $77.99 $77.99 $79.99 $80.99 $81.54 $81.99 $83.49 $83.49 $83.50
8 Ziwi Peak Dog Air-Dried Mackerel & Lamb Recipe 4Kg $220.99 $231.72 $234.95 $237.60 $237.60 $239.95 $249.99 $250.80 $255.99 $255.99 $256.49 $259.00 $264.99 $269.10 $269.99 $271.70 $276.59 $278.10 $281.00 $284.99 $289.99 $299.99 $299.99 $309.00
9 Ziwi Peak Dog Air-Dried Beef Recipe 1Kg $57.75 $63.65 $63.95 $64.36 $64.95 $66.29 $67.32 $67.99 $69.69 $69.69 $69.95 $70.19 $72.99 $73.39 $75.15 $75.95 $75.99 $77.99 $77.99 $77.99 $79.99 $81.54 $81.99 $83.49 $83.49 $83.50 $205.00
10 Ziwi Peak Dog Air-Dried Lamb Recipe 4Kg $216.22 $218.95 $219.95 $220.99 $233.64 $233.99 $234.95 $239.00 $240.99 $240.99 $244.99 $249.99 $249.99 $251.10 $253.10 $257.40 $259.99 $259.99 $259.99 $261.00 $264.29 $278.99 $278.99 $279.99 $286.00
11 Ziwi Peak Dog Wet Mackerel & Lamb Recipe 390g $6.95 $6.99 $7.99
12 Ziwi Peak Dog Air-Dried Tripe & Lamb Recipe 2.5Kg $138.95 $139.50 $140.24 $140.65 $141.45 $144.95 $144.99 $147.84 $148.49 $149.95 $155.95 $155.99 $155.99 $155.99 $159.30 $159.95 $159.99 $162.90 $164.99 $164.99 $165.00 $169.99 $176.00 $176.95 $176.99 $176.99 $181.00 $475.00
13 Ziwi Peak Dog Air-Dried Beef Recipe 4Kg $216.22 $218.95 $220.99 $223.72 $233.64 $233.99 $234.95 $239.00 $240.99 $240.99 $244.99 $249.99 $251.10 $253.10 $257.40 $259.99 $259.99 $259.99 $261.00 $264.29 $278.99 $278.99 $279.99 $286.00
14 Ziwi Peak Dog Air-Dried Lamb Recipe 454g $29.25 $32.89 $32.95 $34.32 $34.99 $35.95 $36.13 $36.99 $37.20 $37.95 $38.25 $38.95 $39.09 $39.09 $39.99 $40.28 $41.99 $42.49 $42.50 $42.50 $43.99 $44.75 $44.99 $45.49 $45.49 $45.50
15 Ziwi Peak Dog Air-Dried Mackerel & Lamb Recipe 454g $34.95 $35.42 $36.13 $36.96 $37.99 $39.95 $39.99 $39.99 $39.99 $40.00 $40.95 $41.00 $41.85 $41.95 $41.99 $42.49 $44.24 $44.99 $45.00 $46.50 $46.99 $46.99 $46.99 $47.99 $49.15 $49.95 $49.99 $49.99
16 Ziwi Peak Dog Wet Free Range Chicken Recipe 390g X 12 $75.95 $76.00 $78.90 $78.90 $81.59 $86.29 $87.54 $89.88 $93.50 $93.96 $96.99 $99.99 $104.39 $107.49
17 Ziwi Peak Dog Wet Lamb Recipe 390g X 12 $78.90 $78.90 $80.65 $81.59 $86.17 $87.54 $89.88 $93.96 $99.99 $104.39 $107.49
18 Ziwi Peak Dog Wet East Cape Recipe 170g $3.87 $3.87 $4.49 $5.09 $5.95 $5.99
19 Ziwi Peak Dog Wet Beef Recipe 390g $6.35 $6.45 $6.95 $6.95 $6.99 $7.00 $7.49 $7.71 $8.57 $8.99 $92.95
20 Ziwi Peak Dog Wet Hauraki Plains Recipe 170g $3.57 $3.57 $4.99 $5.50 $54.95
21 Ziwi Peak Dog Wet Free Range Chicken Recipe 170g $4.50 $4.95 $4.95 $4.99 $5.50
22 Ziwi Peak Dog Venison Shank Bone Oral Chews Full $18.86 $20.95 $23.17 $25.87 $26.95 $27.99 $32.99 $34.95 $34.97 $34.99 $37.49 $37.50
23 Ziwi Peak Dog Air-Dried Mackerel & Lamb Recipe 2.5Kg $140.24 $151.65 $152.06 $152.96 $152.96 $159.72 $159.95 $159.99 $159.99 $159.99 $161.99 $164.99 $168.95 $169.95 $169.99 $172.80 $172.80 $172.99 $173.00 $178.29 $179.00 $179.99 $179.99 $179.99 $189.99 $191.95 $191.99 $192.00
24 Ziwi Peak Good Dog Reward Lamb 85g $9.24 $10.00 $10.76 $11.24 $11.95 $11.95 $11.95 $11.99 $11.99 $12.95 $13.99 $14.26 $14.26 $14.49 $14.49 $15.49 $15.49 $15.50 $21.56
25 Ziwi Peak Dog Canned Food Venison 390g $9.45
26 Ziwi Peak Dog Venison Shank Bone Oral Chews Half $11.66 $12.95 $13.14 $14.62 $15.95 $17.19 $19.67 $19.99 $19.99 $20.95 $21.49 $21.50 $23.99
27 Ziwi Peak Good Dog Reward Venison 85g $11.95 $13.99 $13.99 $14.72 $14.99 $15.99 $15.99 $16.95 $17.49 $17.99 $18.49 $19.49 $19.49 $19.50 $22.99 $24.86
28 Ziwi Peak Dog Venison Lung & Kidney 60g $18.74 $19.49 $19.99 $23.34 $23.99 $23.99 $25.50 $25.50
29 Ziwi Peak Dog Wet Venison Recipe 170g X 12 $62.95 $71.88
and saves data.csv (screenshot from LibreOffice):
But I recommend to save the prices under Store Names, not under "price_XY":
import requests
import pandas as pd
from bs4 import BeautifulSoup
url = "https://99petshops.com.au/Search?brandName=Ziwi%20Peak&animalCode=DOG&storeId=89%2F&page=1"
soup = BeautifulSoup(requests.get(url).content, "html.parser")
all_info = []
for item in soup.select(".pd-info"):
title = item.h2.get_text(strip=True)
price = item.select_one('span:-soup-contains("Price")').span.text
d = {"Title": title, "Price": price}
for p in item.select(".sp-price"):
d[p.find_next("img")["alt"]] = p.get_text(strip=True)
all_info.append(d)
df = pd.DataFrame(all_info).fillna("")
print(df)
df.to_csv("data.csv", index=False)
Prints:
Title Price VetShopAustralia Vet Products Direct Pet Shop Direct Petso PetPost Pet Chemist Your PetPA Pet Circle Petbarn World for Pets Lucky Pet Stefmar Budget Pet Products Best Friends Pets Kellyville Pets iPetStore My Pet Warehouse Pet City Pets Unleashed PetO PETstock Pet House Habitat Pets Pet Culture Peticular Vet n Pet Direct Petzoo Woonona Pet Foods Bundi Pet Pets on the Park Mega Pet Warehouse eBay Petceutics The Animal Pharmacy Pacific Pet Supplies Sierra Pet Products Woofers World Discount Pet Meds
0 Ziwi Peak Dog Air-Dried Free Range Chicken Recipe 1Kg $57.75 $64.60 $64.60 $64.95 $64.95 $64.99 $66.29 $67.32 $69.69 $69.69 $69.95 $70.19 $71.99 $72.99 $73.39 $75.15 $75.95 $76.99 $77.99 $77.99 $79.99 $81.54 $81.99 $83.49 $83.49 $83.50
1 Ziwi Peak Dog Air-Dried Free Range Chicken Recipe 2.5Kg $127.80 $140.65 $141.68 $140.24 $147.84 $152.14 $152.14 $149.95 $148.49 $166.99 $159.99 $162.90 $159.30 $155.95 $167.00 $164.99 $164.99 $169.99 $181.00 $176.99 $176.99 $176.99 $176.95 $138.95 $144.99 $159.95 $164.99
2 Ziwi Peak Dog Air-Dried Lamb Recipe 2.5Kg $127.80 $140.65 $140.63 $140.24 $147.84 $159.99 $159.99 $149.95 $148.49 $155.99 $159.99 $162.90 $159.30 $155.95 $165.00 $164.99 $169.99 $181.00 $174.99 $176.99 $176.99 $176.95 $138.95 $144.99 $164.99 $160.15
3 Ziwi Peak Dog Air-Dried Beef Recipe 454g $29.25 $34.99 $36.13 $34.32 $35.99 $39.09 $37.95 $38.25 $38.99 $35.99 $40.28 $40.95 $38.95 $41.99 $42.50 $42.50 $44.99 $44.75 $42.49 $45.49 $45.49 $45.50 $32.95 $35.95 $37.20 $43.99
4 Ziwi Peak Dog Air-Dried Free Range Chicken Recipe 454g $29.25 $34.95 $36.13 $34.32 $39.09 $39.09 $37.95 $38.25 $36.99 $37.99 $40.28 $40.95 $38.95 $41.99 $42.50 $42.50 $44.99 $42.49 $45.49 $45.49 $45.50 $32.95 $35.95 $35.00 $43.99
5 Ziwi Peak Dog Air-Dried Free Range Chicken Recipe 4Kg $216.22 $222.75 $222.75 $219.95 $223.61 $220.99 $233.64 $230.23 $238.99 $228.85 $233.99 $244.99 $249.99 $257.40 $251.10 $261.00 $259.99 $259.99 $279.99 $286.00 $259.99
...
Answered By - Andrej Kesely
0 comments:
Post a Comment
Note: Only a member of this blog may post a comment.