[FIXED] How to write csv and insert scrape data

Issue

I am designing scraping project for my research but i am stuck in to write scrape data in csv. Please help me for that?

i have successfully scrape data but i want to store it in csv here below is my code

need to write code to pull all of the html from a website then save it to a csv file.

I believe I somehow need to turn the links into a list and then write the list, but I'm unsure how to do that.

This is what I have so far:

import requests
import time
from bs4 import BeautifulSoup
import csv



# Collect and parse first page
page = requests.get('https://www.myamcat.com/jobs')
soup = BeautifulSoup(page.content, 'lxml')

print("Wait Scraper is working on ")
time.sleep(10)
if(page.status_code != 200):
    
    print("Error in Scraping check the url")
else:
    
    print("Successfully scrape the data")
    time.sleep(10)
    print("Loading data in csv")
    file = csv.writer(open('dataminer.csv', 'w'))
    file.writerow(['ProfileName', 'CompanyName', 'Salary', 'Job', 'Location']) 
       
    for pname in soup.find_all(class_="profile-name"):
        
        #print(pname.text)
        profname = pname.text
        file.writerow([profname, ])
        
    for cname in soup.find_all(class_="company_name"):

        print(cname.text)
                   
    for salary in soup.find_all(class_="salary"):

        print(salary.text)
                 

    for lpa in soup.find_all(class_="jobText"):

        print(lpa.text) 

    for loc in soup.find_all(class_="location"):

        print(loc.text)

        

            

Solution

Make a dict and save the data into it then save to csv, check below code!

import requests
import time
from bs4 import BeautifulSoup
import csv



# Collect and parse first page
page = requests.get('https://www.myamcat.com/jobs')
soup = BeautifulSoup(page.content, 'lxml')
data = []
print("Wait Scrapper is working on ")
if(page.status_code != 200):
    print("Error in Srapping check the url")
else:
    print("Successfully scrape the data")
    for x in soup.find_all('div',attrs={'class':'job-page'}):
        data.append({
            'pname':x.find(class_="profile-name").text.encode('utf-8'),
            'cname':x.find(class_="company_name").text.encode('utf-8'),
            'salary':x.find(class_="salary").text.encode('utf-8'),
            'lpa':x.find(class_="jobText").text.encode('utf-8'),
            'loc':x.find(class_="location").text.encode('utf-8')})

print("Loading data in csv")
with open('dataminer.csv', 'w') as f:
    fields = ['salary', 'loc', 'cname', 'pname', 'lpa']
    writer = csv.DictWriter(f, fieldnames=fields)
    writer.writeheader()
    writer.writerows(data)

Answered By - Sohan Das