[FIXED] How I can scrape subpages and merge it with page info?

Issue

I use scrapy for parsing the page. The page has subpages (categories) from which I also need to get information and combine it all in one element (maybe save the information from the additional pages as a json), which I add to csv. I've tried different options, such as:

requests = scrapy.Request(url, meta={'meta_item': item}, callback=self.parse_category)

Or

yield scrapy.Request(url, meta={'meta_item': item}, callback=self.parse_category)

But neither method works the way I want it to.

For example, I took pages from https://www.webscorer.com/findraces?pg=results (example: https://www.webscorer.com/seriesresult?seriesid=211565 ) and get information from this page. After that, I need to get additional information from category (example: https://www.webscorer.com/seriesresult?seriesid=211565&gender=F ): example and put all of them in csv. My code now:

class WebscorerSpider(scrapy.Spider):
name = 'webscorer'
allowed_domains = ['webscorer.com']

def start_requests(self):
    url = f'https://www.webscorer.com/findraces?pg=results'
    yield scrapy.Request(url, callback=self.parse_page)

def parse_page(self, response, **kwargs):
    for href in response.css('table.results-table tbody tr a::attr("href")').extract():
        url = response.urljoin(href)
        url = 'https://www.webscorer.com/seriesresult?seriesid=211565'
        yield scrapy.Request(url, callback=self.parse)

def parse(self, response: Response, **kwargs):
    latlong_match = re.search('lat=(.*)&lng=(.*)', response.css('span#FSrc::text').get())
    item = dict()

    for href in response.css('table.category-table .category-name').css('a::attr("href")').extract():
        url = response.urljoin(href)

        # requests = scrapy.Request(url, meta={'meta_item': item}, callback=self.parse_category)

        yield scrapy.Request(url, meta={'meta_item': item}, callback=self.parse_category)

    yield WebscorerEvent(name=response.css('h1.race-name::text').get(),
                         source_url=response.request.url,
                         sport_discipline=response.css('td.spec+td').css('strong::text').get(),
                         description=response.css('span.regnotes span::text').get(),
                         hero_image=response.css('p.associated-race-pic img::attr(src)').get(),
                         start_date=parse_webscorer_date(response.css('p.race-date::text').get()),
                         location={
                              "link": f"https://www.google.com/maps/search/?api=1&query={latlong_match.group(1)},{latlong_match.group(2)}",
                              "description": response.css('td.spec:contains("Location:")+td strong::text').get()})

def parse_category(self, response, **kwargs):
    item = response.meta['meta_item']
    # print(item)
    item['winner'] = response.css('table.results-table .r-racername span::text').get()

    return item

Solution

You did yield WebscorerEvent, so you already "dropped" the item before getting the data needed on the next page.

You could do something like:

def parse(self, response: Response, **kwargs):
    latlong_match = re.search('lat=(.*)&lng=(.*)', response.css('span#FSrc::text').get())
    item = {
        "name": response.css('h1.race-name::text').get(),
        "source_url": response.request.url,
        "sport_discipline": response.css('td.spec+td').css('strong::text').get(),
        "description": response.css('span.regnotes span::text').get(),
        "hero_image": response.css('p.associated-race-pic img::attr(src)').get(),
        "start_date": parse_webscorer_date(response.css('p.race-date::text').get()),
        "location": {
            "link": f"https://www.google.com/maps/search/?api=1&query={latlong_match.group(1)},{latlong_match.group(2)}",
            "description": response.css('td.spec:contains("Location:")+td strong::text').get()
        }
    }

    for href in response.css('table.category-table .category-name').css('a::attr("href")').extract():
        url = response.urljoin(href)

        yield scrapy.Request(url, meta={'meta_item': item}, callback=self.parse_category)

def parse_category(self, response, **kwargs):
    item = response.meta['meta_item']
    item['winner'] = response.css('table.results-table .r-racername span::text').get()

    yield WebscorerEvent(item)

So in that way you only yield the item in the end, with all data needed.

Answered By - JPBeckner