[FIXED] How to reduce number of selenium webdriver instances being spawned by scrapy on running crawl on a spider?

Issue

On running crawl process for any spider, Scrapy tends to spawn a lot of (27 average varying between 19 - 30) Firefox instances, even if the spider being run is not using selenium.

I have tried driver.quit() inside def __del__(self) in each of the spiders using selenium. The problem still persists.

The Firefox instances stay open even after the crawling process is finished.

example spider using selenium:

import logging
import time
from os.path import abspath, dirname, join
import requests
import scrapy
import selenium
from scrapy.http import HtmlResponse
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.remote.remote_connection import LOGGER
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait

LOGGER.setLevel(logging.ERROR)

PATH_DIR = dirname(abspath(__file__))
GECKODRIVER_PATH = abspath(join(PATH_DIR, "../../geckodriver"))
WAIT_TIME = 10

class ExampleSpider(sso_singapore.SsoSpider):

    name = "Example"

    options = Options()
    options.headless = True
    driver = webdriver.Firefox(options=options, executable_path=GECKODRIVER_PATH)

    def __del__(self):
        self.driver.quit()

    def parse(self, response):

        meta = response.meta
        try:
            self.driver.get(response.url)
            body = self.driver.page_source
            try:
                element = WebDriverWait(self.driver, WAIT_TIME).until(
                    EC.presence_of_element_located(
                        (By.ID, '//select[@id="rows_sort"]/option[text()="All"]')
                    )
                )
            except:
                pass
            response = HtmlResponse(
                self.driver.current_url, body=body, encoding="utf-8"
            )

        except Exception as e:
            logging.error(str(e))
        finally:
            self.driver.quit()
       # Create Items based on response

    def start_requests(self):

        for url, meta in zip(urls, meta_list):
            yield scrapy.Request(url, callback=parse, meta=meta)

Any help will be much appreciated.

Solution

from scrapy import signals

class ExampleSpider(sso_singapore.SsoSpider):

    def __init__(self, *args, **kwargs):
        options = Options()
        options.headless = True
        self.driver = webdriver.Firefox(options=options, executable_path="your_path")

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super(ExampleSpider, cls).from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
        return spider

    def spider_closed(self, spider):
        self.driver.quit()

This should do the job.

More on Scrapy signals:

https://docs.scrapy.org/en/latest/topics/signals.html

You can also use Pipeline if you have many spiders and don't want to add the same driver.quit() logic:

class YourPipeline:

    @classmethod
    def from_crawler(cls, crawler):
        pipeline = cls()
        crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
        return pipeline

    def spider_closed(self, spider):
        if hasattr(spider, 'driver'):
            spider.driver.quit()

Answered By - Desprit