[FIXED] Getting "TypeError: can't pickle thread.lock objects" when an object is deepcopied with log configs

Issue

I am getting TypeError: can't pickle thread.lock objects when an object is deep copied. Following is my class structure.

import logging
import copy

DEFAULT_LOG_PATH = r"C:\temp\loging.log"
class LoggerConfig(object):
    def __init__(self, log_obj):
        self._logger = log_obj
        self.default_level = logging.DEBUG
        self.set_default_config()

    def set_default_config(self):
        formatter = self._get_default_formatter()
        self.fh = self._create_file_handler(DEFAULT_LOG_PATH,
                                   self.default_level, formatter)
        self.ch = self._create_stream_handler(self.default_level, formatter)
        self._logger.addHandler(self.fh)
        self._logger.addHandler(self.ch)

    def _get_default_formatter(self):
        msg_format = '%(asctime)s.%(msecs).03d: %(levelname)s:%(message)s'
        date_format = '%d-%b-%y %H:%M:%S'
        formatter = logging.Formatter(msg_format, datefmt=date_format)
        return formatter

    def _create_stream_handler(self, log_level, formatter):
        ch = logging.StreamHandler()
        ch.setLevel(log_level)
        ch.setFormatter(formatter)
        return ch

    def _create_file_handler(self, filename, log_level, formatter):
        fh = logging.FileHandler(filename)
        fh.setLevel(log_level)
        fh.setFormatter(formatter)

        return fh

class DataSpec(object):
    def __init__(self, default=None):
        self.default = copy.deepcopy(default)

class Base(object):
    def __init__(self):
        self.logger = logging.getLogger("logs")
        self.log_config = LoggerConfig(self.logger)

class Simple(Base):
    def Run(self):
        print("In Simple Class")

class Complex(Base):
    def __init__(self):
        s = DataSpec(Simple())

    def Run(self):
        print("In complex")

c = Complex()
c.Run()

When the deepcopy occurs in DataSpec.init(), the following exception occurs.

Traceback (most recent call last):
File "C:/temp/temp.py", line 72, in <module>
    c = Complex()
  File "C:/temp/temp.py", line 67, in __init__
    s = DataSpec(Simple())
  File "C:/temp/temp.py", line 52, in __init__
    self.default = copy.deepcopy(default)
  File "C:\Python27\lib\copy.py", line 190, in deepcopy
    y = _reconstruct(x, rv, 1, memo)
  File "C:\Python27\lib\copy.py", line 334, in _reconstruct
    state = deepcopy(state, memo)
  File "C:\Python27\lib\copy.py", line 163, in deepcopy
    y = copier(x, memo)
  File "C:\Python27\lib\copy.py", line 257, in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
  File "C:\Python27\lib\copy.py", line 190, in deepcopy
    y = _reconstruct(x, rv, 1, memo)
  File "C:\Python27\lib\copy.py", line 334, in _reconstruct
    state = deepcopy(state, memo)
  File "C:\Python27\lib\copy.py", line 163, in deepcopy
    y = copier(x, memo)
  File "C:\Python27\lib\copy.py", line 257, in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
  File "C:\Python27\lib\copy.py", line 163, in deepcopy
    y = copier(x, memo)
  File "C:\Python27\lib\copy.py", line 230, in _deepcopy_list
    y.append(deepcopy(a, memo))
  File "C:\Python27\lib\copy.py", line 190, in deepcopy
    y = _reconstruct(x, rv, 1, memo)
  File "C:\Python27\lib\copy.py", line 334, in _reconstruct
    state = deepcopy(state, memo)
  File "C:\Python27\lib\copy.py", line 163, in deepcopy
    y = copier(x, memo)
  File "C:\Python27\lib\copy.py", line 257, in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
  File "C:\Python27\lib\copy.py", line 190, in deepcopy
    y = _reconstruct(x, rv, 1, memo)
  File "C:\Python27\lib\copy.py", line 334, in _reconstruct
    state = deepcopy(state, memo)
  File "C:\Python27\lib\copy.py", line 163, in deepcopy
    y = copier(x, memo)
  File "C:\Python27\lib\copy.py", line 257, in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
  File "C:\Python27\lib\copy.py", line 182, in deepcopy
    rv = reductor(2)
TypeError: can't pickle thread.lock objects

Here, I saw that when I comment both self._logger.addHandler() in LogConfig class, it doesn't throw this error. Can any one help me to find out workaround? I need to add those handlers to save the logs.

Solution

Finally, I've find a solution. Here, in the Base class it needs to override deepcopy method and avoid deepcopying the logger, log_config object. The class will look like below.

class Base(object):
    def __init__(self):
        self.deep_cp_attr = None
        self.logger = logging.getLogger("logs")
        self.log_config = LoggerConfig(self.logger)

    def __deepcopy__(self, memodict={}):
        cpyobj = type(self)() # shallow copy of whole object 
        cpyobj.deep_cp_attr = copy.deepcopy(self.other_attr, memodict) # deepcopy required attr

        return cpyobj

Answered By - smm