[FIXED] UnpicklingError: A load persistent id instruction was encountered, but no persistent_load function was specified

Issue

I was trying to run a python file named api.py. In this file, I'm loading the pickle file of the Deep Learning model that was built and trained using PyTorch.

api.py In api.py the below-given functions are the most important ones.

def load_model_weights(model_architecture, weights_path):
  if os.path.isfile(weights_path):
      cherrypy.log("CHERRYPYLOG Loading model from: {}".format(weights_path))
      model_architecture.load_state_dict(torch.load(weights_path))
  else:
      raise ValueError("Path not found {}".format(weights_path))

        
def load_recommender(vector_dim, hidden, activation, dropout, weights_path):

    rencoder_api = model.AutoEncoder(layer_sizes=[vector_dim] + [int(l) for l in hidden.split(',')],
                               nl_type=activation,
                               is_constrained=False,
                               dp_drop_prob=dropout,
                               last_layer_activations=False)
    load_model_weights(rencoder_api, weights_path) 
    rencoder_api.eval()
    rencoder_api = rencoder_api.cuda()
    return rencoder_api

The directory structure

📦MP1
 ┣ 📂.ipynb_checkpoints
 ┃ ┗ 📜RS_netflix3months_100epochs_64,128,128-checkpoint.ipynb
 ┣ 📂data
 ┃ ┣ 📜AutoEncoder.png
 ┃ ┣ 📜collaborative_filtering.gif
 ┃ ┣ 📜movie_titles.txt
 ┃ ┗ 📜shut_up.gif
 ┣ 📂DeepRecommender
 ┃ ┣ 📂data_utils
 ┃ ┃ ┣ 📜movielens_data_convert.py
 ┃ ┃ ┗ 📜netflix_data_convert.py
 ┃ ┣ 📂reco_encoder
 ┃ ┃ ┣ 📂data
 ┃ ┃ ┃ ┣ 📂__pycache__
 ┃ ┃ ┃ ┃ ┣ 📜input_layer.cpython-37.pyc
 ┃ ┃ ┃ ┃ ┣ 📜input_layer_api.cpython-37.pyc
 ┃ ┃ ┃ ┃ ┗ 📜__init__.cpython-37.pyc
 ┃ ┃ ┃ ┣ 📜input_layer.py
 ┃ ┃ ┃ ┣ 📜input_layer_api.py
 ┃ ┃ ┃ ┗ 📜__init__.py
 ┃ ┃ ┣ 📂model
 ┃ ┃ ┃ ┣ 📂__pycache__
 ┃ ┃ ┃ ┃ ┣ 📜model.cpython-37.pyc
 ┃ ┃ ┃ ┃ ┗ 📜__init__.cpython-37.pyc
 ┃ ┃ ┃ ┣ 📜model.py
 ┃ ┃ ┃ ┗ 📜__init__.py
 ┃ ┃ ┣ 📂__pycache__
 ┃ ┃ ┃ ┗ 📜__init__.cpython-37.pyc
 ┃ ┃ ┗ 📜__init__.py
 ┃ ┣ 📂__pycache__
 ┃ ┃ ┗ 📜__init__.cpython-37.pyc
 ┃ ┣ 📜compute_RMSE.py
 ┃ ┣ 📜infer.py
 ┃ ┣ 📜run.py
 ┃ ┗ 📜__init__.py
 ┣ 📂model_save
 ┃ ┣ 📂model.epoch_99
 ┃ ┃ ┗ 📂archive
 ┃ ┃ ┃ ┣ 📂data
 ┃ ┃ ┃ ┃ ┣ 📜92901648
 ┃ ┃ ┃ ┃ ┣ 📜92901728
 ┃ ┃ ┃ ┃ ┣ 📜92901808
 ┃ ┃ ┃ ┃ ┣ 📜92901888
 ┃ ┃ ┃ ┃ ┣ 📜92901968
 ┃ ┃ ┃ ┃ ┣ 📜92902048
 ┃ ┃ ┃ ┃ ┣ 📜92902128
 ┃ ┃ ┃ ┃ ┣ 📜92902208
 ┃ ┃ ┃ ┃ ┣ 📜92902288
 ┃ ┃ ┃ ┃ ┣ 📜92902368
 ┃ ┃ ┃ ┃ ┣ 📜92902448
 ┃ ┃ ┃ ┃ ┗ 📜92902608
 ┃ ┃ ┃ ┣ 📜data.pkl
 ┃ ┃ ┃ ┗ 📜version
 ┃ ┣ 📜model.epoch_99.zip
 ┃ ┗ 📜model.onnx
 ┣ 📂Netflix
 ┃ ┣ 📂N1Y_TEST
 ┃ ┃ ┗ 📜n1y.test.txt
 ┃ ┣ 📂N1Y_TRAIN
 ┃ ┃ ┗ 📜n1y.train.txt
 ┃ ┣ 📂N1Y_VALID
 ┃ ┃ ┗ 📜n1y.valid.txt
 ┃ ┣ 📂N3M_TEST
 ┃ ┃ ┗ 📜n3m.test.txt
 ┃ ┣ 📂N3M_TRAIN
 ┃ ┃ ┗ 📜n3m.train.txt
 ┃ ┣ 📂N3M_VALID
 ┃ ┃ ┗ 📜n3m.valid.txt
 ┃ ┣ 📂N6M_TEST
 ┃ ┃ ┗ 📜n6m.test.txt
 ┃ ┣ 📂N6M_TRAIN
 ┃ ┃ ┗ 📜n6m.train.txt
 ┃ ┣ 📂N6M_VALID
 ┃ ┃ ┗ 📜n6m.valid.txt
 ┃ ┣ 📂NF_TEST
 ┃ ┃ ┗ 📜nf.test.txt
 ┃ ┣ 📂NF_TRAIN
 ┃ ┃ ┗ 📜nf.train.txt
 ┃ ┗ 📂NF_VALID
 ┃ ┃ ┗ 📜nf.valid.txt
 ┣ 📂test
 ┃ ┣ 📂testData_iRec
 ┃ ┃ ┣ 📜.part-00199-f683aa3b-8840-4835-b8bc-a8d1eaa11c78.txt.crc
 ┃ ┃ ┣ 📜part-00000-f683aa3b-8840-4835-b8bc-a8d1eaa11c78.txt
 ┃ ┃ ┣ 📜part-00003-f683aa3b-8840-4835-b8bc-a8d1eaa11c78.txt
 ┃ ┃ ┗ 📜_SUCCESS
 ┃ ┣ 📂testData_uRec
 ┃ ┃ ┣ 📜.part-00000-4a844096-8dd9-425e-9d9d-bd9062cc6940.txt.crc
 ┃ ┃ ┣ 📜._SUCCESS.crc
 ┃ ┃ ┣ 📜part-00161-4a844096-8dd9-425e-9d9d-bd9062cc6940.txt
 ┃ ┃ ┣ 📜part-00196-4a844096-8dd9-425e-9d9d-bd9062cc6940.txt
 ┃ ┃ ┗ 📜part-00199-4a844096-8dd9-425e-9d9d-bd9062cc6940.txt
 ┃ ┣ 📜data_layer_tests.py
 ┃ ┣ 📜test_model.py
 ┃ ┗ 📜__init__.py
 ┣ 📂__pycache__
 ┃ ┣ 📜api.cpython-37.pyc
 ┃ ┣ 📜load_test.cpython-37.pyc
 ┃ ┣ 📜parameters.cpython-37.pyc
 ┃ ┗ 📜utils.cpython-37.pyc
 ┣ 📜api.py
 ┣ 📜compute_RMSE.py
 ┣ 📜load_test.py
 ┣ 📜logger.py
 ┣ 📜netflix_1y_test.csv
 ┣ 📜netflix_1y_train.csv
 ┣ 📜netflix_1y_valid.csv
 ┣ 📜netflix_3m_test.csv
 ┣ 📜netflix_3m_train.csv
 ┣ 📜netflix_3m_valid.csv
 ┣ 📜netflix_6m_test.csv
 ┣ 📜netflix_6m_train.csv
 ┣ 📜netflix_6m_valid.csv
 ┣ 📜netflix_full_test.csv
 ┣ 📜netflix_full_train.csv
 ┣ 📜netflix_full_valid.csv
 ┣ 📜parameters.py
 ┣ 📜preds.txt
 ┣ 📜RS_netflix3months_100epochs_64,128,128.ipynb
 ┗ 📜utils.py

I am getting such an error (serialization.py). Can someone help me with this error?

D:\Anaconda\envs\practise\lib\site-packages\torch\serialization.py in _legacy_load(f, map_location, pickle_module, **pickle_load_args)
    762             "functionality.")
    763 
--> 764     magic_number = pickle_module.load(f, **pickle_load_args)
    765     if magic_number != MAGIC_NUMBER:
    766         raise RuntimeError("Invalid magic number; corrupt file?")

UnpicklingError: A load persistent id instruction was encountered,
but no persistent_load function was specified.

Solution

After searching through PyTorch documentation, I ended up saving the model in the ONNX format and later loaded that ONNX model into PyTorch model and used it for inference.

import onnx
from onnx2pytorch import ConvertModel


def load_model_weights(model_architecture, weights_path):
    if os.path.isfile("model.onnx"):
        cherrypy.log("CHERRYPYLOG Loading model from: {}".format(weights_path))
        onnx_model = onnx.load("model.onnx")
        pytorch_model = ConvertModel(onnx_model)
        ## model_architecture.load_state_dict(torch.load(weights_path))
    else:
        raise ValueError("Path not found {}".format(weights_path))

        
def load_recommender(vector_dim, hidden, activation, dropout, weights_path):

    rencoder_api = model.AutoEncoder(layer_sizes=[vector_dim] + [int(l) for l in hidden.split(',')],
                               nl_type=activation,
                               is_constrained=False,
                               dp_drop_prob=dropout,
                               last_layer_activations=False)
    load_model_weights(rencoder_api, weights_path) 
    rencoder_api.eval()
    rencoder_api = rencoder_api.cuda()
    return rencoder_api

Some useful resources:

torch.save

torch.load

ONNX tutorials

Answered By - Purushothaman Srikanth