[FIXED] Passing **kwargs to function results in `Unexpected keyword argument` in flask app, but not test script

Issue

I'm getting an unexpected keyword argument in my flask app. When I execute the same functions in a test python file, I don't get the TypeError.

In my flask app, I have a worker class and a function:

import concurrent.futures as cf


class Worker:
    """Class for multiprocessing."""

    def __init__(self, workers=1):
        self.workers = workers

    def run(self, fn, files, **kwargs):
        print(kwargs)
        num = len(files)
        with cf.ProcessPoolExecutor(max_workers=self.workers) as executor:
            results = [executor.submit(fn, f, **kwargs) for f in files]
            for i, f in enumerate(cf.as_completed(results), start=1):
                print(f'({i}/{num}): {f.result()}')

def create_archive(folder, extension='.tar.gz', format='PAX', remove_directory=False):
   # do some stuff

I execute the run method Worker like this in one of the routes:

    worker = Worker(workers=5)

    worker.run(
        create_archive,
        valid_folders,
        extension=request.json['extension'],
        format=request.json['format'],
        remove_directory=str2bool(request.json['removeDir'])
    )

and get this error message:

TypeError: create_archive() got an unexpected keyword argument 'extension'

The print statement in the run methods produces this output:

{'extension': '.tar.gz', 'format': 'pax', 'remove_directory': False}

Now, I created a separate test.py script to try and replicate the issue. I get the same output from the print statement, and the code executes properly. Here's the test.py:

import concurrent.futures as cf


class Worker:
    """Class for multiprocessing."""

    def __init__(self, workers=1):
        self.workers = workers

    def run(self, fn, files, **kwargs):
        print(kwargs)
        num = len(files)
        with cf.ProcessPoolExecutor(max_workers=self.workers) as executor:
            results = [executor.submit(fn, f, **kwargs) for f in files]
            for i, f in enumerate(cf.as_completed(results), start=1):
                print(f'({i}/{num}): {f.result()}')


def create_archive(folder, extension='.tar.gz', format='pax', remove_directory=False):
    print(folder, extension, format, remove_directory)
    return f'{folder} complete'


if __name__ == "__main__":

    worker = Worker(workers=5)
    worker.run(
        create_archive,
        ['folder 1', 'folder 2', 'folder 3'],
        extension='.tar',
        format='pax',
        remove_directory=False
    )

I've been looking at this for a long time and I don't see what the issue is. Any ideas as to why I'm getting this error in my flask app but not the test script?

Full error message:

concurrent.futures.process._RemoteTraceback: 
"""
Traceback (most recent call last):
  File "/Users/useridhere/miniconda3/lib/python3.11/concurrent/futures/process.py", line 256, in _process_worker
    r = call_item.fn(*call_item.args, **call_item.kwargs)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
TypeError: create_archive() got an unexpected keyword argument 'extension'
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/useridhere/repos/myproject/venv/lib/python3.11/site-packages/flask/app.py", line 1478, in __call__
    return self.wsgi_app(environ, start_response)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/useridhere/repos/myproject/venv/lib/python3.11/site-packages/flask_socketio/__init__.py", line 43, in __call__
    return super(_SocketIOMiddleware, self).__call__(environ,
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/useridhere/repos/myproject/venv/lib/python3.11/site-packages/engineio/middleware.py", line 74, in __call__
    return self.wsgi_app(environ, start_response)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/useridhere/repos/myproject/venv/lib/python3.11/site-packages/flask/app.py", line 1458, in wsgi_app
    response = self.handle_exception(e)
               ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/useridhere/repos/myproject/venv/lib/python3.11/site-packages/flask_cors/extension.py", line 176, in wrapped_function
    return cors_after_request(app.make_response(f(*args, **kwargs)))
                                                ^^^^^^^^^^^^^^^^^^^^
  File "/Users/useridhere/repos/myproject/venv/lib/python3.11/site-packages/flask/app.py", line 1455, in wsgi_app
    response = self.full_dispatch_request()
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/useridhere/repos/myproject/venv/lib/python3.11/site-packages/flask/app.py", line 869, in full_dispatch_request
    rv = self.handle_user_exception(e)
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/useridhere/repos/myproject/venv/lib/python3.11/site-packages/flask_cors/extension.py", line 176, in wrapped_function
    return cors_after_request(app.make_response(f(*args, **kwargs)))
                                                ^^^^^^^^^^^^^^^^^^^^
  File "/Users/useridhere/repos/myproject/venv/lib/python3.11/site-packages/flask/app.py", line 867, in full_dispatch_request
    rv = self.dispatch_request()
         ^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/useridhere/repos/myproject/venv/lib/python3.11/site-packages/flask/app.py", line 852, in dispatch_request
    return self.ensure_sync(self.view_functions[rule.endpoint])(**view_args)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/useridhere/repos/myproject/app.py", line 93, in create_archive
    worker.run(
  File "/Users/useridhere/repos/myproject/api/utilities.py", line 13, in wrapper
    result = func(*args, **kwargs)
             ^^^^^^^^^^^^^^^^^^^^^
  File "/Users/useridhere/repos/myproject/api/worker.py", line 18, in run
    print(f'({i}/{num}): {f.result()}')
                          ^^^^^^^^^^^^^
  File "/Users/useridhere/miniconda3/lib/python3.11/concurrent/futures/_base.py", line 449, in result
    return self.__get_result()
           ^^^^^^^^^^^^^^^^^^^
  File "/Users/useridhere/miniconda3/lib/python3.11/concurrent/futures/_base.py", line 401, in __get_result
    raise self._exception
TypeError: create_archive() got an unexpected keyword argument 'extension'

Flask app:

app.py (the request.json parameters are strings, the values aren't important because of the error message I'm getting)

import os
from pathlib import Path
from flask import jsonify, request

from entry import app, app_config, socketio
from api.worker import Worker
from api.utilities import str2bool
from api.archives import create_archive


@socketio.on('create-logger', namespace='/archive')
@socketio.on('create-progress', namespace='/archive')
@app.route('/archive/create', methods=['POST'])
def create_archive():

    if not request.json['items']:
        return jsonify({
            'status': 500,
            'message': 'No folders were provided.',
            'invalid': []
        })

    # Confirm items are folders and not files
    invalid_folders = []
    for folder in request.json['items']:
        folder_path = folder['path']
        if not os.path.isdir(folder_path):
            invalid_folders.append(folder_path)

    if invalid_folders:
        return jsonify({
            'status': 500,
            'message': 'All items provided must be folders. Open the developer tools to see a list of the invalid items.',
            'invalid': invalid_folders
        })

    folders = [folder['path'] for folder in request.json['items']]

    valid_folders = []
    existing_archives = []
    for folder in folders:
        archive_filename = os.path.join(Path(folder).parent, Path(
            folder).name + request.json['extension'])

        if os.path.isfile(archive_filename):
            existing_archives.append(archive_filename)
        else:
            valid_folders.append(folder)

    socketio.emit('create-logger', 'Creating archive file(s)...',
                  namespace='/archive')

    worker = Worker(workers=5)

    worker.run(
        create_archive,
        valid_folders,
        extension=request.json['extension'],
        format=request.json['format'],
        remove_directory=str2bool(request.json['removeDir'])
    )

    socketio.emit('create-logger', 'Archiving complete',
                  namespace='/archive')

    return jsonify({
        'status': 200,
        'message': 'Archiving complete.',
        'exists': existing_archives
    })


if __name__ == "__main__":
    socketio.run(app, **app_config)

entry.py

import sys
from flask import Flask
from flask_cors import CORS
from flask_socketio import SocketIO


app = Flask(__name__)
app_config = {"port": sys.argv[1]}

# Developer mode uses app.py
if "app.py" in sys.argv[0]:
    # Update app config
    app_config["debug"] = True

    # CORS settings
    cors = CORS(
        app,
        resources={r"/*": {"origins": "http://localhost*"}},
    )

    # CORS headers
    app.config["CORS_HEADERS"] = "Content-Type"

socketio = SocketIO(app, cors_allowed_origins='*')

archives.py

import sys
sys.path.insert(0, '..')

import os  # NOQA
import tarfile  # NOQA
import fnmatch  # NOQA
import shutil  # NOQA
from pathlib import Path  # NOQA

from entry import socketio  # NOQA
from api.utilities import winapi_path  # NOQA


def create_archive(folder, extension='.tar.gz', format='PAX', remove_directory=False):
    """Create an archive file.

    Parameters
    ----------
    folder : str, path-like object
        Name of the folder to archive.

    extension : str, '.tar' or '.tar.gz' (default)
        File extension for archive file.

    format : str, 'PAX' (default) or 'GNU'
        Archive file format.

    remove_directory : bool, default is False
        Option to remove folder after the archive file is created.

    Returns
    -------
    dict
        Info containing filename and the total files included in the
        archive file.
    """
    socketio.emit('create-logger',
                  f'Creating archive for {folder}', namespace='/archive')

    # Build filename
    filename = os.path.join(Path(folder).parent, Path(folder).name + extension)

    # Initalize info
    info = {'filename': filename, 'total_files': 0}

    # Determine mode
    mode = 'x:' if extension.lower() == '.tar' else 'x:gz'

    # Set format
    fmt = tarfile.GNU_FORMAT if format.lower() == 'gnu' else tarfile.PAX_FORMAT

    with tarfile.open(filename, mode=mode, format=fmt) as tar:
        for dirpath, _, filenames in os.walk(folder):
            if filenames:
                for file in filenames:
                    path = Path(os.path.join(dirpath, file))

                    # arcname provides an alternate name that is relative
                    # to the folder being archived
                    altname = path.relative_to(Path(folder).parent)

                    # Enable long path
                    path = winapi_path(str(path))

                    tar.add(path, recursive=False, arcname=altname)

                    info['total_files'] += 1

    if remove_directory:
        shutil.rmtree(folder)

    socketio.emit('create-progress', 1, namespace='/archive')

    return info

Solution

Well, I finally figured it out. My issue was because my route name was create_archive and my function name was called create_archive. Changing the name of my route fixed my issue.

Old

@app.route('/archive/create', methods=['POST'])
def create_archive():
...

New

@app.route('/archive/create', methods=['POST'])
def archive_create():
...

Answered By - Simon1