[FIXED] How to simultaneously capture and display ipython (jupyter notebook) cell output?

Issue

How do you capture the cell output in jupyter notebook while simultaneously showing the results? I do not want to wait for everything to finish and then running a variable.show().

Solution

I adapted the following custom cell magic from the implementation of the built-in %%capture:

from io import StringIO
import sys

from IPython.core import magic_arguments
from IPython.core.magic import Magics, cell_magic, magics_class
from IPython.utils.capture import CapturedIO


class Tee(StringIO):
    def __init__(self, initial_value='', newline='\n', stream=None):
        self.stream = stream
        super().__init__(initial_value, newline)
    
    def write(self, data):
        if self.stream is not None:
            self.stream.write(data)
        
        super().write(data)


class capture_and_print_output(object):
    stdout = True
    stderr = True
    display = True
    
    def __init__(self, stdout=True, stderr=True, display=True):
        self.stdout = stdout
        self.stderr = stderr
        self.display = display
        self.shell = None
    
    def __enter__(self):
        from IPython.core.getipython import get_ipython
        from IPython.core.displaypub import CapturingDisplayPublisher
        from IPython.core.displayhook import CapturingDisplayHook
        
        self.sys_stdout = sys.stdout
        self.sys_stderr = sys.stderr
        
        if self.display:
            self.shell = get_ipython()
            if self.shell is None:
                self.save_display_pub = None
                self.display = False
        
        stdout = stderr = outputs = None
        if self.stdout:
            stdout = sys.stdout = Tee(stream=sys.stdout)
        if self.stderr:
            stderr = sys.stderr = Tee(stream=sys.stderr)
        if self.display:
            self.save_display_pub = self.shell.display_pub
            self.shell.display_pub = CapturingDisplayPublisher()
            outputs = self.shell.display_pub.outputs
            self.save_display_hook = sys.displayhook
            sys.displayhook = CapturingDisplayHook(shell=self.shell,
                                                   outputs=outputs)
        
        return CapturedIO(stdout, stderr, outputs)
    
    def __exit__(self, exc_type, exc_value, traceback):
        sys.stdout = self.sys_stdout
        sys.stderr = self.sys_stderr
        if self.display and self.shell:
            self.shell.display_pub = self.save_display_pub
            sys.displayhook = self.save_display_hook


@magics_class
class CustomMagics(Magics):
    @magic_arguments.magic_arguments()
    @magic_arguments.argument('output', type=str, default='', nargs='?')
    @magic_arguments.argument('--no-stderr', action='store_true')
    @magic_arguments.argument('--no-stdout', action='store_true')
    @magic_arguments.argument('--no-display', action='store_true')
    @cell_magic
    def tee(self, line, cell):
        args = magic_arguments.parse_argstring(self.tee, line)
        out = not args.no_stdout
        err = not args.no_stderr
        disp = not args.no_display
        with capture_and_print_output(out, err, disp) as io:
            self.shell.run_cell(cell)
        if args.output:
            self.shell.user_ns[args.output] = io

You need to register this cell magic, usually either in your startup profile or by defining it as an extension you load later, for example:

from IPython import get_ipython

get_ipython().register_magics(CustomMagics)

Now you can use it as follows:

>>> %%tee output
    print(42)
42
>>> output.show()
42

One thing to note: it will capture rich outputs as the original %%capture does, but it won't print them in the original cell. If that's important, you may find some way to adapt the above to include rich outputs, perhaps by subclassing CapturingDisplayPublisher/Hook.

Answered By - Seb