From 0c1a053d40c256f2023d2612c9adb88ac32c8fed Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 7 Aug 2021 05:35:06 -0400
Subject: [PATCH 01/32] Add long_callback decorator

---
 dash/dash.py                                  | 156 +++++++++++++++++-
 dash/long_callback/__init__.py                |   2 +
 dash/long_callback/managers/__init__.py       |  50 ++++++
 dash/long_callback/managers/celery_manager.py |  92 +++++++++++
 .../managers/diskcache_manager.py             | 117 +++++++++++++
 .../callbacks/test_basic_callback.py          |  37 +++++
 6 files changed, 453 insertions(+), 1 deletion(-)
 create mode 100644 dash/long_callback/__init__.py
 create mode 100644 dash/long_callback/managers/__init__.py
 create mode 100644 dash/long_callback/managers/celery_manager.py
 create mode 100644 dash/long_callback/managers/diskcache_manager.py

diff --git a/dash/dash.py b/dash/dash.py
index 73b150ee90..d0d18aaf4f 100644
--- a/dash/dash.py
+++ b/dash/dash.py
@@ -30,6 +30,8 @@
     handle_callback_args,
     handle_grouped_callback_args,
     Output,
+    State,
+    Input,
 )
 from .development.base_component import ComponentRegistry
 from .exceptions import PreventUpdate, InvalidResourceError, ProxyError
@@ -60,6 +62,7 @@
     grouping_len,
 )
 
+
 _flask_compress_version = parse_version(get_distribution("flask-compress").version)
 
 # Add explicit mapping for map files
@@ -399,6 +402,7 @@ def __init__(
         self._layout = None
         self._layout_is_function = False
         self.validation_layout = None
+        self._extra_components = []
 
         self._setup_dev_tools()
         self._hot_reload = AttributeDict(
@@ -410,6 +414,7 @@ def __init__(
         )
 
         self._assets_files = []
+        self._long_callback_count = 0
 
         self.logger = logging.getLogger(name)
         self.logger.addHandler(logging.StreamHandler(stream=sys.stdout))
@@ -490,7 +495,17 @@ def layout(self):
         return self._layout
 
     def _layout_value(self):
-        return self._layout() if self._layout_is_function else self._layout
+        layout = self._layout() if self._layout_is_function else self._layout
+
+        # Add extra hidden components
+        if self._extra_components:
+            if not hasattr(layout, "children"):
+                setattr(layout, "children", [])
+            for c in self._extra_components:
+                if c not in layout.children:
+                    layout.children.append(c)
+
+        return layout
 
     @layout.setter
     def layout(self, value):
@@ -1114,6 +1129,145 @@ def add_context(*args, **kwargs):
 
         return wrap_func
 
+    def long_callback(self, callback_manager, *_args, **_kwargs):
+        from . import callback_context  # pylint: disable=import-outside-toplevel
+        import dash_core_components as dcc  # pylint: disable=import-outside-toplevel
+
+        # Extract special long_callback kwargs
+        running = _kwargs.pop("running", ())
+        cancel = _kwargs.pop("cancel", ())
+        progress = _kwargs.pop("progress", ())
+        progress_default = _kwargs.pop("progress_default", None)
+        interval_time = _kwargs.pop("interval", 1000)
+
+        # Parse remaining args just like app.callback
+        (
+            output,
+            flat_inputs,
+            flat_state,
+            inputs_state_indices,
+            prevent_initial_call,
+        ) = handle_grouped_callback_args(_args, _kwargs)
+        inputs_and_state = flat_inputs + flat_state
+        args_deps = map_grouping(lambda i: inputs_and_state[i], inputs_state_indices)
+
+        # Get unique id for this long_callback definition.  This increment is not
+        # thread safe, but it doesn't need to be because callback definitions
+        # happen on the main thread before the app starts
+        self._long_callback_count += 1
+        long_callback_id = self._long_callback_count
+
+        # Create Interval and Store for long callback and add them to the app's
+        # _extra_components list
+        interval_id = f"_long_callback_interval_{long_callback_id}"
+        interval_component = dcc.Interval(id=interval_id, interval=interval_time)
+        store_id = f"_long_callback_store_{long_callback_id}"
+        store_component = dcc.Store(id=store_id, data=dict())
+        self._extra_components.extend([interval_component, store_component])
+
+        # Compute full component plus property name for the cancel dependencies
+        cancel_prop_ids = tuple(
+            ".".join([dep.component_id, dep.component_property]) for dep in cancel
+        )
+
+        def wrapper(fn):
+            background_fn = callback_manager.make_background_fn(
+                fn, progress=bool(progress)
+            )
+
+            def callback(_triggers, user_store_data, user_callback_args):
+                result_key = user_store_data.get("cache_result_key", None)
+                if result_key is None:
+                    # Build result cache key from inputs
+                    result_key = callback_manager.build_cache_key(
+                        fn, user_callback_args
+                    )
+                    user_store_data["cache_result_key"] = result_key
+
+                should_cancel = any(
+                    [
+                        trigger["prop_id"] in cancel_prop_ids
+                        for trigger in callback_context.triggered
+                    ]
+                )
+
+                # Compute grouping of values to set the progress component's to
+                # when cleared
+                if progress_default is None:
+                    clear_progress = (
+                        map_grouping(lambda x: None, progress) if progress else ()
+                    )
+                else:
+                    clear_progress = progress_default
+
+                if should_cancel and result_key is not None:
+                    if callback_manager.has_future(result_key):
+                        callback_manager.delete_future(result_key)
+                    return dict(
+                        user_callback_output=map_grouping(lambda x: no_update, output),
+                        interval_disabled=True,
+                        in_progress=[val for (_, _, val) in running],
+                        progress=clear_progress,
+                        user_store_data=user_store_data,
+                    )
+
+                progress_value = callback_manager.get_progress(result_key)
+
+                if callback_manager.result_ready(result_key):
+                    result = callback_manager.get_result(result_key)
+                    # Clear result key
+                    user_store_data["cache_result_key"] = None
+                    return dict(
+                        user_callback_output=result,
+                        interval_disabled=True,
+                        in_progress=[val for (_, _, val) in running],
+                        progress=clear_progress,
+                        user_store_data=user_store_data,
+                    )
+                elif progress_value:
+                    return dict(
+                        user_callback_output=map_grouping(lambda x: no_update, output),
+                        interval_disabled=False,
+                        in_progress=[val for (_, val, _) in running],
+                        progress=progress_value or {},
+                        user_store_data=user_store_data,
+                    )
+                else:
+                    callback_manager.terminate_unhealthy_future(result_key)
+                    if not callback_manager.has_future(result_key):
+                        callback_manager.call_and_register_background_fn(
+                            result_key, background_fn, user_callback_args
+                        )
+
+                    return dict(
+                        user_callback_output=map_grouping(lambda x: no_update, output),
+                        interval_disabled=False,
+                        in_progress=[val for (_, val, _) in running],
+                        progress=clear_progress,
+                        user_store_data=user_store_data,
+                    )
+
+            return self.callback(
+                inputs=dict(
+                    _triggers=dict(
+                        n_intervals=Input(interval_id, "n_intervals"),
+                        cancel=cancel,
+                    ),
+                    user_store_data=State(store_id, "data"),
+                    user_callback_args=args_deps,
+                ),
+                output=dict(
+                    user_callback_output=output,
+                    interval_disabled=Output(interval_id, "disabled"),
+                    in_progress=[dep for (dep, _, _) in running],
+                    progress=progress,
+                    user_store_data=Output(store_id, "data"),
+                ),
+                prevent_initial_call=prevent_initial_call,
+            )(callback)
+
+        return wrapper
+
     def dispatch(self):
         body = flask.request.get_json()
         flask.g.inputs_list = inputs = body.get(  # pylint: disable=assigning-non-slot
diff --git a/dash/long_callback/__init__.py b/dash/long_callback/__init__.py
new file mode 100644
index 0000000000..54af16e92c
--- /dev/null
+++ b/dash/long_callback/__init__.py
@@ -0,0 +1,2 @@
+from .managers.celery_manager import CeleryLongCallbackManager  # noqa: F401,E402
+from .managers.diskcache_manager import DiskcacheLongCallbackManager  # noqa: F401,E402
diff --git a/dash/long_callback/managers/__init__.py b/dash/long_callback/managers/__init__.py
new file mode 100644
index 0000000000..62d20c27ad
--- /dev/null
+++ b/dash/long_callback/managers/__init__.py
@@ -0,0 +1,50 @@
+from abc import ABC
+import inspect
+import hashlib
+
+
+class BaseLongCallbackManager(ABC):
+    def __init__(self, cache_by):
+        if cache_by is not None and not isinstance(cache_by, list):
+            cache_by = [cache_by]
+
+        self.cache_by = cache_by
+
+    def delete_future(self, key):
+        raise NotImplementedError
+
+    def terminate_unhealthy_future(self, key):
+        raise NotImplementedError
+
+    def has_future(self, key):
+        raise NotImplementedError
+
+    def get_future(self, key, default=None):
+        raise NotImplementedError
+
+    def make_background_fn(self, fn, progress):
+        raise NotImplementedError
+
+    def call_and_register_background_fn(self, key, background_fn, args):
+        raise NotImplementedError
+
+    def get_progress(self, key):
+        raise NotImplementedError
+
+    def result_ready(self, key):
+        raise NotImplementedError
+
+    def get_result(self, key):
+        raise NotImplementedError
+
+    def build_cache_key(self, fn, args):
+        fn_source = inspect.getsource(fn)
+        hash_dict = dict(args=args, fn_source=fn_source)
+
+        if self.cache_by is not None:
+            # Caching enabled
+            for i, cache_item in enumerate(self.cache_by):
+                # Call cache function
+                hash_dict[f"cache_key_{i}"] = cache_item()
+
+        return hashlib.sha1(str(hash_dict).encode("utf-8")).hexdigest()
diff --git a/dash/long_callback/managers/celery_manager.py b/dash/long_callback/managers/celery_manager.py
new file mode 100644
index 0000000000..e1d2aeea34
--- /dev/null
+++ b/dash/long_callback/managers/celery_manager.py
@@ -0,0 +1,92 @@
+import json
+from dash_labs.plugins.long_callback.managers import BaseLongCallbackManager
+from _plotly_utils.utils import PlotlyJSONEncoder
+
+
+class CeleryLongCallbackManager(BaseLongCallbackManager):
+    def __init__(self, celery_app, cache_by=None):
+        super().__init__(cache_by)
+        self.celery_app = celery_app
+        self.callback_futures = dict()
+
+    def init(self, app):
+        pass
+
+    def delete_future(self, key):
+        if key in self.callback_futures:
+            future = self.callback_futures.pop(key)
+            self.celery_app.control.terminate(future.task_id)
+            return True
+        return False
+
+    def terminate_unhealthy_future(self, key):
+        if key in self.callback_futures:
+            future = self.callback_futures[key]
+            if future.status != "PENDING":
+                return self.delete_future(key)
+        return False
+
+    def has_future(self, key):
+        return key in self.callback_futures
+
+    def get_future(self, key, default=None):
+        return self.callback_futures.get(key, default)
+
+    def make_background_fn(self, fn, progress=False):
+        return make_celery_fn(fn, self.celery_app, progress)
+
+    def call_and_register_background_fn(self, key, background_fn, *args, **kwargs):
+        future = background_fn.delay(*args, **kwargs)
+        self.callback_futures[key] = future
+
+    def get_progress(self, key):
+        future = self.get_future(key)
+        if future is not None:
+            progress_info = future.info if future.state == "PROGRESS" else None
+            if progress_info is not None:
+                return json.loads(progress_info["progress_value"])
+
+        return None
+
+    def result_ready(self, key):
+        future = self.get_future(key)
+        if future:
+            return future.ready()
+        else:
+            return False
+
+    def get_result(self, key):
+        future = self.callback_futures.get(key, None)
+        if future:
+            result = future.get(timeout=1)
+            # Clear result if not caching
+            if self.cache_by is None:
+                self.delete_future(key)
+            return result
+        else:
+            return None
+
+
+def make_celery_fn(user_fn, celery_app, progress):
+    @celery_app.task(bind=True)
+    def _celery_fn(self, user_callback_args):
+        def _set_progress(progress_value):
+            # JSON serialize with PlotlyEncoder
+            self.update_state(
+                state="PROGRESS",
+                meta={
+                    "progress_value": json.dumps(progress_value, cls=PlotlyJSONEncoder)
+                },
+            )
+
+        maybe_progress = [_set_progress] if progress else []
+        if isinstance(user_callback_args, dict):
+            user_callback_output = user_fn(*maybe_progress, **user_callback_args)
+        elif isinstance(user_callback_args, list):
+            user_callback_output = user_fn(*maybe_progress, *user_callback_args)
+        else:
+            user_callback_output = user_fn(*maybe_progress, user_callback_args)
+
+        return user_callback_output
+
+    return _celery_fn
diff --git a/dash/long_callback/managers/diskcache_manager.py b/dash/long_callback/managers/diskcache_manager.py
new file mode 100644
index 0000000000..71030c87d8
--- /dev/null
+++ b/dash/long_callback/managers/diskcache_manager.py
@@ -0,0 +1,117 @@
+import platform
+from . import BaseLongCallbackManager
+
+
+class DiskcacheLongCallbackManager(BaseLongCallbackManager):
+    def __init__(self, cache, cache_by=None, expire=None):
+        import diskcache  # pylint: disable=import-outside-toplevel
+
+        if not isinstance(cache, diskcache.Cache):
+            raise ValueError("First argument must be a diskcache.Cache object")
+        super().__init__(cache_by)
+
+        # Handle process class import
+        if platform.system() == "Windows":
+            try:
+                from multiprocess import (  # pylint: disable=import-outside-toplevel
+                    Process,
+                )
+            except ImportError:
+                raise ImportError(
+                    """\
+    When running on Windows, the long_callback decorator requires the
+    multiprocess package which can be install using pip...
+
+        $ pip install multiprocess
+
+    or conda.
+
+        $ conda install -c conda-forge multiprocess\n"""
+                )
+        else:
+            from multiprocessing import (  # pylint: disable=import-outside-toplevel
+                Process,
+            )
+
+        self.Process = Process
+        self.cache = cache
+        self.callback_futures = dict()
+        self.expire = expire
+
+    def delete_future(self, key):
+        if key in self.callback_futures:
+            future = self.callback_futures.pop(key, None)
+            if future:
+                future.kill()
+                future.join()
+                return True
+        return False
+
+    def clear_cache_entry(self, key):
+        self.cache.delete(key)
+
+    def terminate_unhealthy_future(self, key):
+        return False
+
+    def has_future(self, key):
+        return self.callback_futures.get(key, None) is not None
+
+    def get_future(self, key, default=None):
+        return self.callback_futures.get(key, default)
+
+    def make_background_fn(self, fn, progress=False):
+        return make_update_cache(fn, self.cache, progress, self.expire)
+
+    @staticmethod
+    def _make_progress_key(key):
+        return key + "-progress"
+
+    def call_and_register_background_fn(self, key, background_fn, args):
+        self.delete_future(key)
+        future = self.Process(
+            target=background_fn, args=(key, self._make_progress_key(key), args)
+        )
+        future.start()
+        self.callback_futures[key] = future
+
+    def get_progress(self, key):
+        future = self.get_future(key)
+        if future is not None:
+            progress_key = self._make_progress_key(key)
+            return self.cache.get(progress_key)
+        return None
+
+    def result_ready(self, key):
+        return self.cache.get(key) not in (None, "__undefined__")
+
+    def get_result(self, key):
+        # Get result value
+        result = self.cache.get(key)
+        if result == "__undefined__":
+            result = None
+
+        # Clear result if not caching
+        if self.cache_by is None and result is not None:
+            self.clear_cache_entry(key)
+
+        # Always delete_future (even if we didn't clear cache) so that we can
+        # handle the case where cache entry is cleared externally.
+        self.delete_future(key)
+        return result
+
+
+def make_update_cache(fn, cache, progress, expire):
+    def _callback(result_key, progress_key, user_callback_args):
+        def _set_progress(progress_value):
+            cache.set(progress_key, progress_value)
+
+        maybe_progress = [_set_progress] if progress else []
+        if isinstance(user_callback_args, dict):
+            user_callback_output = fn(*maybe_progress, **user_callback_args)
+        elif isinstance(user_callback_args, (list, tuple)):
+            user_callback_output = fn(*maybe_progress, *user_callback_args)
+        else:
+            user_callback_output = fn(*maybe_progress, user_callback_args)
+        cache.set(result_key, user_callback_output, expire=expire)
+
+    return _callback
diff --git a/tests/integration/callbacks/test_basic_callback.py b/tests/integration/callbacks/test_basic_callback.py
index c486db2114..12c2b2f356 100644
--- a/tests/integration/callbacks/test_basic_callback.py
+++ b/tests/integration/callbacks/test_basic_callback.py
@@ -701,3 +701,40 @@ def follower_output(v):
     assert not dash_duo.redux_state_is_loading
 
     assert dash_duo.get_logs() == []
+
+
+def test_cbsc016_extra_components_callback(dash_duo):
+    lock = Lock()
+
+    app = dash.Dash(__name__)
+    app._extra_components.append(dcc.Store(id="extra-store", data=123))
+
+    app.layout = html.Div(
+        [
+            dcc.Input(id="input", value="initial value"),
+            html.Div(html.Div([1.5, None, "string", html.Div(id="output-1")])),
+        ]
+    )
+    store_data = Value("i", 0)
+
+    @app.callback(
+        Output("output-1", "children"),
+        [Input("input", "value"), Input("extra-store", "data")],
+    )
+    def update_output(value, data):
+        with lock:
+            store_data.value = data
+            return value
+
+    dash_duo.start_server(app)
+
+    assert dash_duo.find_element("#output-1").text == "initial value"
+
+    input_ = dash_duo.find_element("#input")
+    dash_duo.clear_input(input_)
+    input_.send_keys("A")
+
+    wait.until(lambda: dash_duo.find_element("#output-1").text == "A", 2)
+
+    assert store_data.value == 123
+    assert dash_duo.get_logs() == []

From 5a004e7b96a28076ed503a58dd8bdf809bd56818 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Wed, 11 Aug 2021 12:08:26 -0400
Subject: [PATCH 02/32] Rework long_callback to avoid disabling interval until
 all requests are handled

---
 dash/dash.py | 43 +++++++++++++++++++++++--------------------
 1 file changed, 23 insertions(+), 20 deletions(-)

diff --git a/dash/dash.py b/dash/dash.py
index d0d18aaf4f..7169638147 100644
--- a/dash/dash.py
+++ b/dash/dash.py
@@ -1176,15 +1176,13 @@ def wrapper(fn):
             )
 
             def callback(_triggers, user_store_data, user_callback_args):
-                result_key = user_store_data.get("cache_result_key", None)
-                if result_key is None:
-                    # Build result cache key from inputs
-                    result_key = callback_manager.build_cache_key(
-                        fn, user_callback_args
-                    )
-                    user_store_data["cache_result_key"] = result_key
+                # Build result cache key from inputs
+                pending_key = callback_manager.build_cache_key(
+                    fn, user_callback_args
+                )
+                current_key = user_store_data.get("current_key", None)
 
-                should_cancel = any(
+                should_cancel = pending_key == current_key or any(
                     [
                         trigger["prop_id"] in cancel_prop_ids
                         for trigger in callback_context.triggered
@@ -1200,9 +1198,12 @@ def callback(_triggers, user_store_data, user_callback_args):
                 else:
                     clear_progress = progress_default
 
-                if should_cancel and result_key is not None:
-                    if callback_manager.has_future(result_key):
-                        callback_manager.delete_future(result_key)
+                if should_cancel:
+                    user_store_data["current_key"] = None
+                    user_store_data["pending_key"] = None
+
+                    if pending_key and callback_manager.has_future(pending_key):
+                        callback_manager.delete_future(pending_key)
                     return dict(
                         user_callback_output=map_grouping(lambda x: no_update, output),
                         interval_disabled=True,
@@ -1211,15 +1212,16 @@ def callback(_triggers, user_store_data, user_callback_args):
                         user_store_data=user_store_data,
                     )
 
-                progress_value = callback_manager.get_progress(result_key)
+                progress_value = callback_manager.get_progress(pending_key)
 
-                if callback_manager.result_ready(result_key):
-                    result = callback_manager.get_result(result_key)
-                    # Clear result key
-                    user_store_data["cache_result_key"] = None
+                if callback_manager.result_ready(pending_key):
+                    result = callback_manager.get_result(pending_key)
+                    # Set current key (hash of data stored in client)
+                    # to pending key (hash of data requested by client)
+                    user_store_data["current_key"] = pending_key
                     return dict(
                         user_callback_output=result,
-                        interval_disabled=True,
+                        interval_disabled=False,
                         in_progress=[val for (_, _, val) in running],
                         progress=clear_progress,
                         user_store_data=user_store_data,
@@ -1233,10 +1235,11 @@ def callback(_triggers, user_store_data, user_callback_args):
                         user_store_data=user_store_data,
                     )
                 else:
-                    callback_manager.terminate_unhealthy_future(result_key)
-                    if not callback_manager.has_future(result_key):
+                    user_store_data["pending_key"] = pending_key
+                    callback_manager.terminate_unhealthy_future(pending_key)
+                    if not callback_manager.has_future(pending_key):
                         callback_manager.call_and_register_background_fn(
-                            result_key, background_fn, user_callback_args
+                            pending_key, background_fn, user_callback_args
                         )
 
                     return dict(

From f5dfd6bc37912315813af89a6e302e6b7810e864 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Wed, 11 Aug 2021 14:38:21 -0400
Subject: [PATCH 03/32] Add long_callback tests

---
 dash/dash.py                                  |   4 +-
 dash/testing/plugin.py                        |  11 +
 requires-testing.txt                          |   1 +
 .../callbacks/test_long_callback.py           | 311 ++++++++++++++++++
 4 files changed, 324 insertions(+), 3 deletions(-)
 create mode 100644 tests/integration/callbacks/test_long_callback.py

diff --git a/dash/dash.py b/dash/dash.py
index 7169638147..2aade2176b 100644
--- a/dash/dash.py
+++ b/dash/dash.py
@@ -1177,9 +1177,7 @@ def wrapper(fn):
 
             def callback(_triggers, user_store_data, user_callback_args):
                 # Build result cache key from inputs
-                pending_key = callback_manager.build_cache_key(
-                    fn, user_callback_args
-                )
+                pending_key = callback_manager.build_cache_key(fn, user_callback_args)
                 current_key = user_store_data.get("current_key", None)
 
                 should_cancel = pending_key == current_key or any(
diff --git a/dash/testing/plugin.py b/dash/testing/plugin.py
index 69699747c6..ce61805b32 100644
--- a/dash/testing/plugin.py
+++ b/dash/testing/plugin.py
@@ -185,3 +185,14 @@ def dashjl(request, dashjl_server, tmpdir):
         pause=request.config.getoption("pause"),
     ) as dc:
         yield dc
+
+
+@pytest.fixture
+def diskcache_manager():
+    from dash.long_callback import (  # pylint: disable=import-outside-toplevel
+        DiskcacheLongCallbackManager,
+    )
+    import diskcache  # pylint: disable=import-outside-toplevel
+
+    cache = diskcache.Cache()
+    return DiskcacheLongCallbackManager(cache)
diff --git a/requires-testing.txt b/requires-testing.txt
index 9885e9b3c1..c0380dcfdf 100644
--- a/requires-testing.txt
+++ b/requires-testing.txt
@@ -8,3 +8,4 @@ cryptography<3.4;python_version<"3.7"
 requests[security]>=2.21.0
 beautifulsoup4>=4.8.2
 waitress>=1.4.4
+diskcache=>5.2.1
diff --git a/tests/integration/callbacks/test_long_callback.py b/tests/integration/callbacks/test_long_callback.py
new file mode 100644
index 0000000000..762c8a7fbb
--- /dev/null
+++ b/tests/integration/callbacks/test_long_callback.py
@@ -0,0 +1,311 @@
+from multiprocessing import Lock, Value
+import time
+
+import dash_core_components as dcc
+import dash_html_components as html
+import dash
+from dash.dependencies import Input, Output, State
+
+
+def test_lcb001_fast_input(dash_duo, diskcache_manager):
+    """
+    Make sure that we settle to the correct final value when handling rapid inputs
+    """
+    lock = Lock()
+
+    app = dash.Dash(__name__)
+    app.layout = html.Div(
+        [
+            dcc.Input(id="input", value="initial value"),
+            html.Div(html.Div([1.5, None, "string", html.Div(id="output-1")])),
+        ]
+    )
+
+    @app.long_callback(
+        diskcache_manager,
+        Output("output-1", "children"),
+        [Input("input", "value")],
+        interval=500,
+    )
+    def update_output(value):
+        time.sleep(0.1)
+        return value
+
+    dash_duo.start_server(app)
+    dash_duo.wait_for_text_to_equal("#output-1", "initial value", 2)
+    input_ = dash_duo.find_element("#input")
+    dash_duo.clear_input(input_)
+
+    for key in "hello world":
+        with lock:
+            input_.send_keys(key)
+
+    dash_duo.wait_for_text_to_equal("#output-1", "hello world", 4)
+    assert not dash_duo.redux_state_is_loading
+    assert dash_duo.get_logs() == []
+
+
+def test_lcb002_long_callback_running(dash_duo, diskcache_manager):
+    app = dash.Dash(__name__)
+    app.layout = html.Div(
+        [
+            html.Button(id="button-1", children="Click Here", n_clicks=0),
+            html.Div(id="status", children="Finished"),
+            html.Div(id="result", children="Not clicked"),
+        ]
+    )
+
+    @app.long_callback(
+        diskcache_manager,
+        Output("result", "children"),
+        [Input("button-1", "n_clicks")],
+        running=[(Output("status", "children"), "Running", "Finished")],
+        interval=500,
+    )
+    def update_output(n_clicks):
+        time.sleep(2)
+        return f"Clicked {n_clicks} time(s)"
+
+    dash_duo.start_server(app)
+    dash_duo.wait_for_text_to_equal("#result", "Clicked 0 time(s)", 4)
+    dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+
+    # Click button and check that status has changed to "Running"
+    dash_duo.find_element("#button-1").click()
+    dash_duo.wait_for_text_to_equal("#status", "Running", 2)
+
+    # Wait for calculation to finish, then check that status is "Finished"
+    dash_duo.wait_for_text_to_equal("#result", "Clicked 1 time(s)", 4)
+    dash_duo.wait_for_text_to_equal("#status", "Finished", 2)
+
+    # Click button twice and check that status has changed to "Running"
+    dash_duo.find_element("#button-1").click()
+    dash_duo.find_element("#button-1").click()
+    dash_duo.wait_for_text_to_equal("#status", "Running", 2)
+
+    # Wait for calculation to finish, then check that status is "Finished"
+    dash_duo.wait_for_text_to_equal("#result", "Clicked 3 time(s)", 4)
+    dash_duo.wait_for_text_to_equal("#status", "Finished", 2)
+
+    assert not dash_duo.redux_state_is_loading
+    assert dash_duo.get_logs() == []
+
+
+def test_lcb003_long_callback_running_cancel(dash_duo, diskcache_manager):
+    lock = Lock()
+    app = dash.Dash(__name__)
+    app.layout = html.Div(
+        [
+            dcc.Input(id="input", value="initial value"),
+            html.Button(id="run-button", children="Run"),
+            html.Button(id="cancel-button", children="Cancel"),
+            html.Div(id="status", children="Finished"),
+            html.Div(id="result", children="No results"),
+        ]
+    )
+
+    @app.long_callback(
+        diskcache_manager,
+        Output("result", "children"),
+        [Input("run-button", "n_clicks"), State("input", "value")],
+        running=[(Output("status", "children"), "Running", "Finished")],
+        cancel=[Input("cancel-button", "n_clicks")],
+        interval=500,
+    )
+    def update_output(n_clicks, value):
+        time.sleep(2)
+        return f"Processed '{value}'"
+
+    dash_duo.start_server(app)
+    dash_duo.wait_for_text_to_equal("#result", "Processed 'initial value'", 4)
+    dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+
+    # Update input text box
+    input_ = dash_duo.find_element("#input")
+    dash_duo.clear_input(input_)
+
+    for key in "hello world":
+        with lock:
+            input_.send_keys(key)
+
+    # Click run button and check that status has changed to "Running"
+    dash_duo.find_element("#run-button").click()
+    dash_duo.wait_for_text_to_equal("#status", "Running", 2)
+
+    # Then click Cancel button and make sure that the status changes to finish
+    # without update result
+    dash_duo.find_element("#cancel-button").click()
+    dash_duo.wait_for_text_to_equal("#result", "Processed 'initial value'", 4)
+    dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+
+    # Click run button again, and let it finish
+    dash_duo.find_element("#run-button").click()
+    dash_duo.wait_for_text_to_equal("#status", "Running", 2)
+    dash_duo.wait_for_text_to_equal("#result", "Processed 'hello world'", 4)
+    dash_duo.wait_for_text_to_equal("#status", "Finished", 2)
+
+    assert not dash_duo.redux_state_is_loading
+    assert dash_duo.get_logs() == []
+
+
+def test_lcb004_long_callback_progress(dash_duo, diskcache_manager):
+    app = dash.Dash(__name__)
+    app.layout = html.Div(
+        [
+            dcc.Input(id="input", value="hello, world"),
+            html.Button(id="run-button", children="Run"),
+            html.Button(id="cancel-button", children="Cancel"),
+            html.Div(id="status", children="Finished"),
+            html.Div(id="result", children="No results"),
+        ]
+    )
+
+    @app.long_callback(
+        diskcache_manager,
+        Output("result", "children"),
+        [Input("run-button", "n_clicks"), State("input", "value")],
+        progress=Output("status", "children"),
+        progress_default="Finished",
+        cancel=[Input("cancel-button", "n_clicks")],
+        interval=500,
+        prevent_initial_callback=True,
+    )
+    def update_output(set_progress, n_clicks, value):
+        for i in range(4):
+            set_progress(f"Progress {i}/4")
+            time.sleep(1)
+        return f"Processed '{value}'"
+
+    dash_duo.start_server(app)
+    dash_duo.wait_for_text_to_equal("#result", "No results", 2)
+    dash_duo.wait_for_text_to_equal("#status", "Finished", 2)
+
+    # Click run button and check that status eventually cycles to 2/4
+    dash_duo.find_element("#run-button").click()
+    dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 4)
+
+    # Then click Cancel button and make sure that the status changes to finish
+    # without updating result
+    dash_duo.find_element("#cancel-button").click()
+    dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+    dash_duo.wait_for_text_to_equal("#result", "No results", 4)
+
+    # Click run button and allow callback to finish
+    dash_duo.find_element("#run-button").click()
+    dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 4)
+    dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+    dash_duo.wait_for_text_to_equal("#result", "Processed 'hello, world'", 2)
+
+    # Click run button again with same input.
+    # without caching, this should rerun callback and display progress
+    dash_duo.find_element("#run-button").click()
+    dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 4)
+    dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+    dash_duo.wait_for_text_to_equal("#result", "Processed 'hello, world'", 2)
+
+    assert not dash_duo.redux_state_is_loading
+    assert dash_duo.get_logs() == []
+
+
+def test_lcb005_long_callback_caching(dash_duo, diskcache_manager):
+    lock = Lock()
+
+    # Control return value of cache_by function using multiprocessing value
+    cache_key = Value("i", 0)
+
+    def cache_fn():
+        return cache_key.value
+
+    diskcache_manager.cache_by = [cache_fn]
+
+    app = dash.Dash(__name__)
+    app.layout = html.Div(
+        [
+            dcc.Input(id="input", value="AAA"),
+            html.Button(id="run-button", children="Run", n_clicks=0),
+            html.Div(id="status", children="Finished"),
+            html.Div(id="result", children="No results"),
+        ]
+    )
+
+    @app.long_callback(
+        diskcache_manager,
+        [Output("result", "children"), Output("run-button", "n_clicks")],
+        [Input("run-button", "n_clicks"), State("input", "value")],
+        progress=Output("status", "children"),
+        progress_default="Finished",
+        interval=500,
+        prevent_initial_callback=True,
+    )
+    def update_output(set_progress, _n_clicks, value):
+        for i in range(4):
+            set_progress(f"Progress {i}/4")
+            time.sleep(2)
+        return f"Result for '{value}'", 0
+
+    dash_duo.start_server(app)
+    dash_duo.wait_for_text_to_equal("#result", "No results", 2)
+    dash_duo.wait_for_text_to_equal("#status", "Finished", 2)
+
+    # Click run button and check that status eventually cycles to 2/4
+    dash_duo.find_element("#run-button").click()
+    dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 6)
+    dash_duo.wait_for_text_to_equal("#status", "Finished", 6)
+    dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 1)
+
+    # Update input text box to BBB
+    input_ = dash_duo.find_element("#input")
+    dash_duo.clear_input(input_)
+    for key in "BBB":
+        with lock:
+            input_.send_keys(key)
+
+    # Click run button and check that status eventually cycles to 2/4
+    dash_duo.find_element("#run-button").click()
+    dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 6)
+    dash_duo.wait_for_text_to_equal("#status", "Finished", 6)
+    dash_duo.wait_for_text_to_equal("#result", "Result for 'BBB'", 1)
+
+    # Update input text box back to AAA
+    input_ = dash_duo.find_element("#input")
+    dash_duo.clear_input(input_)
+    for key in "AAA":
+        with lock:
+            input_.send_keys(key)
+
+    # Click run button and this time the cached result is used,
+    # So we can get the result right away
+    dash_duo.find_element("#run-button").click()
+    dash_duo.wait_for_text_to_equal("#status", "Finished", 1)
+    dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 1)
+
+    # Update input text box back to BBB
+    input_ = dash_duo.find_element("#input")
+    dash_duo.clear_input(input_)
+    for key in "BBB":
+        with lock:
+            input_.send_keys(key)
+
+    # Click run button and this time the cached result is used,
+    # So we can get the result right away
+    dash_duo.find_element("#run-button").click()
+    dash_duo.wait_for_text_to_equal("#status", "Finished", 1)
+    dash_duo.wait_for_text_to_equal("#result", "Result for 'BBB'", 1)
+
+    # Update input text box back to AAA
+    input_ = dash_duo.find_element("#input")
+    dash_duo.clear_input(input_)
+    for key in "AAA":
+        with lock:
+            input_.send_keys(key)
+
+    # Change cache key
+    cache_key.value = 1
+
+    dash_duo.find_element("#run-button").click()
+    dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 6)
+    dash_duo.wait_for_text_to_equal("#status", "Finished", 6)
+    dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 1)
+
+    assert not dash_duo.redux_state_is_loading
+    assert dash_duo.get_logs() == []

From 0b106242a7944fdab4f7714448043657378a3530 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Wed, 11 Aug 2021 14:57:35 -0400
Subject: [PATCH 04/32] Have the diskcache long_callback manager rely on
 multiprocess on all platforms

---
 .../managers/diskcache_manager.py             | 42 ++++++++-----------
 requires-testing.txt                          |  3 +-
 2 files changed, 19 insertions(+), 26 deletions(-)

diff --git a/dash/long_callback/managers/diskcache_manager.py b/dash/long_callback/managers/diskcache_manager.py
index 71030c87d8..353ee51159 100644
--- a/dash/long_callback/managers/diskcache_manager.py
+++ b/dash/long_callback/managers/diskcache_manager.py
@@ -1,38 +1,30 @@
-import platform
 from . import BaseLongCallbackManager
 
 
 class DiskcacheLongCallbackManager(BaseLongCallbackManager):
     def __init__(self, cache, cache_by=None, expire=None):
-        import diskcache  # pylint: disable=import-outside-toplevel
-
-        if not isinstance(cache, diskcache.Cache):
-            raise ValueError("First argument must be a diskcache.Cache object")
-        super().__init__(cache_by)
-
-        # Handle process class import
-        if platform.system() == "Windows":
-            try:
-                from multiprocess import (  # pylint: disable=import-outside-toplevel
-                    Process,
-                )
-            except ImportError:
-                raise ImportError(
-                    """\
-    When running on Windows, the long_callback decorator requires the
-    multiprocess package which can be install using pip...
+        try:
+            import diskcache  # pylint: disable=import-outside-toplevel
+            from multiprocess import (  # pylint: disable=import-outside-toplevel
+                Process,
+            )
+        except ImportError:
+            raise ImportError(
+                """\
+DiskcacheLongCallbackManager requires the multiprocess and diskcache packages which
+can be installed using pip...
 
-        $ pip install multiprocess
+    $ pip install multiprocess diskcache
 
-    or conda.
+or conda.
 
-        $ conda install -c conda-forge multiprocess\n"""
-                )
-        else:
-            from multiprocessing import (  # pylint: disable=import-outside-toplevel
-                Process,
+    $ conda install -c conda-forge multiprocess diskcache\n"""
             )
 
+        if not isinstance(cache, diskcache.Cache):
+            raise ValueError("First argument must be a diskcache.Cache object")
+        super().__init__(cache_by)
+
         self.Process = Process
         self.cache = cache
         self.callback_futures = dict()
diff --git a/requires-testing.txt b/requires-testing.txt
index c0380dcfdf..c3ce4fbece 100644
--- a/requires-testing.txt
+++ b/requires-testing.txt
@@ -8,4 +8,5 @@ cryptography<3.4;python_version<"3.7"
 requests[security]>=2.21.0
 beautifulsoup4>=4.8.2
 waitress>=1.4.4
-diskcache=>5.2.1
+diskcache>=5.2.1
+multiprocess>=0.70.12

From 813fc0084e7b8628b204edfd45b7cc57e4164d9c Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 12 Aug 2021 05:59:17 -0400
Subject: [PATCH 05/32] long_callback docstring

---
 dash/dash.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/dash/dash.py b/dash/dash.py
index 2aade2176b..028ce374df 100644
--- a/dash/dash.py
+++ b/dash/dash.py
@@ -1130,6 +1130,61 @@ def add_context(*args, **kwargs):
         return wrap_func
 
     def long_callback(self, callback_manager, *_args, **_kwargs):
+        """
+        Normally used as a decorator, `@app.long_callback` is an alternative to
+        `@app.callback` designed for callbacks that take a long time to run,
+        without locking up the Dash app or timing out.
+
+        `@long_callback` is designed to support multiple callback managers.
+        Two long callback managers are currently implemented:
+
+            - A diskcache manager (`DiskcacheLongCallbackManager`) that runs callback
+              logic in a separate process and stores the results to disk using the
+              diskcache library. This is the easiest backend to use for local
+              development.
+            - A Celery manager (`CeleryLongCallbackManager`) that runs callback logic
+              in a celery worker and returns results to the Dash app through a Celery
+              broker like RabbitMQ or Redis.
+
+        The first argument to `@long_callback` should be a callback manager instance.
+
+        :param callback_manager:
+            A long callback manager instance. Currently one of
+            `DiskcacheLongCallbackManager` or `CeleryLongCallbackManager`
+
+        The following arguments may include any valid arguments to `@app.callback`.
+        In addition, `@app.long_callback` supports the following optional
+        keyword arguments:
+
+        :Keyword Arguments:
+            :param running:
+                A list of 3-element tuples. The first element of each tuple should be
+                an `Output` dependency object referencing a property of a component in
+                the app layout. The second element is the value that the property
+                should be set to while the callback is running, and the third element
+                is the value the property should be set to when the callback completes.
+            :param cancel:
+                A list of `Input` dependency objects that reference a property of a
+                component in the app's layout.  When the value of this property changes
+                while a callback is running, the callback is canceled.
+                Note that the value of the property is not significant, any change in
+                value will result in the cancellation of the running job (if any).
+            :param progress:
+                An `Output` dependency grouping that references properties of
+                components in the app's layout. When provided, the decorated function
+                will be called with an extra argument as the first argument to the
+                function.  This argument, is a function handle that the decorated
+                function should call in order to provide updates to the app on its
+                current progress. This function accepts a single argument, which
+                correspond to the grouping of properties specified in the provided
+                `Output` dependency grouping
+            :param progress_default:
+                A grouping of values that should be assigned to the components
+                specified by the `progress` argument when the callback is not in
+                progress. If `progress_default` is not provided, all the dependency
+                properties specified in `progress` will be set to `None` when the
+                callback is not running.
+        """
         from . import callback_context  # pylint: disable=import-outside-toplevel
         import dash_core_components as dcc  # pylint: disable=import-outside-toplevel
 

From 4ddecf2d52de48d4f55a944313b982291ff12e4c Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 12 Aug 2021 07:11:20 -0400
Subject: [PATCH 06/32] Fix import

---
 dash/long_callback/managers/celery_manager.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/dash/long_callback/managers/celery_manager.py b/dash/long_callback/managers/celery_manager.py
index e1d2aeea34..fcb5353e74 100644
--- a/dash/long_callback/managers/celery_manager.py
+++ b/dash/long_callback/managers/celery_manager.py
@@ -1,6 +1,7 @@
 import json
-from dash_labs.plugins.long_callback.managers import BaseLongCallbackManager
+
 from _plotly_utils.utils import PlotlyJSONEncoder
+from dash.long_callback.managers import BaseLongCallbackManager
 
 
 class CeleryLongCallbackManager(BaseLongCallbackManager):

From 1ac543ec3f8eb00011eb30d5083b8c5f225b338b Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 12 Aug 2021 07:22:43 -0400
Subject: [PATCH 07/32] flakes

---
 dash/dash.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/dash/dash.py b/dash/dash.py
index 028ce374df..158ac5655b 100644
--- a/dash/dash.py
+++ b/dash/dash.py
@@ -1185,7 +1185,9 @@ def long_callback(self, callback_manager, *_args, **_kwargs):
                 properties specified in `progress` will be set to `None` when the
                 callback is not running.
         """
-        from . import callback_context  # pylint: disable=import-outside-toplevel
+        from dash._callback_context import (  # pylint: disable=import-outside-toplevel
+            callback_context,
+        )
         import dash_core_components as dcc  # pylint: disable=import-outside-toplevel
 
         # Extract special long_callback kwargs
@@ -1236,10 +1238,8 @@ def callback(_triggers, user_store_data, user_callback_args):
                 current_key = user_store_data.get("current_key", None)
 
                 should_cancel = pending_key == current_key or any(
-                    [
-                        trigger["prop_id"] in cancel_prop_ids
-                        for trigger in callback_context.triggered
-                    ]
+                    trigger["prop_id"] in cancel_prop_ids
+                    for trigger in callback_context.triggered
                 )
 
                 # Compute grouping of values to set the progress component's to

From aa676bd2c4a70797ede71acc62c1a752570180e9 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 12 Aug 2021 07:31:14 -0400
Subject: [PATCH 08/32] pylint

---
 dash/long_callback/managers/celery_manager.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dash/long_callback/managers/celery_manager.py b/dash/long_callback/managers/celery_manager.py
index fcb5353e74..5c51249cde 100644
--- a/dash/long_callback/managers/celery_manager.py
+++ b/dash/long_callback/managers/celery_manager.py
@@ -36,8 +36,8 @@ def get_future(self, key, default=None):
     def make_background_fn(self, fn, progress=False):
         return make_celery_fn(fn, self.celery_app, progress)
 
-    def call_and_register_background_fn(self, key, background_fn, *args, **kwargs):
-        future = background_fn.delay(*args, **kwargs)
+    def call_and_register_background_fn(self, key, background_fn, args):
+        future = background_fn.delay(args)
         self.callback_futures[key] = future
 
     def get_progress(self, key):
@@ -53,8 +53,8 @@ def result_ready(self, key):
         future = self.get_future(key)
         if future:
             return future.ready()
-        else:
-            return False
+
+        return False
 
     def get_result(self, key):
         future = self.callback_futures.get(key, None)

From 5ff33cfd2dfd878e6fcdbf1142b041c25331144a Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 12 Aug 2021 09:23:17 -0400
Subject: [PATCH 09/32] Python 3.6 compat

---
 dash/long_callback/managers/diskcache_manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dash/long_callback/managers/diskcache_manager.py b/dash/long_callback/managers/diskcache_manager.py
index 353ee51159..df3e52dbfa 100644
--- a/dash/long_callback/managers/diskcache_manager.py
+++ b/dash/long_callback/managers/diskcache_manager.py
@@ -34,7 +34,7 @@ def delete_future(self, key):
         if key in self.callback_futures:
             future = self.callback_futures.pop(key, None)
             if future:
-                future.kill()
+                future.terminate()
                 future.join()
                 return True
         return False

From 1da38bea860a7208caadad6ee0770d6c4e29fb5b Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 14 Aug 2021 11:21:31 -0400
Subject: [PATCH 10/32] Refactor long calblack mangaers and tests Test with
 celery long callback manager as well as diskcache

---
 dash/dash.py                                  |  34 +-
 dash/long_callback/managers/__init__.py       |  19 +-
 dash/long_callback/managers/celery_manager.py | 135 ++++----
 .../managers/diskcache_manager.py             | 119 ++++---
 .../callbacks/test_long_callback.py           | 311 ------------------
 tests/integration/long_callback/app1.py       |  33 ++
 tests/integration/long_callback/app2.py       |  34 ++
 tests/integration/long_callback/app3.py       |  38 +++
 tests/integration/long_callback/app4.py       |  42 +++
 tests/integration/long_callback/app5.py       |  52 +++
 .../long_callback/test_long_callback.py       | 245 ++++++++++++++
 tests/integration/long_callback/utils.py      |  30 ++
 12 files changed, 647 insertions(+), 445 deletions(-)
 delete mode 100644 tests/integration/callbacks/test_long_callback.py
 create mode 100644 tests/integration/long_callback/app1.py
 create mode 100644 tests/integration/long_callback/app2.py
 create mode 100644 tests/integration/long_callback/app3.py
 create mode 100644 tests/integration/long_callback/app4.py
 create mode 100644 tests/integration/long_callback/app5.py
 create mode 100644 tests/integration/long_callback/test_long_callback.py
 create mode 100644 tests/integration/long_callback/utils.py

diff --git a/dash/dash.py b/dash/dash.py
index 158ac5655b..4f667bfe2f 100644
--- a/dash/dash.py
+++ b/dash/dash.py
@@ -1228,14 +1228,13 @@ def long_callback(self, callback_manager, *_args, **_kwargs):
         )
 
         def wrapper(fn):
-            background_fn = callback_manager.make_background_fn(
-                fn, progress=bool(progress)
-            )
+            background_fn = callback_manager.make_job_fn(fn, progress=bool(progress))
 
             def callback(_triggers, user_store_data, user_callback_args):
                 # Build result cache key from inputs
                 pending_key = callback_manager.build_cache_key(fn, user_callback_args)
                 current_key = user_store_data.get("current_key", None)
+                pending_job = user_store_data.get("pending_job", None)
 
                 should_cancel = pending_key == current_key or any(
                     trigger["prop_id"] in cancel_prop_ids
@@ -1254,9 +1253,10 @@ def callback(_triggers, user_store_data, user_callback_args):
                 if should_cancel:
                     user_store_data["current_key"] = None
                     user_store_data["pending_key"] = None
+                    user_store_data["pending_job"] = None
+
+                    callback_manager.terminate_job(pending_job)
 
-                    if pending_key and callback_manager.has_future(pending_key):
-                        callback_manager.delete_future(pending_key)
                     return dict(
                         user_callback_output=map_grouping(lambda x: no_update, output),
                         interval_disabled=True,
@@ -1265,10 +1265,14 @@ def callback(_triggers, user_store_data, user_callback_args):
                         user_store_data=user_store_data,
                     )
 
-                progress_value = callback_manager.get_progress(pending_key)
+                # Look up progress value if a job is in progress
+                if pending_job:
+                    progress_value = callback_manager.get_progress(pending_key)
+                else:
+                    progress_value = None
 
                 if callback_manager.result_ready(pending_key):
-                    result = callback_manager.get_result(pending_key)
+                    result = callback_manager.get_result(pending_key, pending_job)
                     # Set current key (hash of data stored in client)
                     # to pending key (hash of data requested by client)
                     user_store_data["current_key"] = pending_key
@@ -1288,10 +1292,20 @@ def callback(_triggers, user_store_data, user_callback_args):
                         user_store_data=user_store_data,
                     )
                 else:
+                    # Check if there is a running calculation that can now
+                    # be canceled
+                    old_pending_key = user_store_data.get("pending_key", None)
+                    if (
+                        old_pending_key
+                        and old_pending_key != pending_key
+                        and callback_manager.job_running(pending_job)
+                    ):
+                        callback_manager.terminate_job(pending_job)
+
                     user_store_data["pending_key"] = pending_key
-                    callback_manager.terminate_unhealthy_future(pending_key)
-                    if not callback_manager.has_future(pending_key):
-                        callback_manager.call_and_register_background_fn(
+                    callback_manager.terminate_unhealthy_job(pending_job)
+                    if not callback_manager.job_running(pending_job):
+                        user_store_data["pending_job"] = callback_manager.call_job_fn(
                             pending_key, background_fn, user_callback_args
                         )
 
diff --git a/dash/long_callback/managers/__init__.py b/dash/long_callback/managers/__init__.py
index 62d20c27ad..14826f39f4 100644
--- a/dash/long_callback/managers/__init__.py
+++ b/dash/long_callback/managers/__init__.py
@@ -10,22 +10,19 @@ def __init__(self, cache_by):
 
         self.cache_by = cache_by
 
-    def delete_future(self, key):
+    def terminate_job(self, job):
         raise NotImplementedError
 
-    def terminate_unhealthy_future(self, key):
+    def terminate_unhealthy_job(self, job):
         raise NotImplementedError
 
-    def has_future(self, key):
+    def job_running(self, job):
         raise NotImplementedError
 
-    def get_future(self, key, default=None):
+    def make_job_fn(self, fn, progress):
         raise NotImplementedError
 
-    def make_background_fn(self, fn, progress):
-        raise NotImplementedError
-
-    def call_and_register_background_fn(self, key, background_fn, args):
+    def call_job_fn(self, key, job_fn, args):
         raise NotImplementedError
 
     def get_progress(self, key):
@@ -34,7 +31,7 @@ def get_progress(self, key):
     def result_ready(self, key):
         raise NotImplementedError
 
-    def get_result(self, key):
+    def get_result(self, key, job):
         raise NotImplementedError
 
     def build_cache_key(self, fn, args):
@@ -48,3 +45,7 @@ def build_cache_key(self, fn, args):
                 hash_dict[f"cache_key_{i}"] = cache_item()
 
         return hashlib.sha1(str(hash_dict).encode("utf-8")).hexdigest()
+
+    @staticmethod
+    def _make_progress_key(key):
+        return key + "-progress"
diff --git a/dash/long_callback/managers/celery_manager.py b/dash/long_callback/managers/celery_manager.py
index 5c51249cde..0907095fba 100644
--- a/dash/long_callback/managers/celery_manager.py
+++ b/dash/long_callback/managers/celery_manager.py
@@ -5,89 +5,102 @@
 
 
 class CeleryLongCallbackManager(BaseLongCallbackManager):
-    def __init__(self, celery_app, cache_by=None):
+    def __init__(self, celery_app, cache_by=None, expire=None):
+        import celery
+
+        if not isinstance(celery_app, celery.Celery):
+            raise ValueError("First argument must be a celery.Celery object")
+
         super().__init__(cache_by)
-        self.celery_app = celery_app
-        self.callback_futures = dict()
+        self.handle = celery_app
+        self.expire = expire
 
-    def init(self, app):
-        pass
+    def terminate_job(self, job):
+        if job is None:
+            return
 
-    def delete_future(self, key):
-        if key in self.callback_futures:
-            future = self.callback_futures.pop(key)
-            self.celery_app.control.terminate(future.task_id)
-            return True
-        return False
+        self.handle.control.terminate(job)
 
-    def terminate_unhealthy_future(self, key):
-        if key in self.callback_futures:
-            future = self.callback_futures[key]
-            if future.status != "PENDING":
-                return self.delete_future(key)
+    def terminate_unhealthy_job(self, job):
+        task = self.get_task(job)
+        if task and task.status in ("FAILURE", "REVOKED"):
+            return self.terminate_job(job)
         return False
 
-    def has_future(self, key):
-        return key in self.callback_futures
-
-    def get_future(self, key, default=None):
-        return self.callback_futures.get(key, default)
+    def job_running(self, job):
+        future = self.get_task(job)
+        return future and future.status in (
+            "PENDING",
+            "RECEIVED",
+            "STARTED",
+            "RETRY",
+            "PROGRESS",
+        )
+
+    def make_job_fn(self, fn, progress=False):
+        return _make_job_fn(fn, self.handle, progress)
+
+    def get_task(self, job):
+        if job:
+            return self.handle.AsyncResult(job)
+        else:
+            return None
 
-    def make_background_fn(self, fn, progress=False):
-        return make_celery_fn(fn, self.celery_app, progress)
+    def clear_cache_entry(self, key):
+        self.handle.backend.delete(key)
 
-    def call_and_register_background_fn(self, key, background_fn, args):
-        future = background_fn.delay(args)
-        self.callback_futures[key] = future
+    def call_job_fn(self, key, job_fn, args):
+        task = job_fn.delay(key, self._make_progress_key(key), args)
+        return task.task_id
 
     def get_progress(self, key):
-        future = self.get_future(key)
-        if future is not None:
-            progress_info = future.info if future.state == "PROGRESS" else None
-            if progress_info is not None:
-                return json.loads(progress_info["progress_value"])
-
-        return None
+        progress_key = self._make_progress_key(key)
+        progress_data = self.handle.backend.get(progress_key)
+        if progress_data:
+            return json.loads(progress_data)
+        else:
+            return None
 
     def result_ready(self, key):
-        future = self.get_future(key)
-        if future:
-            return future.ready()
+        return self.handle.backend.get(key) is not None
 
-        return False
+    def get_result(self, key, job):
+        # Get result value
+        result = self.handle.backend.get(key)
+        if result is None:
+            return None
+
+        result = json.loads(result)
 
-    def get_result(self, key):
-        future = self.callback_futures.get(key, None)
-        if future:
-            result = future.get(timeout=1)
-            # Clear result if not caching
-            if self.cache_by is None:
-                self.delete_future(key)
-            return result
+        # Clear result if not caching
+        if self.cache_by is None:
+            self.clear_cache_entry(key)
         else:
-            return None
+            if self.expire:
+                # Set/update expiration time
+                self.handle.backend.expire(key, self.expire)
+        self.clear_cache_entry(self._make_progress_key(key))
+
+        self.terminate_job(job)
+        return result
+
 
+def _make_job_fn(fn, celery_app, progress):
+    cache = celery_app.backend
 
-def make_celery_fn(user_fn, celery_app, progress):
-    @celery_app.task(bind=True)
-    def _celery_fn(self, user_callback_args):
+    @celery_app.task
+    def job_fn(result_key, progress_key, user_callback_args):
         def _set_progress(progress_value):
-            # JSON serialize with PlotlyEncoder
-            self.update_state(
-                state="PROGRESS",
-                meta={
-                    "progress_value": json.dumps(progress_value, cls=PlotlyJSONEncoder)
-                },
-            )
+            cache.set(progress_key, json.dumps(progress_value, cls=PlotlyJSONEncoder))
 
         maybe_progress = [_set_progress] if progress else []
         if isinstance(user_callback_args, dict):
-            user_callback_output = user_fn(*maybe_progress, **user_callback_args)
+            user_callback_output = fn(*maybe_progress, **user_callback_args)
         elif isinstance(user_callback_args, list):
-            user_callback_output = user_fn(*maybe_progress, *user_callback_args)
+            user_callback_output = fn(*maybe_progress, *user_callback_args)
         else:
-            user_callback_output = user_fn(*maybe_progress, user_callback_args)
+            user_callback_output = fn(*maybe_progress, user_callback_args)
 
-        return user_callback_output
+        cache.set(result_key, json.dumps(user_callback_output, cls=PlotlyJSONEncoder))
 
-    return _celery_fn
+    return job_fn
diff --git a/dash/long_callback/managers/diskcache_manager.py b/dash/long_callback/managers/diskcache_manager.py
index df3e52dbfa..791731bc04 100644
--- a/dash/long_callback/managers/diskcache_manager.py
+++ b/dash/long_callback/managers/diskcache_manager.py
@@ -1,99 +1,110 @@
 from . import BaseLongCallbackManager
 
+_pending_value = "__$pending__"
+
 
 class DiskcacheLongCallbackManager(BaseLongCallbackManager):
     def __init__(self, cache, cache_by=None, expire=None):
         try:
             import diskcache  # pylint: disable=import-outside-toplevel
-            from multiprocess import (  # pylint: disable=import-outside-toplevel
-                Process,
-            )
+            import psutil  # noqa: F401,E402 pylint: disable=import-outside-toplevel,unused-variable
+            import multiprocess  # noqa: F401,E402 pylint: disable=import-outside-toplevel,unused-variable
         except ImportError:
             raise ImportError(
                 """\
-DiskcacheLongCallbackManager requires the multiprocess and diskcache packages which
-can be installed using pip...
+DiskcacheLongCallbackManager requires the multiprocess, diskcache, and psutil packages
+which can be installed using pip...
 
     $ pip install multiprocess diskcache
 
 or conda.
 
-    $ conda install -c conda-forge multiprocess diskcache\n"""
+    $ conda install -c conda-forge multiprocess diskcache psutil\n"""
             )
 
         if not isinstance(cache, diskcache.Cache):
             raise ValueError("First argument must be a diskcache.Cache object")
         super().__init__(cache_by)
-
-        self.Process = Process
-        self.cache = cache
-        self.callback_futures = dict()
+        self.handle = cache
         self.expire = expire
 
-    def delete_future(self, key):
-        if key in self.callback_futures:
-            future = self.callback_futures.pop(key, None)
-            if future:
-                future.terminate()
-                future.join()
-                return True
-        return False
+    def terminate_job(self, job):
+        import psutil  # pylint: disable=import-outside-toplevel
 
-    def clear_cache_entry(self, key):
-        self.cache.delete(key)
+        if job is None:
+            return
+
+        # Use diskcache transaction so multiple process don't try to kill the
+        # process at the same time
+        with self.handle.transact():
+            if psutil.pid_exists(job):
+                process = psutil.Process(job)
+                for proc in process.children(recursive=True):
+                    proc.kill()
+                process.kill()
+                process.wait(1)
+
+    def terminate_unhealthy_job(self, job):
+        import psutil  # pylint: disable=import-outside-toplevel
+
+        if job and psutil.pid_exists(job):
+            if not self.job_running(job):
+                self.terminate_job(job)
+                return True
 
-    def terminate_unhealthy_future(self, key):
         return False
 
-    def has_future(self, key):
-        return self.callback_futures.get(key, None) is not None
+    def job_running(self, job):
+        import psutil  # pylint: disable=import-outside-toplevel
 
-    def get_future(self, key, default=None):
-        return self.callback_futures.get(key, default)
+        if job and psutil.pid_exists(job):
+            proc = psutil.Process(job)
+            return proc.status() != psutil.STATUS_ZOMBIE
+        return False
 
-    def make_background_fn(self, fn, progress=False):
-        return make_update_cache(fn, self.cache, progress, self.expire)
+    def make_job_fn(self, fn, progress=False):
+        return _make_job_fn(fn, self.handle, progress)
 
-    @staticmethod
-    def _make_progress_key(key):
-        return key + "-progress"
+    def clear_cache_entry(self, key):
+        self.handle.delete(key)
 
-    def call_and_register_background_fn(self, key, background_fn, args):
-        self.delete_future(key)
-        future = self.Process(
-            target=background_fn, args=(key, self._make_progress_key(key), args)
+    def call_job_fn(self, key, job_fn, args):
+        from multiprocess import (  # pylint: disable=import-outside-toplevel,no-name-in-module
+            Process,
         )
-        future.start()
-        self.callback_futures[key] = future
+
+        proc = Process(target=job_fn, args=(key, self._make_progress_key(key), args))
+        proc.start()
+        return proc.pid
 
     def get_progress(self, key):
-        future = self.get_future(key)
-        if future is not None:
-            progress_key = self._make_progress_key(key)
-            return self.cache.get(progress_key)
-        return None
+        progress_key = self._make_progress_key(key)
+        return self.handle.get(progress_key)
 
     def result_ready(self, key):
-        return self.cache.get(key) not in (None, "__undefined__")
+        return self.handle.get(key) is not None
 
-    def get_result(self, key):
+    def get_result(self, key, job):
         # Get result value
-        result = self.cache.get(key)
-        if result == "__undefined__":
-            result = None
+        result = self.handle.get(key)
+        if result is None:
+            return None
 
         # Clear result if not caching
-        if self.cache_by is None and result is not None:
+        if self.cache_by is None:
             self.clear_cache_entry(key)
+        else:
+            if self.expire:
+                self.handle.touch(key, expire=self.expire)
+
+        self.clear_cache_entry(self._make_progress_key(key))
 
-        # Always delete_future (even if we didn't clear cache) so that we can
-        # handle the case where cache entry is cleared externally.
-        self.delete_future(key)
+        self.terminate_job(job)
         return result
 
 
-def make_update_cache(fn, cache, progress, expire):
-    def _callback(result_key, progress_key, user_callback_args):
+def _make_job_fn(fn, cache, progress):
+    def job_fn(result_key, progress_key, user_callback_args):
         def _set_progress(progress_value):
             cache.set(progress_key, progress_value)
 
@@ -104,6 +115,6 @@ def _set_progress(progress_value):
             user_callback_output = fn(*maybe_progress, *user_callback_args)
         else:
             user_callback_output = fn(*maybe_progress, user_callback_args)
-        cache.set(result_key, user_callback_output, expire=expire)
+        cache.set(result_key, user_callback_output)
 
-    return _callback
+    return job_fn
diff --git a/tests/integration/callbacks/test_long_callback.py b/tests/integration/callbacks/test_long_callback.py
deleted file mode 100644
index 762c8a7fbb..0000000000
--- a/tests/integration/callbacks/test_long_callback.py
+++ /dev/null
@@ -1,311 +0,0 @@
-from multiprocessing import Lock, Value
-import time
-
-import dash_core_components as dcc
-import dash_html_components as html
-import dash
-from dash.dependencies import Input, Output, State
-
-
-def test_lcb001_fast_input(dash_duo, diskcache_manager):
-    """
-    Make sure that we settle to the correct final value when handling rapid inputs
-    """
-    lock = Lock()
-
-    app = dash.Dash(__name__)
-    app.layout = html.Div(
-        [
-            dcc.Input(id="input", value="initial value"),
-            html.Div(html.Div([1.5, None, "string", html.Div(id="output-1")])),
-        ]
-    )
-
-    @app.long_callback(
-        diskcache_manager,
-        Output("output-1", "children"),
-        [Input("input", "value")],
-        interval=500,
-    )
-    def update_output(value):
-        time.sleep(0.1)
-        return value
-
-    dash_duo.start_server(app)
-    dash_duo.wait_for_text_to_equal("#output-1", "initial value", 2)
-    input_ = dash_duo.find_element("#input")
-    dash_duo.clear_input(input_)
-
-    for key in "hello world":
-        with lock:
-            input_.send_keys(key)
-
-    dash_duo.wait_for_text_to_equal("#output-1", "hello world", 4)
-    assert not dash_duo.redux_state_is_loading
-    assert dash_duo.get_logs() == []
-
-
-def test_lcb002_long_callback_running(dash_duo, diskcache_manager):
-    app = dash.Dash(__name__)
-    app.layout = html.Div(
-        [
-            html.Button(id="button-1", children="Click Here", n_clicks=0),
-            html.Div(id="status", children="Finished"),
-            html.Div(id="result", children="Not clicked"),
-        ]
-    )
-
-    @app.long_callback(
-        diskcache_manager,
-        Output("result", "children"),
-        [Input("button-1", "n_clicks")],
-        running=[(Output("status", "children"), "Running", "Finished")],
-        interval=500,
-    )
-    def update_output(n_clicks):
-        time.sleep(2)
-        return f"Clicked {n_clicks} time(s)"
-
-    dash_duo.start_server(app)
-    dash_duo.wait_for_text_to_equal("#result", "Clicked 0 time(s)", 4)
-    dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
-
-    # Click button and check that status has changed to "Running"
-    dash_duo.find_element("#button-1").click()
-    dash_duo.wait_for_text_to_equal("#status", "Running", 2)
-
-    # Wait for calculation to finish, then check that status is "Finished"
-    dash_duo.wait_for_text_to_equal("#result", "Clicked 1 time(s)", 4)
-    dash_duo.wait_for_text_to_equal("#status", "Finished", 2)
-
-    # Click button twice and check that status has changed to "Running"
-    dash_duo.find_element("#button-1").click()
-    dash_duo.find_element("#button-1").click()
-    dash_duo.wait_for_text_to_equal("#status", "Running", 2)
-
-    # Wait for calculation to finish, then check that status is "Finished"
-    dash_duo.wait_for_text_to_equal("#result", "Clicked 3 time(s)", 4)
-    dash_duo.wait_for_text_to_equal("#status", "Finished", 2)
-
-    assert not dash_duo.redux_state_is_loading
-    assert dash_duo.get_logs() == []
-
-
-def test_lcb003_long_callback_running_cancel(dash_duo, diskcache_manager):
-    lock = Lock()
-    app = dash.Dash(__name__)
-    app.layout = html.Div(
-        [
-            dcc.Input(id="input", value="initial value"),
-            html.Button(id="run-button", children="Run"),
-            html.Button(id="cancel-button", children="Cancel"),
-            html.Div(id="status", children="Finished"),
-            html.Div(id="result", children="No results"),
-        ]
-    )
-
-    @app.long_callback(
-        diskcache_manager,
-        Output("result", "children"),
-        [Input("run-button", "n_clicks"), State("input", "value")],
-        running=[(Output("status", "children"), "Running", "Finished")],
-        cancel=[Input("cancel-button", "n_clicks")],
-        interval=500,
-    )
-    def update_output(n_clicks, value):
-        time.sleep(2)
-        return f"Processed '{value}'"
-
-    dash_duo.start_server(app)
-    dash_duo.wait_for_text_to_equal("#result", "Processed 'initial value'", 4)
-    dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
-
-    # Update input text box
-    input_ = dash_duo.find_element("#input")
-    dash_duo.clear_input(input_)
-
-    for key in "hello world":
-        with lock:
-            input_.send_keys(key)
-
-    # Click run button and check that status has changed to "Running"
-    dash_duo.find_element("#run-button").click()
-    dash_duo.wait_for_text_to_equal("#status", "Running", 2)
-
-    # Then click Cancel button and make sure that the status changes to finish
-    # without update result
-    dash_duo.find_element("#cancel-button").click()
-    dash_duo.wait_for_text_to_equal("#result", "Processed 'initial value'", 4)
-    dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
-
-    # Click run button again, and let it finish
-    dash_duo.find_element("#run-button").click()
-    dash_duo.wait_for_text_to_equal("#status", "Running", 2)
-    dash_duo.wait_for_text_to_equal("#result", "Processed 'hello world'", 4)
-    dash_duo.wait_for_text_to_equal("#status", "Finished", 2)
-
-    assert not dash_duo.redux_state_is_loading
-    assert dash_duo.get_logs() == []
-
-
-def test_lcb004_long_callback_progress(dash_duo, diskcache_manager):
-    app = dash.Dash(__name__)
-    app.layout = html.Div(
-        [
-            dcc.Input(id="input", value="hello, world"),
-            html.Button(id="run-button", children="Run"),
-            html.Button(id="cancel-button", children="Cancel"),
-            html.Div(id="status", children="Finished"),
-            html.Div(id="result", children="No results"),
-        ]
-    )
-
-    @app.long_callback(
-        diskcache_manager,
-        Output("result", "children"),
-        [Input("run-button", "n_clicks"), State("input", "value")],
-        progress=Output("status", "children"),
-        progress_default="Finished",
-        cancel=[Input("cancel-button", "n_clicks")],
-        interval=500,
-        prevent_initial_callback=True,
-    )
-    def update_output(set_progress, n_clicks, value):
-        for i in range(4):
-            set_progress(f"Progress {i}/4")
-            time.sleep(1)
-        return f"Processed '{value}'"
-
-    dash_duo.start_server(app)
-    dash_duo.wait_for_text_to_equal("#result", "No results", 2)
-    dash_duo.wait_for_text_to_equal("#status", "Finished", 2)
-
-    # Click run button and check that status eventually cycles to 2/4
-    dash_duo.find_element("#run-button").click()
-    dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 4)
-
-    # Then click Cancel button and make sure that the status changes to finish
-    # without updating result
-    dash_duo.find_element("#cancel-button").click()
-    dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
-    dash_duo.wait_for_text_to_equal("#result", "No results", 4)
-
-    # Click run button and allow callback to finish
-    dash_duo.find_element("#run-button").click()
-    dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 4)
-    dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
-    dash_duo.wait_for_text_to_equal("#result", "Processed 'hello, world'", 2)
-
-    # Click run button again with same input.
-    # without caching, this should rerun callback and display progress
-    dash_duo.find_element("#run-button").click()
-    dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 4)
-    dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
-    dash_duo.wait_for_text_to_equal("#result", "Processed 'hello, world'", 2)
-
-    assert not dash_duo.redux_state_is_loading
-    assert dash_duo.get_logs() == []
-
-
-def test_lcb005_long_callback_caching(dash_duo, diskcache_manager):
-    lock = Lock()
-
-    # Control return value of cache_by function using multiprocessing value
-    cache_key = Value("i", 0)
-
-    def cache_fn():
-        return cache_key.value
-
-    diskcache_manager.cache_by = [cache_fn]
-
-    app = dash.Dash(__name__)
-    app.layout = html.Div(
-        [
-            dcc.Input(id="input", value="AAA"),
-            html.Button(id="run-button", children="Run", n_clicks=0),
-            html.Div(id="status", children="Finished"),
-            html.Div(id="result", children="No results"),
-        ]
-    )
-
-    @app.long_callback(
-        diskcache_manager,
-        [Output("result", "children"), Output("run-button", "n_clicks")],
-        [Input("run-button", "n_clicks"), State("input", "value")],
-        progress=Output("status", "children"),
-        progress_default="Finished",
-        interval=500,
-        prevent_initial_callback=True,
-    )
-    def update_output(set_progress, _n_clicks, value):
-        for i in range(4):
-            set_progress(f"Progress {i}/4")
-            time.sleep(2)
-        return f"Result for '{value}'", 0
-
-    dash_duo.start_server(app)
-    dash_duo.wait_for_text_to_equal("#result", "No results", 2)
-    dash_duo.wait_for_text_to_equal("#status", "Finished", 2)
-
-    # Click run button and check that status eventually cycles to 2/4
-    dash_duo.find_element("#run-button").click()
-    dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 6)
-    dash_duo.wait_for_text_to_equal("#status", "Finished", 6)
-    dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 1)
-
-    # Update input text box to BBB
-    input_ = dash_duo.find_element("#input")
-    dash_duo.clear_input(input_)
-    for key in "BBB":
-        with lock:
-            input_.send_keys(key)
-
-    # Click run button and check that status eventually cycles to 2/4
-    dash_duo.find_element("#run-button").click()
-    dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 6)
-    dash_duo.wait_for_text_to_equal("#status", "Finished", 6)
-    dash_duo.wait_for_text_to_equal("#result", "Result for 'BBB'", 1)
-
-    # Update input text box back to AAA
-    input_ = dash_duo.find_element("#input")
-    dash_duo.clear_input(input_)
-    for key in "AAA":
-        with lock:
-            input_.send_keys(key)
-
-    # Click run button and this time the cached result is used,
-    # So we can get the result right away
-    dash_duo.find_element("#run-button").click()
-    dash_duo.wait_for_text_to_equal("#status", "Finished", 1)
-    dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 1)
-
-    # Update input text box back to BBB
-    input_ = dash_duo.find_element("#input")
-    dash_duo.clear_input(input_)
-    for key in "BBB":
-        with lock:
-            input_.send_keys(key)
-
-    # Click run button and this time the cached result is used,
-    # So we can get the result right away
-    dash_duo.find_element("#run-button").click()
-    dash_duo.wait_for_text_to_equal("#status", "Finished", 1)
-    dash_duo.wait_for_text_to_equal("#result", "Result for 'BBB'", 1)
-
-    # Update input text box back to AAA
-    input_ = dash_duo.find_element("#input")
-    dash_duo.clear_input(input_)
-    for key in "AAA":
-        with lock:
-            input_.send_keys(key)
-
-    # Change cache key
-    cache_key.value = 1
-
-    dash_duo.find_element("#run-button").click()
-    dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 6)
-    dash_duo.wait_for_text_to_equal("#status", "Finished", 6)
-    dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 1)
-
-    assert not dash_duo.redux_state_is_loading
-    assert dash_duo.get_logs() == []
diff --git a/tests/integration/long_callback/app1.py b/tests/integration/long_callback/app1.py
new file mode 100644
index 0000000000..3dbb20be64
--- /dev/null
+++ b/tests/integration/long_callback/app1.py
@@ -0,0 +1,33 @@
+import dash
+from dash.dependencies import Input, Output
+import dash_html_components as html
+import dash_core_components as dcc
+import time
+
+from utils import get_long_callback_manager
+
+long_callback_manager = get_long_callback_manager()
+handle = long_callback_manager.handle
+
+app = dash.Dash(__name__)
+app.layout = html.Div(
+    [
+        dcc.Input(id="input", value="initial value"),
+        html.Div(html.Div([1.5, None, "string", html.Div(id="output-1")])),
+    ]
+)
+
+
+@app.long_callback(
+    long_callback_manager,
+    Output("output-1", "children"),
+    [Input("input", "value")],
+    interval=500,
+)
+def update_output(value):
+    time.sleep(0.1)
+    return value
+
+
+if __name__ == "__main__":
+    app.run_server(debug=True)
diff --git a/tests/integration/long_callback/app2.py b/tests/integration/long_callback/app2.py
new file mode 100644
index 0000000000..e7a7ed6a83
--- /dev/null
+++ b/tests/integration/long_callback/app2.py
@@ -0,0 +1,34 @@
+import dash
+from dash.dependencies import Input, Output
+import dash_html_components as html
+import time
+
+from utils import get_long_callback_manager
+
+long_callback_manager = get_long_callback_manager()
+handle = long_callback_manager.handle
+
+app = dash.Dash(__name__)
+app.layout = html.Div(
+    [
+        html.Button(id="button-1", children="Click Here", n_clicks=0),
+        html.Div(id="status", children="Finished"),
+        html.Div(id="result", children="Not clicked"),
+    ]
+)
+
+
+@app.long_callback(
+    long_callback_manager,
+    Output("result", "children"),
+    [Input("button-1", "n_clicks")],
+    running=[(Output("status", "children"), "Running", "Finished")],
+    interval=500,
+)
+def update_output(n_clicks):
+    time.sleep(2)
+    return f"Clicked {n_clicks} time(s)"
+
+
+if __name__ == "__main__":
+    app.run_server(debug=True)
diff --git a/tests/integration/long_callback/app3.py b/tests/integration/long_callback/app3.py
new file mode 100644
index 0000000000..da022dd7af
--- /dev/null
+++ b/tests/integration/long_callback/app3.py
@@ -0,0 +1,38 @@
+import dash
+from dash.dependencies import Input, State, Output
+import dash_html_components as html
+import dash_core_components as dcc
+import time
+
+from utils import get_long_callback_manager
+
+long_callback_manager = get_long_callback_manager()
+handle = long_callback_manager.handle
+
+app = dash.Dash(__name__)
+app.layout = html.Div(
+    [
+        dcc.Input(id="input", value="initial value"),
+        html.Button(id="run-button", children="Run"),
+        html.Button(id="cancel-button", children="Cancel"),
+        html.Div(id="status", children="Finished"),
+        html.Div(id="result", children="No results"),
+    ]
+)
+
+
+@app.long_callback(
+    long_callback_manager,
+    Output("result", "children"),
+    [Input("run-button", "n_clicks"), State("input", "value")],
+    running=[(Output("status", "children"), "Running", "Finished")],
+    cancel=[Input("cancel-button", "n_clicks")],
+    interval=500,
+)
+def update_output(n_clicks, value):
+    time.sleep(2)
+    return f"Processed '{value}'"
+
+
+if __name__ == "__main__":
+    app.run_server(debug=True)
diff --git a/tests/integration/long_callback/app4.py b/tests/integration/long_callback/app4.py
new file mode 100644
index 0000000000..331cdce04d
--- /dev/null
+++ b/tests/integration/long_callback/app4.py
@@ -0,0 +1,42 @@
+import dash
+from dash.dependencies import Input, State, Output
+import dash_html_components as html
+import dash_core_components as dcc
+import time
+
+from utils import get_long_callback_manager
+
+long_callback_manager = get_long_callback_manager()
+handle = long_callback_manager.handle
+
+
+app = dash.Dash(__name__)
+app.layout = html.Div(
+    [
+        dcc.Input(id="input", value="hello, world"),
+        html.Button(id="run-button", children="Run"),
+        html.Button(id="cancel-button", children="Cancel"),
+        html.Div(id="status", children="Finished"),
+        html.Div(id="result", children="No results"),
+    ]
+)
+
+
+@app.long_callback(
+    long_callback_manager,
+    Output("result", "children"),
+    [Input("run-button", "n_clicks"), State("input", "value")],
+    progress=Output("status", "children"),
+    progress_default="Finished",
+    cancel=[Input("cancel-button", "n_clicks")],
+    interval=500,
+)
+def update_output(set_progress, n_clicks, value):
+    for i in range(4):
+        set_progress(f"Progress {i}/4")
+        time.sleep(1)
+    return f"Processed '{value}'"
+
+
+if __name__ == "__main__":
+    app.run_server(debug=True)
diff --git a/tests/integration/long_callback/app5.py b/tests/integration/long_callback/app5.py
new file mode 100644
index 0000000000..872e51f5ee
--- /dev/null
+++ b/tests/integration/long_callback/app5.py
@@ -0,0 +1,52 @@
+import dash
+from dash.dependencies import Input, State, Output
+import dash_html_components as html
+import dash_core_components as dcc
+import time
+from multiprocessing import Value
+
+from utils import get_long_callback_manager
+
+long_callback_manager = get_long_callback_manager()
+handle = long_callback_manager.handle
+
+
+app = dash.Dash(__name__)
+app._cache_key = Value("i", 0)
+
+
+# Control return value of cache_by function using multiprocessing value
+def cache_fn():
+    return app._cache_key.value
+
+
+long_callback_manager.cache_by = [cache_fn]
+
+
+app.layout = html.Div(
+    [
+        dcc.Input(id="input", value="AAA"),
+        html.Button(id="run-button", children="Run"),
+        html.Div(id="status", children="Finished"),
+        html.Div(id="result", children="No results"),
+    ]
+)
+
+
+@app.long_callback(
+    long_callback_manager,
+    Output("result", "children"),
+    [Input("run-button", "n_clicks"), State("input", "value")],
+    progress=Output("status", "children"),
+    progress_default="Finished",
+    interval=500,
+)
+def update_output(set_progress, _n_clicks, value):
+    for i in range(4):
+        set_progress(f"Progress {i}/4")
+        time.sleep(2)
+    return f"Result for '{value}'"
+
+
+if __name__ == "__main__":
+    app.run_server(debug=True)
diff --git a/tests/integration/long_callback/test_long_callback.py b/tests/integration/long_callback/test_long_callback.py
new file mode 100644
index 0000000000..cb72e33f45
--- /dev/null
+++ b/tests/integration/long_callback/test_long_callback.py
@@ -0,0 +1,245 @@
+from multiprocessing import Lock
+import os
+from contextlib import contextmanager
+import subprocess
+import tempfile
+import pytest
+import shutil
+import time
+
+from dash.testing.application_runners import import_app
+import psutil
+
+
+def kill(proc_pid):
+    process = psutil.Process(proc_pid)
+    for proc in process.children(recursive=True):
+        proc.kill()
+    process.kill()
+
+
+@pytest.fixture(params=["diskcache", "celery"])
+def manager(request):
+    return request.param
+
+
+@contextmanager
+def setup_long_callback_app(manager_name, app_name):
+    if manager_name == "celery":
+        os.environ["LONG_CALLBACK_MANAGER"] = "celery"
+        worker = subprocess.Popen(
+            [
+                "celery",
+                "-A",
+                f"{app_name}:handle",
+                "worker",
+                "--concurrency",
+                "2",
+                "--loglevel=info",
+            ],
+            preexec_fn=os.setpgrp,
+        )
+        try:
+            yield import_app(app_name)
+        finally:
+            # Interval may run one more time after settling on final app state
+            # Sleep for 1 interval of time
+            time.sleep(0.5)
+            os.environ.pop("LONG_CALLBACK_MANAGER")
+            kill(worker.pid)
+
+    elif manager_name == "diskcache":
+        os.environ["LONG_CALLBACK_MANAGER"] = "diskcache"
+        cache_directory = tempfile.mkdtemp(prefix="lc-diskcache-")
+        print(cache_directory)
+        os.environ["DISKCACHE_DIR"] = cache_directory
+        try:
+            yield import_app(app_name)
+        finally:
+            # Interval may run one more time after settling on final app state
+            # Sleep for 1 interval of time
+            time.sleep(0.5)
+            shutil.rmtree(cache_directory, ignore_errors=True)
+            os.environ.pop("LONG_CALLBACK_MANAGER")
+            os.environ.pop("DISKCACHE_DIR")
+
+
+def test_lcb001_fast_input(dash_duo, manager):
+    """
+    Make sure that we settle to the correct final value when handling rapid inputs
+    """
+    lock = Lock()
+    with setup_long_callback_app(manager, "app1") as app:
+        dash_duo.start_server(app)
+        dash_duo.wait_for_text_to_equal("#output-1", "initial value", 15)
+        input_ = dash_duo.find_element("#input")
+        dash_duo.clear_input(input_)
+
+        for key in "hello world":
+            with lock:
+                input_.send_keys(key)
+
+        dash_duo.wait_for_text_to_equal("#output-1", "hello world", 4)
+
+    assert not dash_duo.redux_state_is_loading
+    assert dash_duo.get_logs() == []
+
+
+def test_lcb002_long_callback_running(dash_duo, manager):
+    with setup_long_callback_app(manager, "app2") as app:
+        dash_duo.start_server(app)
+        dash_duo.wait_for_text_to_equal("#result", "Clicked 0 time(s)", 15)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+
+        # Click button and check that status has changed to "Running"
+        dash_duo.find_element("#button-1").click()
+        dash_duo.wait_for_text_to_equal("#status", "Running", 4)
+
+        # Wait for calculation to finish, then check that status is "Finished"
+        dash_duo.wait_for_text_to_equal("#result", "Clicked 1 time(s)", 6)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+
+        # Click button twice and check that status has changed to "Running"
+        dash_duo.find_element("#button-1").click()
+        dash_duo.find_element("#button-1").click()
+        dash_duo.wait_for_text_to_equal("#status", "Running", 4)
+
+        # Wait for calculation to finish, then check that status is "Finished"
+        dash_duo.wait_for_text_to_equal("#result", "Clicked 3 time(s)", 10)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+
+    assert not dash_duo.redux_state_is_loading
+    assert dash_duo.get_logs() == []
+
+
+def test_lcb003_long_callback_running_cancel(dash_duo, manager):
+    lock = Lock()
+
+    with setup_long_callback_app(manager, "app3") as app:
+        dash_duo.start_server(app)
+        dash_duo.wait_for_text_to_equal("#result", "Processed 'initial value'", 15)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 6)
+
+        # Update input text box
+        input_ = dash_duo.find_element("#input")
+        dash_duo.clear_input(input_)
+
+        for key in "hello world":
+            with lock:
+                input_.send_keys(key)
+
+        # Click run button and check that status has changed to "Running"
+        dash_duo.find_element("#run-button").click()
+        dash_duo.wait_for_text_to_equal("#status", "Running", 4)
+
+        # Then click Cancel button and make sure that the status changes to finish
+        # without update result
+        dash_duo.find_element("#cancel-button").click()
+        dash_duo.wait_for_text_to_equal("#result", "Processed 'initial value'", 8)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+
+        # Click run button again, and let it finish
+        dash_duo.find_element("#run-button").click()
+        dash_duo.wait_for_text_to_equal("#status", "Running", 4)
+        dash_duo.wait_for_text_to_equal("#result", "Processed 'hello world'", 4)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+
+    assert not dash_duo.redux_state_is_loading
+    assert dash_duo.get_logs() == []
+
+
+def test_lcb004_long_callback_progress(dash_duo, manager):
+    with setup_long_callback_app(manager, "app4") as app:
+        dash_duo.start_server(app)
+
+        # check that status eventually cycles to 2/4
+        dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 15)
+
+        # Then click Cancel button and make sure that the status changes to finish
+        # without updating result
+        dash_duo.find_element("#cancel-button").click()
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
+        dash_duo.wait_for_text_to_equal("#result", "No results", 8)
+
+        # Click run button and allow callback to finish
+        dash_duo.find_element("#run-button").click()
+        dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 15)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 15)
+        dash_duo.wait_for_text_to_equal("#result", "Processed 'hello, world'", 4)
+
+        # Click run button again with same input.
+        # without caching, this should rerun callback and display progress
+        dash_duo.find_element("#run-button").click()
+        dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 15)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 15)
+        dash_duo.wait_for_text_to_equal("#result", "Processed 'hello, world'", 4)
+
+    assert not dash_duo.redux_state_is_loading
+    assert dash_duo.get_logs() == []
+
+
+def test_lcb005_long_callback_caching(dash_duo, manager):
+    lock = Lock()
+
+    with setup_long_callback_app(manager, "app5") as app:
+        dash_duo.start_server(app)
+        dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 15)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 15)
+        dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 4)
+
+        # Update input text box to BBB
+        input_ = dash_duo.find_element("#input")
+        dash_duo.clear_input(input_)
+        for key in "BBB":
+            with lock:
+                input_.send_keys(key)
+
+        # Click run button and check that status eventually cycles to 2/4
+        dash_duo.find_element("#run-button").click()
+        dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 20)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
+        dash_duo.wait_for_text_to_equal("#result", "Result for 'BBB'", 4)
+
+        # # Update input text box back to AAA
+        # input_ = dash_duo.find_element("#input")
+        # dash_duo.clear_input(input_)
+        # for key in "AAA":
+        #     with lock:
+        #         input_.send_keys(key)
+        #
+        # # Click run button and this time the cached result is used,
+        # # So we can get the result right away
+        # dash_duo.find_element("#run-button").click()
+        # dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+        # dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 4)
+
+        # # Update input text box back to BBB
+        # input_ = dash_duo.find_element("#input")
+        # dash_duo.clear_input(input_)
+        # for key in "BBB":
+        #     with lock:
+        #         input_.send_keys(key)
+        #
+        # # Click run button and this time the cached result is used,
+        # # So we can get the result right away
+        # dash_duo.find_element("#run-button").click()
+        # dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+        # dash_duo.wait_for_text_to_equal("#result", "Result for 'BBB'", 4)
+        #
+        # # Update input text box back to AAA
+        # input_ = dash_duo.find_element("#input")
+        # dash_duo.clear_input(input_)
+        # for key in "AAA":
+        #     with lock:
+        #         input_.send_keys(key)
+        #
+        # # Change cache key
+        # app._cache_key.value = 1
+        #
+        # dash_duo.find_element("#run-button").click()
+        # dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 20)
+        # dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
+        # dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 4)
+        #
+        # assert not dash_duo.redux_state_is_loading
+        # assert dash_duo.get_logs() == []
diff --git a/tests/integration/long_callback/utils.py b/tests/integration/long_callback/utils.py
new file mode 100644
index 0000000000..51e25282bb
--- /dev/null
+++ b/tests/integration/long_callback/utils.py
@@ -0,0 +1,30 @@
+import os
+
+
+def get_long_callback_manager():
+    """
+    Get the long callback mangaer configured by environment variables
+    """
+    if os.environ.get("LONG_CALLBACK_MANAGER", None) == "celery":
+        from dash.long_callback import CeleryLongCallbackManager
+        from celery import Celery
+
+        celery_app = Celery(
+            __name__,
+            broker="redis://localhost:6379/0",
+            backend="redis://localhost:6379/1",
+        )
+        long_callback_manager = CeleryLongCallbackManager(celery_app)
+    elif os.environ.get("LONG_CALLBACK_MANAGER", None) == "diskcache":
+        from dash.long_callback import DiskcacheLongCallbackManager
+        import diskcache
+
+        cache = diskcache.Cache(os.environ.get("DISKCACHE_DIR"))
+        long_callback_manager = DiskcacheLongCallbackManager(cache)
+    else:
+        raise ValueError(
+            "Invalid long callback manager specified as LONG_CALLBACK_MANAGER "
+            "environment variable"
+        )
+
+    return long_callback_manager

From 02b36dfeea166e48e2f66bc18135ca72c2e11912 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 14 Aug 2021 12:19:32 -0400
Subject: [PATCH 11/32] Add cache_args_to_skip option to long_callback

---
 dash/dash.py                                  |  19 +++-
 dash/long_callback/managers/__init__.py       |  14 ++-
 tests/integration/long_callback/app5.py       |   1 +
 .../long_callback/test_long_callback.py       | 103 ++++++++++--------
 tests/integration/long_callback/utils.py      |   4 +-
 5 files changed, 93 insertions(+), 48 deletions(-)

diff --git a/dash/dash.py b/dash/dash.py
index 4f667bfe2f..3c1187b1a1 100644
--- a/dash/dash.py
+++ b/dash/dash.py
@@ -1184,6 +1184,11 @@ def long_callback(self, callback_manager, *_args, **_kwargs):
                 progress. If `progress_default` is not provided, all the dependency
                 properties specified in `progress` will be set to `None` when the
                 callback is not running.
+            :param cache_args_to_ignore:
+                Arguments to ignore when caching is enabled. If callback is configured
+                with keyword arguments (Input/State provided in a dict),
+                this should be a list of argument names as strings. Otherwise,
+                this should be a list of argument indices as integers.
         """
         from dash._callback_context import (  # pylint: disable=import-outside-toplevel
             callback_context,
@@ -1196,6 +1201,7 @@ def long_callback(self, callback_manager, *_args, **_kwargs):
         progress = _kwargs.pop("progress", ())
         progress_default = _kwargs.pop("progress_default", None)
         interval_time = _kwargs.pop("interval", 1000)
+        cache_args_to_ignore = _kwargs.pop("cache_args_to_ignore", [])
 
         # Parse remaining args just like app.callback
         (
@@ -1232,7 +1238,9 @@ def wrapper(fn):
 
             def callback(_triggers, user_store_data, user_callback_args):
                 # Build result cache key from inputs
-                pending_key = callback_manager.build_cache_key(fn, user_callback_args)
+                pending_key = callback_manager.build_cache_key(
+                    fn, user_callback_args, cache_args_to_ignore
+                )
                 current_key = user_store_data.get("current_key", None)
                 pending_job = user_store_data.get("pending_job", None)
 
@@ -1276,9 +1284,16 @@ def callback(_triggers, user_store_data, user_callback_args):
                     # Set current key (hash of data stored in client)
                     # to pending key (hash of data requested by client)
                     user_store_data["current_key"] = pending_key
+
+                    # Disable interval if this value was pulled from cache.
+                    # If this value was the result of a background calculation, don't
+                    # disable yet. If no other calculations are in progress,
+                    # interval will be disabled in should_cancel logic above
+                    # the next time the interval fires.
+                    interval_disabled = pending_job is None
                     return dict(
                         user_callback_output=result,
-                        interval_disabled=False,
+                        interval_disabled=interval_disabled,
                         in_progress=[val for (_, _, val) in running],
                         progress=clear_progress,
                         user_store_data=user_store_data,
diff --git a/dash/long_callback/managers/__init__.py b/dash/long_callback/managers/__init__.py
index 14826f39f4..70a876e139 100644
--- a/dash/long_callback/managers/__init__.py
+++ b/dash/long_callback/managers/__init__.py
@@ -34,8 +34,20 @@ def result_ready(self, key):
     def get_result(self, key, job):
         raise NotImplementedError
 
-    def build_cache_key(self, fn, args):
+    def build_cache_key(self, fn, args, cache_args_to_ignore):
         fn_source = inspect.getsource(fn)
+
+        if not isinstance(cache_args_to_ignore, (list, tuple)):
+            cache_args_to_ignore = [cache_args_to_ignore]
+
+        if cache_args_to_ignore:
+            if isinstance(args, dict):
+                args = {k: v for k, v in args.items() if k not in cache_args_to_ignore}
+            else:
+                args = [
+                    arg for i, arg in enumerate(args) if i not in cache_args_to_ignore
+                ]
+
         hash_dict = dict(args=args, fn_source=fn_source)
 
         if self.cache_by is not None:
diff --git a/tests/integration/long_callback/app5.py b/tests/integration/long_callback/app5.py
index 872e51f5ee..baed6a4a49 100644
--- a/tests/integration/long_callback/app5.py
+++ b/tests/integration/long_callback/app5.py
@@ -40,6 +40,7 @@ def cache_fn():
     progress=Output("status", "children"),
     progress_default="Finished",
     interval=500,
+    cache_args_to_ignore=0,
 )
 def update_output(set_progress, _n_clicks, value):
     for i in range(4):
diff --git a/tests/integration/long_callback/test_long_callback.py b/tests/integration/long_callback/test_long_callback.py
index cb72e33f45..8eb2bb4461 100644
--- a/tests/integration/long_callback/test_long_callback.py
+++ b/tests/integration/long_callback/test_long_callback.py
@@ -9,6 +9,9 @@
 
 from dash.testing.application_runners import import_app
 import psutil
+import redis
+
+parent_dir = os.path.dirname(os.path.realpath(__file__))
 
 
 def kill(proc_pid):
@@ -27,6 +30,17 @@ def manager(request):
 def setup_long_callback_app(manager_name, app_name):
     if manager_name == "celery":
         os.environ["LONG_CALLBACK_MANAGER"] = "celery"
+        os.environ["CELERY_BROKER"] = "redis://localhost:6379/0"
+        os.environ["CELERY_BACKEND"] = "redis://localhost:6379/1"
+
+        # Clear redis of cached values
+        redis_conn = redis.Redis(host="localhost", port=6379, db=1)
+        cache_keys = redis_conn.keys()
+        if cache_keys:
+            redis_conn.delete(*cache_keys)
+
+        print(f"parent_dir: {parent_dir}")
+
         worker = subprocess.Popen(
             [
                 "celery",
@@ -38,6 +52,7 @@ def setup_long_callback_app(manager_name, app_name):
                 "--loglevel=info",
             ],
             preexec_fn=os.setpgrp,
+            cwd=parent_dir,
         )
         try:
             yield import_app(app_name)
@@ -46,6 +61,8 @@ def setup_long_callback_app(manager_name, app_name):
             # Sleep for 1 interval of time
             time.sleep(0.5)
             os.environ.pop("LONG_CALLBACK_MANAGER")
+            os.environ.pop("CELERY_BROKER")
+            os.environ.pop("CELERY_BACKEND")
             kill(worker.pid)
 
     elif manager_name == "diskcache":
@@ -200,46 +217,46 @@ def test_lcb005_long_callback_caching(dash_duo, manager):
         dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
         dash_duo.wait_for_text_to_equal("#result", "Result for 'BBB'", 4)
 
-        # # Update input text box back to AAA
-        # input_ = dash_duo.find_element("#input")
-        # dash_duo.clear_input(input_)
-        # for key in "AAA":
-        #     with lock:
-        #         input_.send_keys(key)
-        #
-        # # Click run button and this time the cached result is used,
-        # # So we can get the result right away
-        # dash_duo.find_element("#run-button").click()
-        # dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
-        # dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 4)
-
-        # # Update input text box back to BBB
-        # input_ = dash_duo.find_element("#input")
-        # dash_duo.clear_input(input_)
-        # for key in "BBB":
-        #     with lock:
-        #         input_.send_keys(key)
-        #
-        # # Click run button and this time the cached result is used,
-        # # So we can get the result right away
-        # dash_duo.find_element("#run-button").click()
-        # dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
-        # dash_duo.wait_for_text_to_equal("#result", "Result for 'BBB'", 4)
-        #
-        # # Update input text box back to AAA
-        # input_ = dash_duo.find_element("#input")
-        # dash_duo.clear_input(input_)
-        # for key in "AAA":
-        #     with lock:
-        #         input_.send_keys(key)
-        #
-        # # Change cache key
-        # app._cache_key.value = 1
-        #
-        # dash_duo.find_element("#run-button").click()
-        # dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 20)
-        # dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
-        # dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 4)
-        #
-        # assert not dash_duo.redux_state_is_loading
-        # assert dash_duo.get_logs() == []
+        # Update input text box back to AAA
+        input_ = dash_duo.find_element("#input")
+        dash_duo.clear_input(input_)
+        for key in "AAA":
+            with lock:
+                input_.send_keys(key)
+
+        # Click run button and this time the cached result is used,
+        # So we can get the result right away
+        dash_duo.find_element("#run-button").click()
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+        dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 4)
+
+        # Update input text box back to BBB
+        input_ = dash_duo.find_element("#input")
+        dash_duo.clear_input(input_)
+        for key in "BBB":
+            with lock:
+                input_.send_keys(key)
+
+        # Click run button and this time the cached result is used,
+        # So we can get the result right away
+        dash_duo.find_element("#run-button").click()
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+        dash_duo.wait_for_text_to_equal("#result", "Result for 'BBB'", 4)
+
+        # Update input text box back to AAA
+        input_ = dash_duo.find_element("#input")
+        dash_duo.clear_input(input_)
+        for key in "AAA":
+            with lock:
+                input_.send_keys(key)
+
+        # Change cache key
+        app._cache_key.value = 1
+
+        dash_duo.find_element("#run-button").click()
+        dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 20)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
+        dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 4)
+
+        assert not dash_duo.redux_state_is_loading
+        assert dash_duo.get_logs() == []
diff --git a/tests/integration/long_callback/utils.py b/tests/integration/long_callback/utils.py
index 51e25282bb..c6882df1fc 100644
--- a/tests/integration/long_callback/utils.py
+++ b/tests/integration/long_callback/utils.py
@@ -11,8 +11,8 @@ def get_long_callback_manager():
 
         celery_app = Celery(
             __name__,
-            broker="redis://localhost:6379/0",
-            backend="redis://localhost:6379/1",
+            broker=os.environ.get("CELERY_BROKER"),
+            backend=os.environ.get("CELERY_BACKEND"),
         )
         long_callback_manager = CeleryLongCallbackManager(celery_app)
     elif os.environ.get("LONG_CALLBACK_MANAGER", None) == "diskcache":

From a1b8a398ba3460b748d71584f3cc780cf312507b Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 14 Aug 2021 14:13:01 -0400
Subject: [PATCH 12/32] Add dual long_callback test Set explicit celery task
 name to avoid task conflict

---
 dash/dash.py                                  |  30 ++++-
 dash/long_callback/managers/celery_manager.py |  12 +-
 tests/integration/long_callback/app1.py       |   2 +-
 tests/integration/long_callback/app2.py       |   3 +-
 tests/integration/long_callback/app3.py       |   2 +-
 tests/integration/long_callback/app4.py       |   3 +-
 tests/integration/long_callback/app5.py       |   3 +-
 tests/integration/long_callback/app6.py       |  71 ++++++++++
 ...allback.py => test_basic_long_callback.py} | 122 +++++++++++++++++-
 9 files changed, 224 insertions(+), 24 deletions(-)
 create mode 100644 tests/integration/long_callback/app6.py
 rename tests/integration/long_callback/{test_long_callback.py => test_basic_long_callback.py} (66%)

diff --git a/dash/dash.py b/dash/dash.py
index 3c1187b1a1..612def5a93 100644
--- a/dash/dash.py
+++ b/dash/dash.py
@@ -263,6 +263,10 @@ class Dash(object):
     Set to None or '' if you don't want the document.title to change or if you
     want to control the document.title through a separate component or
     clientside callback.
+
+    :param long_callback_manager: Long callback manager instance to support the
+    ``@app.long_callback`` decorator. Currently one of
+    ``DiskcacheLongCallbackManager`` or ``CeleryLongCallbackManager``
     """
 
     def __init__(
@@ -291,6 +295,7 @@ def __init__(
         plugins=None,
         title="Dash",
         update_title="Updating...",
+        long_callback_manager=None,
         **obsolete,
     ):
         _validate.check_obsolete(obsolete)
@@ -415,6 +420,7 @@ def __init__(
 
         self._assets_files = []
         self._long_callback_count = 0
+        self._long_callback_manager = long_callback_manager
 
         self.logger = logging.getLogger(name)
         self.logger.addHandler(logging.StreamHandler(stream=sys.stdout))
@@ -1129,7 +1135,7 @@ def add_context(*args, **kwargs):
 
         return wrap_func
 
-    def long_callback(self, callback_manager, *_args, **_kwargs):
+    def long_callback(self, *_args, **_kwargs):
         """
         Normally used as a decorator, `@app.long_callback` is an alternative to
         `@app.callback` designed for callbacks that take a long time to run,
@@ -1146,17 +1152,16 @@ def long_callback(self, callback_manager, *_args, **_kwargs):
               in a celery worker and returns results to the Dash app through a Celery
               broker like RabbitMQ or Redis.
 
-        The first argument to `@long_callback` should be a callback manager instance.
-
-        :param callback_manager:
-            A long callback manager instance. Currently one of
-            `DiskcacheLongCallbackManager` or `CeleryLongCallbackManager`
-
         The following arguments may include any valid arguments to `@app.callback`.
         In addition, `@app.long_callback` supports the following optional
         keyword arguments:
 
         :Keyword Arguments:
+            :param manager:
+                A long callback manager instance. Currently one of
+                `DiskcacheLongCallbackManager` or `CeleryLongCallbackManager`.
+                Defaults to the `long_callback_manager` instance provided to the
+                `dash.Dash constructor`.
             :param running:
                 A list of 3-element tuples. The first element of each tuple should be
                 an `Output` dependency object referencing a property of a component in
@@ -1195,6 +1200,17 @@ def long_callback(self, callback_manager, *_args, **_kwargs):
         )
         import dash_core_components as dcc  # pylint: disable=import-outside-toplevel
 
+        # Get long callback manager
+        callback_manager = _kwargs.pop("manager", self._long_callback_manager)
+        if callback_manager is None:
+            raise ValueError(
+                "The @app.long_callback decorator requires a long callback manager\n"
+                "instance.  This may be provided to the app using the \n"
+                "long_callback_manager argument to the dash.Dash constructor, or\n"
+                "it may be provided to the @app.long_callback decorator as the \n"
+                "manager argument"
+            )
+
         # Extract special long_callback kwargs
         running = _kwargs.pop("running", ())
         cancel = _kwargs.pop("cancel", ())
diff --git a/dash/long_callback/managers/celery_manager.py b/dash/long_callback/managers/celery_manager.py
index 0907095fba..b8a1fa1650 100644
--- a/dash/long_callback/managers/celery_manager.py
+++ b/dash/long_callback/managers/celery_manager.py
@@ -1,4 +1,6 @@
 import json
+import inspect
+import hashlib
 
 from _plotly_utils.utils import PlotlyJSONEncoder
 from dash.long_callback.managers import BaseLongCallbackManager
@@ -88,8 +90,14 @@ def get_result(self, key, job):
 def _make_job_fn(fn, celery_app, progress):
     cache = celery_app.backend
 
-    @celery_app.task
-    def job_fn(result_key, progress_key, user_callback_args):
+    # Hash function source and module to create a unique (but stable) celery task name
+    fn_source = inspect.getsource(fn)
+    fn_module = fn.__module__
+    fn_str = fn_module + "\n" + fn_source
+    fn_hash = hashlib.sha1(fn_str.encode("utf-8")).hexdigest()
+
+    @celery_app.task(name=fn_hash)
+    def job_fn(result_key, progress_key, user_callback_args, fn=fn):
         def _set_progress(progress_value):
             cache.set(progress_key, json.dumps(progress_value, cls=PlotlyJSONEncoder))
 
diff --git a/tests/integration/long_callback/app1.py b/tests/integration/long_callback/app1.py
index 3dbb20be64..253d39496d 100644
--- a/tests/integration/long_callback/app1.py
+++ b/tests/integration/long_callback/app1.py
@@ -19,10 +19,10 @@
 
 
 @app.long_callback(
-    long_callback_manager,
     Output("output-1", "children"),
     [Input("input", "value")],
     interval=500,
+    manager=long_callback_manager,
 )
 def update_output(value):
     time.sleep(0.1)
diff --git a/tests/integration/long_callback/app2.py b/tests/integration/long_callback/app2.py
index e7a7ed6a83..b622852a89 100644
--- a/tests/integration/long_callback/app2.py
+++ b/tests/integration/long_callback/app2.py
@@ -8,7 +8,7 @@
 long_callback_manager = get_long_callback_manager()
 handle = long_callback_manager.handle
 
-app = dash.Dash(__name__)
+app = dash.Dash(__name__, long_callback_manager=long_callback_manager)
 app.layout = html.Div(
     [
         html.Button(id="button-1", children="Click Here", n_clicks=0),
@@ -19,7 +19,6 @@
 
 
 @app.long_callback(
-    long_callback_manager,
     Output("result", "children"),
     [Input("button-1", "n_clicks")],
     running=[(Output("status", "children"), "Running", "Finished")],
diff --git a/tests/integration/long_callback/app3.py b/tests/integration/long_callback/app3.py
index da022dd7af..c74d34e57a 100644
--- a/tests/integration/long_callback/app3.py
+++ b/tests/integration/long_callback/app3.py
@@ -22,12 +22,12 @@
 
 
 @app.long_callback(
-    long_callback_manager,
     Output("result", "children"),
     [Input("run-button", "n_clicks"), State("input", "value")],
     running=[(Output("status", "children"), "Running", "Finished")],
     cancel=[Input("cancel-button", "n_clicks")],
     interval=500,
+    manager=long_callback_manager,
 )
 def update_output(n_clicks, value):
     time.sleep(2)
diff --git a/tests/integration/long_callback/app4.py b/tests/integration/long_callback/app4.py
index 331cdce04d..1d6ae5bee3 100644
--- a/tests/integration/long_callback/app4.py
+++ b/tests/integration/long_callback/app4.py
@@ -10,7 +10,7 @@
 handle = long_callback_manager.handle
 
 
-app = dash.Dash(__name__)
+app = dash.Dash(__name__, long_callback_manager=long_callback_manager)
 app.layout = html.Div(
     [
         dcc.Input(id="input", value="hello, world"),
@@ -23,7 +23,6 @@
 
 
 @app.long_callback(
-    long_callback_manager,
     Output("result", "children"),
     [Input("run-button", "n_clicks"), State("input", "value")],
     progress=Output("status", "children"),
diff --git a/tests/integration/long_callback/app5.py b/tests/integration/long_callback/app5.py
index baed6a4a49..d2d988070b 100644
--- a/tests/integration/long_callback/app5.py
+++ b/tests/integration/long_callback/app5.py
@@ -11,7 +11,7 @@
 handle = long_callback_manager.handle
 
 
-app = dash.Dash(__name__)
+app = dash.Dash(__name__, long_callback_manager=long_callback_manager)
 app._cache_key = Value("i", 0)
 
 
@@ -34,7 +34,6 @@ def cache_fn():
 
 
 @app.long_callback(
-    long_callback_manager,
     Output("result", "children"),
     [Input("run-button", "n_clicks"), State("input", "value")],
     progress=Output("status", "children"),
diff --git a/tests/integration/long_callback/app6.py b/tests/integration/long_callback/app6.py
new file mode 100644
index 0000000000..c81d279602
--- /dev/null
+++ b/tests/integration/long_callback/app6.py
@@ -0,0 +1,71 @@
+import dash
+from dash.dependencies import Input, State, Output
+import dash_html_components as html
+import dash_core_components as dcc
+import time
+from multiprocessing import Value
+
+from utils import get_long_callback_manager
+
+long_callback_manager = get_long_callback_manager()
+handle = long_callback_manager.handle
+
+app = dash.Dash(__name__, long_callback_manager=long_callback_manager)
+app._cache_key = Value("i", 0)
+
+
+# Control return value of cache_by function using multiprocessing value
+def cache_fn():
+    return app._cache_key.value
+
+
+long_callback_manager.cache_by = [cache_fn]
+
+
+app.layout = html.Div(
+    [
+        dcc.Input(id="input1", value="AAA"),
+        html.Button(id="run-button1", children="Run"),
+        html.Div(id="status1", children="Finished"),
+        html.Div(id="result1", children="No results"),
+        html.Hr(),
+        dcc.Input(id="input2", value="aaa"),
+        html.Button(id="run-button2", children="Run"),
+        html.Div(id="status2", children="Finished"),
+        html.Div(id="result2", children="No results"),
+    ]
+)
+
+
+@app.long_callback(
+    Output("result1", "children"),
+    [Input("run-button1", "n_clicks"), State("input1", "value")],
+    progress=Output("status1", "children"),
+    progress_default="Finished",
+    interval=500,
+    cache_args_to_ignore=[0],
+)
+def update_output1(set_progress, _n_clicks, value):
+    for i in range(4):
+        set_progress(f"Progress {i}/4")
+        time.sleep(2)
+    return f"Result for '{value}'"
+
+
+@app.long_callback(
+    Output("result2", "children"),
+    dict(button=Input("run-button2", "n_clicks"), value=State("input2", "value")),
+    progress=Output("status2", "children"),
+    progress_default="Finished",
+    interval=500,
+    cache_args_to_ignore=["button"],
+)
+def update_output2(set_progress, button, value):
+    for i in range(4):
+        set_progress(f"Progress {i}/4")
+        time.sleep(2)
+    return f"Result for '{value}'"
+
+
+if __name__ == "__main__":
+    app.run_server(debug=True)
diff --git a/tests/integration/long_callback/test_long_callback.py b/tests/integration/long_callback/test_basic_long_callback.py
similarity index 66%
rename from tests/integration/long_callback/test_long_callback.py
rename to tests/integration/long_callback/test_basic_long_callback.py
index 8eb2bb4461..bc19ff062b 100644
--- a/tests/integration/long_callback/test_long_callback.py
+++ b/tests/integration/long_callback/test_basic_long_callback.py
@@ -21,7 +21,12 @@ def kill(proc_pid):
     process.kill()
 
 
-@pytest.fixture(params=["diskcache", "celery"])
+@pytest.fixture(
+    params=[
+        # "diskcache",
+        "celery"
+    ]
+)
 def manager(request):
     return request.param
 
@@ -48,7 +53,7 @@ def setup_long_callback_app(manager_name, app_name):
                 f"{app_name}:handle",
                 "worker",
                 "--concurrency",
-                "2",
+                "1",
                 "--loglevel=info",
             ],
             preexec_fn=os.setpgrp,
@@ -81,7 +86,7 @@ def setup_long_callback_app(manager_name, app_name):
             os.environ.pop("DISKCACHE_DIR")
 
 
-def test_lcb001_fast_input(dash_duo, manager):
+def test_lcbc001_fast_input(dash_duo, manager):
     """
     Make sure that we settle to the correct final value when handling rapid inputs
     """
@@ -102,7 +107,7 @@ def test_lcb001_fast_input(dash_duo, manager):
     assert dash_duo.get_logs() == []
 
 
-def test_lcb002_long_callback_running(dash_duo, manager):
+def test_lcbc002_long_callback_running(dash_duo, manager):
     with setup_long_callback_app(manager, "app2") as app:
         dash_duo.start_server(app)
         dash_duo.wait_for_text_to_equal("#result", "Clicked 0 time(s)", 15)
@@ -129,7 +134,7 @@ def test_lcb002_long_callback_running(dash_duo, manager):
     assert dash_duo.get_logs() == []
 
 
-def test_lcb003_long_callback_running_cancel(dash_duo, manager):
+def test_lcbc003_long_callback_running_cancel(dash_duo, manager):
     lock = Lock()
 
     with setup_long_callback_app(manager, "app3") as app:
@@ -165,7 +170,7 @@ def test_lcb003_long_callback_running_cancel(dash_duo, manager):
     assert dash_duo.get_logs() == []
 
 
-def test_lcb004_long_callback_progress(dash_duo, manager):
+def test_lcbc004_long_callback_progress(dash_duo, manager):
     with setup_long_callback_app(manager, "app4") as app:
         dash_duo.start_server(app)
 
@@ -195,7 +200,7 @@ def test_lcb004_long_callback_progress(dash_duo, manager):
     assert dash_duo.get_logs() == []
 
 
-def test_lcb005_long_callback_caching(dash_duo, manager):
+def test_lcbc005_long_callback_caching(dash_duo, manager):
     lock = Lock()
 
     with setup_long_callback_app(manager, "app5") as app:
@@ -260,3 +265,106 @@ def test_lcb005_long_callback_caching(dash_duo, manager):
 
         assert not dash_duo.redux_state_is_loading
         assert dash_duo.get_logs() == []
+
+
+def test_lcbc006_long_callback_caching_multi(dash_duo, manager):
+    lock = Lock()
+
+    with setup_long_callback_app(manager, "app6") as app:
+        dash_duo.start_server(app)
+        dash_duo.wait_for_text_to_equal("#status1", "Progress 2/4", 15)
+        dash_duo.wait_for_text_to_equal("#status1", "Finished", 15)
+        dash_duo.wait_for_text_to_equal("#result1", "Result for 'AAA'", 4)
+
+        # Check initial status/output of second long_callback
+        dash_duo.wait_for_text_to_equal("#status2", "Finished", 15)
+        dash_duo.wait_for_text_to_equal("#result2", "Result for 'aaa'", 4)
+
+        # Update input text box to BBB
+        input_ = dash_duo.find_element("#input1")
+        dash_duo.clear_input(input_)
+        for key in "BBB":
+            with lock:
+                input_.send_keys(key)
+
+        # Click run button and check that status eventually cycles to 2/4
+        dash_duo.find_element("#run-button1").click()
+        dash_duo.wait_for_text_to_equal("#status1", "Progress 2/4", 20)
+        dash_duo.wait_for_text_to_equal("#status1", "Finished", 8)
+        dash_duo.wait_for_text_to_equal("#result1", "Result for 'BBB'", 4)
+
+        # Check there were no changes in second long_callback output
+        dash_duo.wait_for_text_to_equal("#status2", "Finished", 15)
+        dash_duo.wait_for_text_to_equal("#result2", "Result for 'aaa'", 4)
+
+        # Update input text box back to AAA
+        input_ = dash_duo.find_element("#input1")
+        dash_duo.clear_input(input_)
+        for key in "AAA":
+            with lock:
+                input_.send_keys(key)
+
+        # Click run button and this time the cached result is used,
+        # So we can get the result right away
+        dash_duo.find_element("#run-button1").click()
+        dash_duo.wait_for_text_to_equal("#status1", "Finished", 4)
+        dash_duo.wait_for_text_to_equal("#result1", "Result for 'AAA'", 4)
+
+        # Update input text box back to BBB
+        input_ = dash_duo.find_element("#input1")
+        dash_duo.clear_input(input_)
+        for key in "BBB":
+            with lock:
+                input_.send_keys(key)
+
+        # Click run button and this time the cached result is used,
+        # So we can get the result right away
+        dash_duo.find_element("#run-button1").click()
+        dash_duo.wait_for_text_to_equal("#status1", "Finished", 4)
+        dash_duo.wait_for_text_to_equal("#result1", "Result for 'BBB'", 4)
+
+        # Update second input text box to BBB, make sure there is not a cache hit
+        input_ = dash_duo.find_element("#input2")
+        dash_duo.clear_input(input_)
+        for key in "BBB":
+            with lock:
+                input_.send_keys(key)
+        dash_duo.find_element("#run-button2").click()
+        dash_duo.wait_for_text_to_equal("#status2", "Progress 2/4", 20)
+        dash_duo.wait_for_text_to_equal("#status2", "Finished", 8)
+        dash_duo.wait_for_text_to_equal("#result2", "Result for 'BBB'", 4)
+
+        # Update second input text box back to aaa, check for cache hit
+        input_ = dash_duo.find_element("#input2")
+        dash_duo.clear_input(input_)
+        for key in "aaa":
+            with lock:
+                input_.send_keys(key)
+        dash_duo.find_element("#run-button2").click()
+        dash_duo.wait_for_text_to_equal("#status2", "Finished", 4)
+        dash_duo.wait_for_text_to_equal("#result2", "Result for 'aaa'", 4)
+
+        # Update input text box back to AAA
+        input_ = dash_duo.find_element("#input1")
+        dash_duo.clear_input(input_)
+        for key in "AAA":
+            with lock:
+                input_.send_keys(key)
+
+        # Change cache key to cause cache miss
+        app._cache_key.value = 1
+
+        # Check for cache miss for first long_callback
+        dash_duo.find_element("#run-button1").click()
+        dash_duo.wait_for_text_to_equal("#status1", "Progress 2/4", 20)
+        dash_duo.wait_for_text_to_equal("#status1", "Finished", 8)
+        dash_duo.wait_for_text_to_equal("#result1", "Result for 'AAA'", 4)
+
+        # Check for cache miss for second long_callback
+        dash_duo.find_element("#run-button2").click()
+        dash_duo.wait_for_text_to_equal("#status2", "Progress 2/4", 20)
+        dash_duo.wait_for_text_to_equal("#status2", "Finished", 4)
+        dash_duo.wait_for_text_to_equal("#result2", "Result for 'aaa'", 4)
+
+        assert not dash_duo.redux_state_is_loading
+        assert dash_duo.get_logs() == []

From 5d643328d0eaf33b6d773c23a5c444b36ce83c76 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 14 Aug 2021 15:04:42 -0400
Subject: [PATCH 13/32] Add dual long_callback test Set explicit celery task
 name to avoid task conflict

---
 dash/long_callback/managers/diskcache_manager.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/dash/long_callback/managers/diskcache_manager.py b/dash/long_callback/managers/diskcache_manager.py
index 791731bc04..9ba270f37a 100644
--- a/dash/long_callback/managers/diskcache_manager.py
+++ b/dash/long_callback/managers/diskcache_manager.py
@@ -42,7 +42,10 @@ def terminate_job(self, job):
                 for proc in process.children(recursive=True):
                     proc.kill()
                 process.kill()
-                process.wait(1)
+                try:
+                    process.wait(0.5)
+                except psutil.TimeoutExpired:
+                    pass
 
     def terminate_unhealthy_job(self, job):
         import psutil  # pylint: disable=import-outside-toplevel

From d50f214753a7a61137bfa6dff54e3f044fa5ec16 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 14 Aug 2021 15:47:32 -0400
Subject: [PATCH 14/32] celery tests on circleci (take 1)

---
 .circleci/config.yml                          |  4 ++++
 requires-testing.txt                          |  3 +++
 .../long_callback/test_basic_long_callback.py | 19 +++++++++++--------
 3 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 82204ce4a8..1c7cba3224 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -208,6 +208,8 @@ jobs:
           PERCY_PARALLEL_TOTAL: -1
           PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: True
           PYVERSION: python39
+          REDIS_URL: redis://localhost:6379
+      - image: circleci/redis
     parallelism: 3
     steps:
       - checkout
@@ -252,6 +254,8 @@ jobs:
         environment:
           PERCY_ENABLE: 0
           PYVERSION: python36
+          REDIS_URL: redis://localhost:6379
+      - image: circleci/redis
 
 workflows:
   version: 2
diff --git a/requires-testing.txt b/requires-testing.txt
index c3ce4fbece..906a9c6302 100644
--- a/requires-testing.txt
+++ b/requires-testing.txt
@@ -10,3 +10,6 @@ beautifulsoup4>=4.8.2
 waitress>=1.4.4
 diskcache>=5.2.1
 multiprocess>=0.70.12
+redis>=3.5.3
+psutil>=5.8.0
+celery>=5.1.2
diff --git a/tests/integration/long_callback/test_basic_long_callback.py b/tests/integration/long_callback/test_basic_long_callback.py
index bc19ff062b..3079addb05 100644
--- a/tests/integration/long_callback/test_basic_long_callback.py
+++ b/tests/integration/long_callback/test_basic_long_callback.py
@@ -21,12 +21,14 @@ def kill(proc_pid):
     process.kill()
 
 
-@pytest.fixture(
-    params=[
-        # "diskcache",
-        "celery"
-    ]
-)
+if "REDIS_URL" in os.environ:
+    managers = ["diskcache", "celery"]
+else:
+    print("Skipping celery tests because REDIS_URL is not defined")
+    managers = ["diskcache"]
+
+
+@pytest.fixture(params=managers)
 def manager(request):
     return request.param
 
@@ -35,8 +37,9 @@ def manager(request):
 def setup_long_callback_app(manager_name, app_name):
     if manager_name == "celery":
         os.environ["LONG_CALLBACK_MANAGER"] = "celery"
-        os.environ["CELERY_BROKER"] = "redis://localhost:6379/0"
-        os.environ["CELERY_BACKEND"] = "redis://localhost:6379/1"
+        redis_url = os.environ["REDIS_URL"].rstrip("/")
+        os.environ["CELERY_BROKER"] = f"{redis_url}/0"
+        os.environ["CELERY_BACKEND"] = f"{redis_url}/1"
 
         # Clear redis of cached values
         redis_conn = redis.Redis(host="localhost", port=6379, db=1)

From 809f5da75b86829f0f99078674fd7062f7a4aa23 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 14 Aug 2021 15:55:51 -0400
Subject: [PATCH 15/32] pylist fixes

---
 dash/long_callback/managers/celery_manager.py    | 10 +++++-----
 dash/long_callback/managers/diskcache_manager.py |  4 ++--
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/dash/long_callback/managers/celery_manager.py b/dash/long_callback/managers/celery_manager.py
index b8a1fa1650..553dc5e7c2 100644
--- a/dash/long_callback/managers/celery_manager.py
+++ b/dash/long_callback/managers/celery_manager.py
@@ -8,7 +8,7 @@
 
 class CeleryLongCallbackManager(BaseLongCallbackManager):
     def __init__(self, celery_app, cache_by=None, expire=None):
-        import celery
+        import celery  # pylint: disable=import-outside-toplevel
 
         if not isinstance(celery_app, celery.Celery):
             raise ValueError("First argument must be a celery.Celery object")
@@ -45,8 +45,8 @@ def make_job_fn(self, fn, progress=False):
     def get_task(self, job):
         if job:
             return self.handle.AsyncResult(job)
-        else:
-            return None
+
+        return None
 
     def clear_cache_entry(self, key):
         self.handle.backend.delete(key)
@@ -60,8 +60,8 @@ def get_progress(self, key):
         progress_data = self.handle.backend.get(progress_key)
         if progress_data:
             return json.loads(progress_data)
-        else:
-            return None
+
+        return None
 
     def result_ready(self, key):
         return self.handle.backend.get(key) is not None
diff --git a/dash/long_callback/managers/diskcache_manager.py b/dash/long_callback/managers/diskcache_manager.py
index 9ba270f37a..e41fc71dfb 100644
--- a/dash/long_callback/managers/diskcache_manager.py
+++ b/dash/long_callback/managers/diskcache_manager.py
@@ -7,8 +7,8 @@ class DiskcacheLongCallbackManager(BaseLongCallbackManager):
     def __init__(self, cache, cache_by=None, expire=None):
         try:
             import diskcache  # pylint: disable=import-outside-toplevel
-            import psutil  # noqa: F401,E402 pylint: disable=import-outside-toplevel,unused-variable
-            import multiprocess  # noqa: F401,E402 pylint: disable=import-outside-toplevel,unused-variable
+            import psutil  # noqa: F401,E402 pylint: disable=import-outside-toplevel,unused-import,unused-variable
+            import multiprocess  # noqa: F401,E402 pylint: disable=import-outside-toplevel,unused-import,unused-variable
         except ImportError:
             raise ImportError(
                 """\

From 9ed1b813fed56f26a4519a235d9078d33292e76d Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 14 Aug 2021 16:15:57 -0400
Subject: [PATCH 16/32] pylist fixes

---
 tests/integration/long_callback/__init__.py           |  0
 tests/integration/long_callback/app1.py               |  2 +-
 tests/integration/long_callback/app2.py               |  2 +-
 tests/integration/long_callback/app3.py               |  2 +-
 tests/integration/long_callback/app4.py               |  2 +-
 tests/integration/long_callback/app5.py               |  2 +-
 tests/integration/long_callback/app6.py               |  2 +-
 .../long_callback/test_basic_long_callback.py         | 11 +++--------
 8 files changed, 9 insertions(+), 14 deletions(-)
 create mode 100644 tests/integration/long_callback/__init__.py

diff --git a/tests/integration/long_callback/__init__.py b/tests/integration/long_callback/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/integration/long_callback/app1.py b/tests/integration/long_callback/app1.py
index 253d39496d..4cb61d9903 100644
--- a/tests/integration/long_callback/app1.py
+++ b/tests/integration/long_callback/app1.py
@@ -4,7 +4,7 @@
 import dash_core_components as dcc
 import time
 
-from utils import get_long_callback_manager
+from tests.integration.long_callback.utils import get_long_callback_manager
 
 long_callback_manager = get_long_callback_manager()
 handle = long_callback_manager.handle
diff --git a/tests/integration/long_callback/app2.py b/tests/integration/long_callback/app2.py
index b622852a89..206b502723 100644
--- a/tests/integration/long_callback/app2.py
+++ b/tests/integration/long_callback/app2.py
@@ -3,7 +3,7 @@
 import dash_html_components as html
 import time
 
-from utils import get_long_callback_manager
+from tests.integration.long_callback.utils import get_long_callback_manager
 
 long_callback_manager = get_long_callback_manager()
 handle = long_callback_manager.handle
diff --git a/tests/integration/long_callback/app3.py b/tests/integration/long_callback/app3.py
index c74d34e57a..0b6681173f 100644
--- a/tests/integration/long_callback/app3.py
+++ b/tests/integration/long_callback/app3.py
@@ -4,7 +4,7 @@
 import dash_core_components as dcc
 import time
 
-from utils import get_long_callback_manager
+from tests.integration.long_callback.utils import get_long_callback_manager
 
 long_callback_manager = get_long_callback_manager()
 handle = long_callback_manager.handle
diff --git a/tests/integration/long_callback/app4.py b/tests/integration/long_callback/app4.py
index 1d6ae5bee3..3f63f88bcb 100644
--- a/tests/integration/long_callback/app4.py
+++ b/tests/integration/long_callback/app4.py
@@ -4,7 +4,7 @@
 import dash_core_components as dcc
 import time
 
-from utils import get_long_callback_manager
+from tests.integration.long_callback.utils import get_long_callback_manager
 
 long_callback_manager = get_long_callback_manager()
 handle = long_callback_manager.handle
diff --git a/tests/integration/long_callback/app5.py b/tests/integration/long_callback/app5.py
index d2d988070b..c319d3de46 100644
--- a/tests/integration/long_callback/app5.py
+++ b/tests/integration/long_callback/app5.py
@@ -5,7 +5,7 @@
 import time
 from multiprocessing import Value
 
-from utils import get_long_callback_manager
+from tests.integration.long_callback.utils import get_long_callback_manager
 
 long_callback_manager = get_long_callback_manager()
 handle = long_callback_manager.handle
diff --git a/tests/integration/long_callback/app6.py b/tests/integration/long_callback/app6.py
index c81d279602..06b4461446 100644
--- a/tests/integration/long_callback/app6.py
+++ b/tests/integration/long_callback/app6.py
@@ -5,7 +5,7 @@
 import time
 from multiprocessing import Value
 
-from utils import get_long_callback_manager
+from tests.integration.long_callback.utils import get_long_callback_manager
 
 long_callback_manager = get_long_callback_manager()
 handle = long_callback_manager.handle
diff --git a/tests/integration/long_callback/test_basic_long_callback.py b/tests/integration/long_callback/test_basic_long_callback.py
index 3079addb05..610a193f37 100644
--- a/tests/integration/long_callback/test_basic_long_callback.py
+++ b/tests/integration/long_callback/test_basic_long_callback.py
@@ -11,8 +11,6 @@
 import psutil
 import redis
 
-parent_dir = os.path.dirname(os.path.realpath(__file__))
-
 
 def kill(proc_pid):
     process = psutil.Process(proc_pid)
@@ -47,23 +45,20 @@ def setup_long_callback_app(manager_name, app_name):
         if cache_keys:
             redis_conn.delete(*cache_keys)
 
-        print(f"parent_dir: {parent_dir}")
-
         worker = subprocess.Popen(
             [
                 "celery",
                 "-A",
-                f"{app_name}:handle",
+                f"tests.integration.long_callback.{app_name}:handle",
                 "worker",
                 "--concurrency",
                 "1",
                 "--loglevel=info",
             ],
             preexec_fn=os.setpgrp,
-            cwd=parent_dir,
         )
         try:
-            yield import_app(app_name)
+            yield import_app(f"tests.integration.long_callback.{app_name}")
         finally:
             # Interval may run one more time after settling on final app state
             # Sleep for 1 interval of time
@@ -79,7 +74,7 @@ def setup_long_callback_app(manager_name, app_name):
         print(cache_directory)
         os.environ["DISKCACHE_DIR"] = cache_directory
         try:
-            yield import_app(app_name)
+            yield import_app(f"tests.integration.long_callback.{app_name}")
         finally:
             # Interval may run one more time after settling on final app state
             # Sleep for 1 interval of time

From f5eec9d0cd55363c6cdb55e8f0b86165f3613d3d Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 14 Aug 2021 18:36:01 -0400
Subject: [PATCH 17/32] CI WIP

---
 .circleci/config.yml                          | 48 +++++++++----------
 package.json                                  |  2 +-
 requires-testing.txt                          |  2 +-
 .../long_callback/test_basic_long_callback.py |  2 +-
 4 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 1c7cba3224..bd081be97f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -259,33 +259,33 @@ jobs:
 
 workflows:
   version: 2
-  python3.9:
-    jobs:
-      - lint-unit-39
-      - build-core-39
-      - build-windows-39
-      - build-misc-39
-      - test-39:
-          requires:
-            - build-core-39
-            - build-misc-39
-      - percy-finalize:
-          requires:
-            - test-39
-      - artifacts:
-          requires:
-            - percy-finalize
-          filters:
-            branches:
-              only:
-                - master
-                - dev
-            tags:
-              only: /v*/
+#  python3.9:
+#    jobs:
+#      - lint-unit-39
+#      - build-core-39
+#      - build-windows-39
+#      - build-misc-39
+#      - test-39:
+#          requires:
+#            - build-core-39
+#            - build-misc-39
+#      - percy-finalize:
+#          requires:
+#            - test-39
+#      - artifacts:
+#          requires:
+#            - percy-finalize
+#          filters:
+#            branches:
+#              only:
+#                - master
+#                - dev
+#            tags:
+#              only: /v*/
 
   python3.6:
     jobs:
-      - lint-unit-36
+#      - lint-unit-36
       - build-core-36
       - build-misc-36
       - test-36:
diff --git a/package.json b/package.json
index cc0397675c..c030349c3c 100644
--- a/package.json
+++ b/package.json
@@ -21,7 +21,7 @@
     "private::test.R.deploy-standard": "npm run private::test.setup-standard && cd \\@plotly/dash-generator-test-component-standard && sudo R CMD INSTALL .",
     "private::test.unit-dash": "pytest tests/unit",
     "private::test.unit-renderer": "cd dash/dash-renderer && npm run test",
-    "private::test.integration-dash": "TESTFILES=$(circleci tests glob \"tests/integration/**/test_*.py\" | circleci tests split --split-by=timings) && pytest --headless --nopercyfinalize --junitxml=test-reports/junit_intg.xml ${TESTFILES}",
+    "private::test.integration-dash": "TESTFILES=$(circleci tests glob \"tests/integration/**/test_*.py\" | circleci tests split --split-by=timings) && pytest -s --headless --nopercyfinalize --junitxml=test-reports/junit_intg.xml ${TESTFILES}",
     "private::test.integration-dash-import": "cd tests/integration/dash && python dash_import_test.py",
     "format": "run-s private::format.*",
     "initialize": "run-s private::initialize.*",
diff --git a/requires-testing.txt b/requires-testing.txt
index 906a9c6302..837777770d 100644
--- a/requires-testing.txt
+++ b/requires-testing.txt
@@ -12,4 +12,4 @@ diskcache>=5.2.1
 multiprocess>=0.70.12
 redis>=3.5.3
 psutil>=5.8.0
-celery>=5.1.2
+celery[redis]>=5.1.2
diff --git a/tests/integration/long_callback/test_basic_long_callback.py b/tests/integration/long_callback/test_basic_long_callback.py
index 610a193f37..647956452e 100644
--- a/tests/integration/long_callback/test_basic_long_callback.py
+++ b/tests/integration/long_callback/test_basic_long_callback.py
@@ -361,7 +361,7 @@ def test_lcbc006_long_callback_caching_multi(dash_duo, manager):
         # Check for cache miss for second long_callback
         dash_duo.find_element("#run-button2").click()
         dash_duo.wait_for_text_to_equal("#status2", "Progress 2/4", 20)
-        dash_duo.wait_for_text_to_equal("#status2", "Finished", 4)
+        dash_duo.wait_for_text_to_equal("#status2", "Finished", 8)
         dash_duo.wait_for_text_to_equal("#result2", "Result for 'aaa'", 4)
 
         assert not dash_duo.redux_state_is_loading

From f79bde3af04c8a00751edf7361ac019665ee05e5 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 14 Aug 2021 19:00:33 -0400
Subject: [PATCH 18/32] CI WIP (2)

---
 tests/integration/long_callback/test_basic_long_callback.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/integration/long_callback/test_basic_long_callback.py b/tests/integration/long_callback/test_basic_long_callback.py
index 647956452e..8952bc3ce4 100644
--- a/tests/integration/long_callback/test_basic_long_callback.py
+++ b/tests/integration/long_callback/test_basic_long_callback.py
@@ -51,6 +51,8 @@ def setup_long_callback_app(manager_name, app_name):
                 "-A",
                 f"tests.integration.long_callback.{app_name}:handle",
                 "worker",
+                "-P",
+                "prefork",
                 "--concurrency",
                 "1",
                 "--loglevel=info",

From cc82b59bc17002c09af6f8b372892043ccb0190b Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 14 Aug 2021 19:19:48 -0400
Subject: [PATCH 19/32] Re-enable tests

---
 .circleci/config.yml | 48 ++++++++++++++++++++++----------------------
 package.json         |  2 +-
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index bd081be97f..1c7cba3224 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -259,33 +259,33 @@ jobs:
 
 workflows:
   version: 2
-#  python3.9:
-#    jobs:
-#      - lint-unit-39
-#      - build-core-39
-#      - build-windows-39
-#      - build-misc-39
-#      - test-39:
-#          requires:
-#            - build-core-39
-#            - build-misc-39
-#      - percy-finalize:
-#          requires:
-#            - test-39
-#      - artifacts:
-#          requires:
-#            - percy-finalize
-#          filters:
-#            branches:
-#              only:
-#                - master
-#                - dev
-#            tags:
-#              only: /v*/
+  python3.9:
+    jobs:
+      - lint-unit-39
+      - build-core-39
+      - build-windows-39
+      - build-misc-39
+      - test-39:
+          requires:
+            - build-core-39
+            - build-misc-39
+      - percy-finalize:
+          requires:
+            - test-39
+      - artifacts:
+          requires:
+            - percy-finalize
+          filters:
+            branches:
+              only:
+                - master
+                - dev
+            tags:
+              only: /v*/
 
   python3.6:
     jobs:
-#      - lint-unit-36
+      - lint-unit-36
       - build-core-36
       - build-misc-36
       - test-36:
diff --git a/package.json b/package.json
index c030349c3c..cc0397675c 100644
--- a/package.json
+++ b/package.json
@@ -21,7 +21,7 @@
     "private::test.R.deploy-standard": "npm run private::test.setup-standard && cd \\@plotly/dash-generator-test-component-standard && sudo R CMD INSTALL .",
     "private::test.unit-dash": "pytest tests/unit",
     "private::test.unit-renderer": "cd dash/dash-renderer && npm run test",
-    "private::test.integration-dash": "TESTFILES=$(circleci tests glob \"tests/integration/**/test_*.py\" | circleci tests split --split-by=timings) && pytest -s --headless --nopercyfinalize --junitxml=test-reports/junit_intg.xml ${TESTFILES}",
+    "private::test.integration-dash": "TESTFILES=$(circleci tests glob \"tests/integration/**/test_*.py\" | circleci tests split --split-by=timings) && pytest --headless --nopercyfinalize --junitxml=test-reports/junit_intg.xml ${TESTFILES}",
     "private::test.integration-dash-import": "cd tests/integration/dash && python dash_import_test.py",
     "format": "run-s private::format.*",
     "initialize": "run-s private::initialize.*",

From b92f89fa22a083e3fc827954dd6b037d65420575 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Mon, 16 Aug 2021 07:44:19 -0400
Subject: [PATCH 20/32] Support single list input argument

---
 dash/dash.py                                     |  2 +-
 dash/long_callback/managers/__init__.py          |  2 +-
 dash/long_callback/managers/celery_manager.py    | 10 +++++-----
 dash/long_callback/managers/diskcache_manager.py | 10 +++++-----
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/dash/dash.py b/dash/dash.py
index 612def5a93..b2f2e305d3 100644
--- a/dash/dash.py
+++ b/dash/dash.py
@@ -1250,7 +1250,7 @@ def long_callback(self, *_args, **_kwargs):
         )
 
         def wrapper(fn):
-            background_fn = callback_manager.make_job_fn(fn, progress=bool(progress))
+            background_fn = callback_manager.make_job_fn(fn, bool(progress), args_deps)
 
             def callback(_triggers, user_store_data, user_callback_args):
                 # Build result cache key from inputs
diff --git a/dash/long_callback/managers/__init__.py b/dash/long_callback/managers/__init__.py
index 70a876e139..b6b05e89a8 100644
--- a/dash/long_callback/managers/__init__.py
+++ b/dash/long_callback/managers/__init__.py
@@ -19,7 +19,7 @@ def terminate_unhealthy_job(self, job):
     def job_running(self, job):
         raise NotImplementedError
 
-    def make_job_fn(self, fn, progress):
+    def make_job_fn(self, fn, progress, args_deps):
         raise NotImplementedError
 
     def call_job_fn(self, key, job_fn, args):
diff --git a/dash/long_callback/managers/celery_manager.py b/dash/long_callback/managers/celery_manager.py
index 553dc5e7c2..f788f23df3 100644
--- a/dash/long_callback/managers/celery_manager.py
+++ b/dash/long_callback/managers/celery_manager.py
@@ -39,8 +39,8 @@ def job_running(self, job):
             "PROGRESS",
         )
 
-    def make_job_fn(self, fn, progress=False):
-        return _make_job_fn(fn, self.handle, progress)
+    def make_job_fn(self, fn, progress, args_deps):
+        return _make_job_fn(fn, self.handle, progress, args_deps)
 
     def get_task(self, job):
         if job:
@@ -87,7 +87,7 @@ def get_result(self, key, job):
         return result
 
 
-def _make_job_fn(fn, celery_app, progress):
+def _make_job_fn(fn, celery_app, progress, args_deps):
     cache = celery_app.backend
 
     # Hash function source and module to create a unique (but stable) celery task name
@@ -102,9 +102,9 @@ def _set_progress(progress_value):
             cache.set(progress_key, json.dumps(progress_value, cls=PlotlyJSONEncoder))
 
         maybe_progress = [_set_progress] if progress else []
-        if isinstance(user_callback_args, dict):
+        if isinstance(args_deps, dict):
             user_callback_output = fn(*maybe_progress, **user_callback_args)
-        elif isinstance(user_callback_args, list):
+        elif isinstance(args_deps, (list, tuple)):
             user_callback_output = fn(*maybe_progress, *user_callback_args)
         else:
             user_callback_output = fn(*maybe_progress, user_callback_args)
diff --git a/dash/long_callback/managers/diskcache_manager.py b/dash/long_callback/managers/diskcache_manager.py
index e41fc71dfb..d1a8d957bb 100644
--- a/dash/long_callback/managers/diskcache_manager.py
+++ b/dash/long_callback/managers/diskcache_manager.py
@@ -65,8 +65,8 @@ def job_running(self, job):
             return proc.status() != psutil.STATUS_ZOMBIE
         return False
 
-    def make_job_fn(self, fn, progress=False):
-        return _make_job_fn(fn, self.handle, progress)
+    def make_job_fn(self, fn, progress, args_deps):
+        return _make_job_fn(fn, self.handle, progress, args_deps)
 
     def clear_cache_entry(self, key):
         self.handle.delete(key)
@@ -106,15 +106,15 @@ def get_result(self, key, job):
         return result
 
 
-def _make_job_fn(fn, cache, progress):
+def _make_job_fn(fn, cache, progress, args_deps):
     def job_fn(result_key, progress_key, user_callback_args):
         def _set_progress(progress_value):
             cache.set(progress_key, progress_value)
 
         maybe_progress = [_set_progress] if progress else []
-        if isinstance(user_callback_args, dict):
+        if isinstance(args_deps, dict):
             user_callback_output = fn(*maybe_progress, **user_callback_args)
-        elif isinstance(user_callback_args, (list, tuple)):
+        elif isinstance(args_deps, (list, tuple)):
             user_callback_output = fn(*maybe_progress, *user_callback_args)
         else:
             user_callback_output = fn(*maybe_progress, user_callback_args)

From e2bd875331e45049e366c64432ca9e60be30722b Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Mon, 16 Aug 2021 07:46:15 -0400
Subject: [PATCH 21/32] Raise informative error when dependency to
 long_callback has pattern-matching id

---
 dash/dash.py                                | 14 +++++++
 dash/dependencies.py                        | 10 +++++
 dash/exceptions.py                          |  4 ++
 tests/unit/dash/long_callback_validation.py | 46 +++++++++++++++++++++
 4 files changed, 74 insertions(+)
 create mode 100644 tests/unit/dash/long_callback_validation.py

diff --git a/dash/dash.py b/dash/dash.py
index b2f2e305d3..8395db627c 100644
--- a/dash/dash.py
+++ b/dash/dash.py
@@ -1199,6 +1199,9 @@ def long_callback(self, *_args, **_kwargs):
             callback_context,
         )
         import dash_core_components as dcc  # pylint: disable=import-outside-toplevel
+        from dash.exceptions import (  # pylint: disable=import-outside-toplevel
+            WildcardInLongCallback,
+        )
 
         # Get long callback manager
         callback_manager = _kwargs.pop("manager", self._long_callback_manager)
@@ -1230,6 +1233,17 @@ def long_callback(self, *_args, **_kwargs):
         inputs_and_state = flat_inputs + flat_state
         args_deps = map_grouping(lambda i: inputs_and_state[i], inputs_state_indices)
 
+        # Disallow wildcard dependencies
+        for deps in [output, flat_inputs, flat_state]:
+            for dep in flatten_grouping(deps):
+                if dep.has_wildcard():
+                    raise WildcardInLongCallback(
+                        f"""
+                        @app.long_callback does not support dependencies with
+                        pattern-matching ids
+                            Received: {repr(dep)}\n"""
+                    )
+
         # Get unique id for this long_callback definition.  This increment is not
         # thread safe, but it doesn't need to be because callback definitions
         # happen on the main thread before the app starts
diff --git a/dash/dependencies.py b/dash/dependencies.py
index 8d3daa84d3..1b2cc6f7f9 100644
--- a/dash/dependencies.py
+++ b/dash/dependencies.py
@@ -101,6 +101,16 @@ def _id_matches(self, other):
     def __hash__(self):
         return hash(str(self))
 
+    def has_wildcard(self):
+        """
+        Return true if id contains a wildcard (MATCH, ALL, or ALLSMALLER)
+        """
+        if isinstance(self.component_id, dict):
+            for v in self.component_id.values():
+                if isinstance(v, _Wildcard):
+                    return True
+        return False
+
 
 class Output(DashDependency):  # pylint: disable=too-few-public-methods
     """Output of a callback."""
diff --git a/dash/exceptions.py b/dash/exceptions.py
index 8a08df4010..0d8b094408 100644
--- a/dash/exceptions.py
+++ b/dash/exceptions.py
@@ -30,6 +30,10 @@ class IDsCantContainPeriods(CallbackException):
     pass
 
 
+class WildcardInLongCallback(CallbackException):
+    pass
+
+
 # Better error name now that more than periods are not permitted.
 class InvalidComponentIdError(IDsCantContainPeriods):
     pass
diff --git a/tests/unit/dash/long_callback_validation.py b/tests/unit/dash/long_callback_validation.py
new file mode 100644
index 0000000000..7d4542bedc
--- /dev/null
+++ b/tests/unit/dash/long_callback_validation.py
@@ -0,0 +1,46 @@
+import pytest
+import mock
+
+import dash
+from dash.exceptions import WildcardInLongCallback
+from dash.dependencies import Input, Output, State, ALL, MATCH, ALLSMALLER
+
+
+def test_wildcard_ids_no_allowed_in_long_callback():
+    """
+    @app.long_callback doesn't support wildcard dependencies yet. This test can
+    be removed if wildcard support is added to @app.long_callback in the future.
+    """
+    app = dash.Dash(long_callback_manager=mock.Mock())
+
+    # ALL
+    with pytest.raises(WildcardInLongCallback):
+
+        @app.long_callback(
+            Output("output", "children"),
+            Input({"type": "filter", "index": ALL}, "value"),
+        )
+        def callback(*args, **kwargs):
+            pass
+
+    # MATCH
+    with pytest.raises(WildcardInLongCallback):
+
+        @app.long_callback(
+            Output({"type": "dynamic-output", "index": MATCH}, "children"),
+            Input({"type": "dynamic-dropdown", "index": MATCH}, "value"),
+            State({"type": "dynamic-dropdown", "index": MATCH}, "id"),
+        )
+        def callback(*args, **kwargs):
+            pass
+
+    # ALLSMALLER
+    with pytest.raises(WildcardInLongCallback):
+
+        @app.long_callback(
+            Output({"type": "output-ex3", "index": MATCH}, "children"),
+            Input({"type": "filter-dropdown-ex3", "index": MATCH}, "value"),
+            Input({"type": "filter-dropdown-ex3", "index": ALLSMALLER}, "value"),
+        )
+        def callback(*args, **kwargs):
+            pass

From 71227cd93e6c6403e6a76fe7343ae23bd6bbf0cf Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Mon, 16 Aug 2021 10:45:39 -0400
Subject: [PATCH 22/32] Remove module string from celery task name hash

This can vary between how celery and dash are launched.
---
 dash/long_callback/managers/celery_manager.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/dash/long_callback/managers/celery_manager.py b/dash/long_callback/managers/celery_manager.py
index f788f23df3..0952182666 100644
--- a/dash/long_callback/managers/celery_manager.py
+++ b/dash/long_callback/managers/celery_manager.py
@@ -92,11 +92,10 @@ def _make_job_fn(fn, celery_app, progress, args_deps):
 
     # Hash function source and module to create a unique (but stable) celery task name
     fn_source = inspect.getsource(fn)
-    fn_module = fn.__module__
-    fn_str = fn_module + "\n" + fn_source
+    fn_str = fn_source
     fn_hash = hashlib.sha1(fn_str.encode("utf-8")).hexdigest()
 
-    @celery_app.task(name=fn_hash)
+    @celery_app.task(name=f"long_callback_{fn_hash}")
     def job_fn(result_key, progress_key, user_callback_args, fn=fn):
         def _set_progress(progress_value):
             cache.set(progress_key, json.dumps(progress_value, cls=PlotlyJSONEncoder))

From 41520e576dff0454882fee8cbca9939146423a9c Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Mon, 16 Aug 2021 10:46:54 -0400
Subject: [PATCH 23/32] validate that celery app has result backend configured

---
 dash/long_callback/managers/celery_manager.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/dash/long_callback/managers/celery_manager.py b/dash/long_callback/managers/celery_manager.py
index 0952182666..62aa2921e2 100644
--- a/dash/long_callback/managers/celery_manager.py
+++ b/dash/long_callback/managers/celery_manager.py
@@ -9,10 +9,16 @@
 class CeleryLongCallbackManager(BaseLongCallbackManager):
     def __init__(self, celery_app, cache_by=None, expire=None):
         import celery  # pylint: disable=import-outside-toplevel
+        from celery.backends.base import (  # pylint: disable=import-outside-toplevel
+            DisabledBackend,
+        )
 
         if not isinstance(celery_app, celery.Celery):
             raise ValueError("First argument must be a celery.Celery object")
 
+        if isinstance(celery_app.backend, DisabledBackend):
+            raise ValueError("Celery instance must be configured with a result backend")
+
         super().__init__(cache_by)
         self.handle = celery_app
         self.expire = expire

From e5f967e80bc838ea27f8f8f40a4fe206280de001 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Mon, 16 Aug 2021 10:48:00 -0400
Subject: [PATCH 24/32] Test celery manager with multiple celery workers

---
 tests/integration/long_callback/test_basic_long_callback.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/long_callback/test_basic_long_callback.py b/tests/integration/long_callback/test_basic_long_callback.py
index 8952bc3ce4..bcfdaa1958 100644
--- a/tests/integration/long_callback/test_basic_long_callback.py
+++ b/tests/integration/long_callback/test_basic_long_callback.py
@@ -54,7 +54,7 @@ def setup_long_callback_app(manager_name, app_name):
                 "-P",
                 "prefork",
                 "--concurrency",
-                "1",
+                "2",
                 "--loglevel=info",
             ],
             preexec_fn=os.setpgrp,

From 2405f11c916f67f24cec6cc7fbc37be58c29c11c Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Mon, 16 Aug 2021 10:49:23 -0400
Subject: [PATCH 25/32] Add long callback manager docstrings

---
 dash/long_callback/managers/celery_manager.py | 17 ++++++++++++++++
 .../managers/diskcache_manager.py             | 20 +++++++++++++++++--
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/dash/long_callback/managers/celery_manager.py b/dash/long_callback/managers/celery_manager.py
index 62aa2921e2..4d6c999049 100644
--- a/dash/long_callback/managers/celery_manager.py
+++ b/dash/long_callback/managers/celery_manager.py
@@ -8,6 +8,23 @@
 
 class CeleryLongCallbackManager(BaseLongCallbackManager):
     def __init__(self, celery_app, cache_by=None, expire=None):
+        """
+        Long callback manager that runs callback logic on a celery task queue,
+        and stores results using a celery result backend.
+
+        :param celery_app:
+            A celery.Celery application instance that must be configured with a
+            result backend. See the celery documentation for information on
+            configuration options.
+        :param cache_by:
+            A list of zero-argument functions.  When provided, caching is enabled and
+            the return values of these functions are combined with the callback
+            function's input arguments and source code to generate cache keys.
+        :param expire:
+            If provided, a cache entry will be removed when it has not been accessed
+            for ``expire`` seconds.  If not provided, the lifetime of cache entries
+            is determined by the default behavior of the celery result backend.
+        """
         import celery  # pylint: disable=import-outside-toplevel
         from celery.backends.base import (  # pylint: disable=import-outside-toplevel
             DisabledBackend,
diff --git a/dash/long_callback/managers/diskcache_manager.py b/dash/long_callback/managers/diskcache_manager.py
index d1a8d957bb..ed23240881 100644
--- a/dash/long_callback/managers/diskcache_manager.py
+++ b/dash/long_callback/managers/diskcache_manager.py
@@ -5,6 +5,22 @@
 
 class DiskcacheLongCallbackManager(BaseLongCallbackManager):
     def __init__(self, cache, cache_by=None, expire=None):
+        """
+        Long callback manager that runs callback logic in a subprocess and stores
+        results on disk using diskcache
+
+        :param cache:
+            A diskcache.Cache or diskcache.FanoutCache instance. See the diskcache
+            documentation for information on configuration options.
+        :param cache_by:
+            A list of zero-argument functions.  When provided, caching is enabled and
+            the return values of these functions are combined with the callback
+            function's input arguments and source code to generate cache keys.
+        :param expire:
+            If provided, a cache entry will be removed when it has not been accessed
+            for ``expire`` seconds.  If not provided, the lifetime of cache entries
+            is determined by the default behavior of the ``cache`` instance.
+        """
         try:
             import diskcache  # pylint: disable=import-outside-toplevel
             import psutil  # noqa: F401,E402 pylint: disable=import-outside-toplevel,unused-import,unused-variable
@@ -15,14 +31,14 @@ def __init__(self, cache, cache_by=None, expire=None):
 DiskcacheLongCallbackManager requires the multiprocess, diskcache, and psutil packages
 which can be installed using pip...
 
-    $ pip install multiprocess diskcache
+    $ pip install multiprocess diskcache psutil
 
 or conda.
 
     $ conda install -c conda-forge multiprocess diskcache psutil\n"""
             )
 
-        if not isinstance(cache, diskcache.Cache):
+        if not isinstance(cache, (diskcache.Cache, diskcache.FanoutCache)):
             raise ValueError("First argument must be a diskcache.Cache object")
         super().__init__(cache_by)
         self.handle = cache

From 7e0f3865f7b871975b61823760d89aa83b3577ed Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Mon, 16 Aug 2021 11:22:45 -0400
Subject: [PATCH 26/32] bump up test wait times

---
 .../long_callback/test_basic_long_callback.py | 84 +++++++++----------
 1 file changed, 42 insertions(+), 42 deletions(-)

diff --git a/tests/integration/long_callback/test_basic_long_callback.py b/tests/integration/long_callback/test_basic_long_callback.py
index bcfdaa1958..2708fd87e0 100644
--- a/tests/integration/long_callback/test_basic_long_callback.py
+++ b/tests/integration/long_callback/test_basic_long_callback.py
@@ -101,7 +101,7 @@ def test_lcbc001_fast_input(dash_duo, manager):
             with lock:
                 input_.send_keys(key)
 
-        dash_duo.wait_for_text_to_equal("#output-1", "hello world", 4)
+        dash_duo.wait_for_text_to_equal("#output-1", "hello world", 8)
 
     assert not dash_duo.redux_state_is_loading
     assert dash_duo.get_logs() == []
@@ -111,24 +111,24 @@ def test_lcbc002_long_callback_running(dash_duo, manager):
     with setup_long_callback_app(manager, "app2") as app:
         dash_duo.start_server(app)
         dash_duo.wait_for_text_to_equal("#result", "Clicked 0 time(s)", 15)
-        dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
 
         # Click button and check that status has changed to "Running"
         dash_duo.find_element("#button-1").click()
-        dash_duo.wait_for_text_to_equal("#status", "Running", 4)
+        dash_duo.wait_for_text_to_equal("#status", "Running", 8)
 
         # Wait for calculation to finish, then check that status is "Finished"
-        dash_duo.wait_for_text_to_equal("#result", "Clicked 1 time(s)", 6)
-        dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+        dash_duo.wait_for_text_to_equal("#result", "Clicked 1 time(s)", 12)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
 
         # Click button twice and check that status has changed to "Running"
         dash_duo.find_element("#button-1").click()
         dash_duo.find_element("#button-1").click()
-        dash_duo.wait_for_text_to_equal("#status", "Running", 4)
+        dash_duo.wait_for_text_to_equal("#status", "Running", 8)
 
         # Wait for calculation to finish, then check that status is "Finished"
-        dash_duo.wait_for_text_to_equal("#result", "Clicked 3 time(s)", 10)
-        dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+        dash_duo.wait_for_text_to_equal("#result", "Clicked 3 time(s)", 12)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
 
     assert not dash_duo.redux_state_is_loading
     assert dash_duo.get_logs() == []
@@ -152,19 +152,19 @@ def test_lcbc003_long_callback_running_cancel(dash_duo, manager):
 
         # Click run button and check that status has changed to "Running"
         dash_duo.find_element("#run-button").click()
-        dash_duo.wait_for_text_to_equal("#status", "Running", 4)
+        dash_duo.wait_for_text_to_equal("#status", "Running", 8)
 
         # Then click Cancel button and make sure that the status changes to finish
         # without update result
         dash_duo.find_element("#cancel-button").click()
-        dash_duo.wait_for_text_to_equal("#result", "Processed 'initial value'", 8)
-        dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+        dash_duo.wait_for_text_to_equal("#result", "Processed 'initial value'", 12)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
 
         # Click run button again, and let it finish
         dash_duo.find_element("#run-button").click()
-        dash_duo.wait_for_text_to_equal("#status", "Running", 4)
-        dash_duo.wait_for_text_to_equal("#result", "Processed 'hello world'", 4)
-        dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
+        dash_duo.wait_for_text_to_equal("#status", "Running", 8)
+        dash_duo.wait_for_text_to_equal("#result", "Processed 'hello world'", 8)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
 
     assert not dash_duo.redux_state_is_loading
     assert dash_duo.get_logs() == []
@@ -187,14 +187,14 @@ def test_lcbc004_long_callback_progress(dash_duo, manager):
         dash_duo.find_element("#run-button").click()
         dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 15)
         dash_duo.wait_for_text_to_equal("#status", "Finished", 15)
-        dash_duo.wait_for_text_to_equal("#result", "Processed 'hello, world'", 4)
+        dash_duo.wait_for_text_to_equal("#result", "Processed 'hello, world'", 8)
 
         # Click run button again with same input.
         # without caching, this should rerun callback and display progress
         dash_duo.find_element("#run-button").click()
         dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 15)
         dash_duo.wait_for_text_to_equal("#status", "Finished", 15)
-        dash_duo.wait_for_text_to_equal("#result", "Processed 'hello, world'", 4)
+        dash_duo.wait_for_text_to_equal("#result", "Processed 'hello, world'", 8)
 
     assert not dash_duo.redux_state_is_loading
     assert dash_duo.get_logs() == []
@@ -207,7 +207,7 @@ def test_lcbc005_long_callback_caching(dash_duo, manager):
         dash_duo.start_server(app)
         dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 15)
         dash_duo.wait_for_text_to_equal("#status", "Finished", 15)
-        dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 4)
+        dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 8)
 
         # Update input text box to BBB
         input_ = dash_duo.find_element("#input")
@@ -219,8 +219,8 @@ def test_lcbc005_long_callback_caching(dash_duo, manager):
         # Click run button and check that status eventually cycles to 2/4
         dash_duo.find_element("#run-button").click()
         dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 20)
-        dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
-        dash_duo.wait_for_text_to_equal("#result", "Result for 'BBB'", 4)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 12)
+        dash_duo.wait_for_text_to_equal("#result", "Result for 'BBB'", 8)
 
         # Update input text box back to AAA
         input_ = dash_duo.find_element("#input")
@@ -232,8 +232,8 @@ def test_lcbc005_long_callback_caching(dash_duo, manager):
         # Click run button and this time the cached result is used,
         # So we can get the result right away
         dash_duo.find_element("#run-button").click()
-        dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
-        dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 4)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
+        dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 8)
 
         # Update input text box back to BBB
         input_ = dash_duo.find_element("#input")
@@ -245,8 +245,8 @@ def test_lcbc005_long_callback_caching(dash_duo, manager):
         # Click run button and this time the cached result is used,
         # So we can get the result right away
         dash_duo.find_element("#run-button").click()
-        dash_duo.wait_for_text_to_equal("#status", "Finished", 4)
-        dash_duo.wait_for_text_to_equal("#result", "Result for 'BBB'", 4)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
+        dash_duo.wait_for_text_to_equal("#result", "Result for 'BBB'", 8)
 
         # Update input text box back to AAA
         input_ = dash_duo.find_element("#input")
@@ -260,8 +260,8 @@ def test_lcbc005_long_callback_caching(dash_duo, manager):
 
         dash_duo.find_element("#run-button").click()
         dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 20)
-        dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
-        dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 4)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 12)
+        dash_duo.wait_for_text_to_equal("#result", "Result for 'AAA'", 8)
 
         assert not dash_duo.redux_state_is_loading
         assert dash_duo.get_logs() == []
@@ -274,11 +274,11 @@ def test_lcbc006_long_callback_caching_multi(dash_duo, manager):
         dash_duo.start_server(app)
         dash_duo.wait_for_text_to_equal("#status1", "Progress 2/4", 15)
         dash_duo.wait_for_text_to_equal("#status1", "Finished", 15)
-        dash_duo.wait_for_text_to_equal("#result1", "Result for 'AAA'", 4)
+        dash_duo.wait_for_text_to_equal("#result1", "Result for 'AAA'", 8)
 
         # Check initial status/output of second long_callback
         dash_duo.wait_for_text_to_equal("#status2", "Finished", 15)
-        dash_duo.wait_for_text_to_equal("#result2", "Result for 'aaa'", 4)
+        dash_duo.wait_for_text_to_equal("#result2", "Result for 'aaa'", 8)
 
         # Update input text box to BBB
         input_ = dash_duo.find_element("#input1")
@@ -290,12 +290,12 @@ def test_lcbc006_long_callback_caching_multi(dash_duo, manager):
         # Click run button and check that status eventually cycles to 2/4
         dash_duo.find_element("#run-button1").click()
         dash_duo.wait_for_text_to_equal("#status1", "Progress 2/4", 20)
-        dash_duo.wait_for_text_to_equal("#status1", "Finished", 8)
-        dash_duo.wait_for_text_to_equal("#result1", "Result for 'BBB'", 4)
+        dash_duo.wait_for_text_to_equal("#status1", "Finished", 12)
+        dash_duo.wait_for_text_to_equal("#result1", "Result for 'BBB'", 8)
 
         # Check there were no changes in second long_callback output
         dash_duo.wait_for_text_to_equal("#status2", "Finished", 15)
-        dash_duo.wait_for_text_to_equal("#result2", "Result for 'aaa'", 4)
+        dash_duo.wait_for_text_to_equal("#result2", "Result for 'aaa'", 8)
 
         # Update input text box back to AAA
         input_ = dash_duo.find_element("#input1")
@@ -307,8 +307,8 @@ def test_lcbc006_long_callback_caching_multi(dash_duo, manager):
         # Click run button and this time the cached result is used,
         # So we can get the result right away
         dash_duo.find_element("#run-button1").click()
-        dash_duo.wait_for_text_to_equal("#status1", "Finished", 4)
-        dash_duo.wait_for_text_to_equal("#result1", "Result for 'AAA'", 4)
+        dash_duo.wait_for_text_to_equal("#status1", "Finished", 8)
+        dash_duo.wait_for_text_to_equal("#result1", "Result for 'AAA'", 8)
 
         # Update input text box back to BBB
         input_ = dash_duo.find_element("#input1")
@@ -320,8 +320,8 @@ def test_lcbc006_long_callback_caching_multi(dash_duo, manager):
         # Click run button and this time the cached result is used,
         # So we can get the result right away
         dash_duo.find_element("#run-button1").click()
-        dash_duo.wait_for_text_to_equal("#status1", "Finished", 4)
-        dash_duo.wait_for_text_to_equal("#result1", "Result for 'BBB'", 4)
+        dash_duo.wait_for_text_to_equal("#status1", "Finished", 8)
+        dash_duo.wait_for_text_to_equal("#result1", "Result for 'BBB'", 8)
 
         # Update second input text box to BBB, make sure there is not a cache hit
         input_ = dash_duo.find_element("#input2")
@@ -331,8 +331,8 @@ def test_lcbc006_long_callback_caching_multi(dash_duo, manager):
                 input_.send_keys(key)
         dash_duo.find_element("#run-button2").click()
         dash_duo.wait_for_text_to_equal("#status2", "Progress 2/4", 20)
-        dash_duo.wait_for_text_to_equal("#status2", "Finished", 8)
-        dash_duo.wait_for_text_to_equal("#result2", "Result for 'BBB'", 4)
+        dash_duo.wait_for_text_to_equal("#status2", "Finished", 12)
+        dash_duo.wait_for_text_to_equal("#result2", "Result for 'BBB'", 8)
 
         # Update second input text box back to aaa, check for cache hit
         input_ = dash_duo.find_element("#input2")
@@ -341,8 +341,8 @@ def test_lcbc006_long_callback_caching_multi(dash_duo, manager):
             with lock:
                 input_.send_keys(key)
         dash_duo.find_element("#run-button2").click()
-        dash_duo.wait_for_text_to_equal("#status2", "Finished", 4)
-        dash_duo.wait_for_text_to_equal("#result2", "Result for 'aaa'", 4)
+        dash_duo.wait_for_text_to_equal("#status2", "Finished", 12)
+        dash_duo.wait_for_text_to_equal("#result2", "Result for 'aaa'", 8)
 
         # Update input text box back to AAA
         input_ = dash_duo.find_element("#input1")
@@ -357,14 +357,14 @@ def test_lcbc006_long_callback_caching_multi(dash_duo, manager):
         # Check for cache miss for first long_callback
         dash_duo.find_element("#run-button1").click()
         dash_duo.wait_for_text_to_equal("#status1", "Progress 2/4", 20)
-        dash_duo.wait_for_text_to_equal("#status1", "Finished", 8)
-        dash_duo.wait_for_text_to_equal("#result1", "Result for 'AAA'", 4)
+        dash_duo.wait_for_text_to_equal("#status1", "Finished", 12)
+        dash_duo.wait_for_text_to_equal("#result1", "Result for 'AAA'", 8)
 
         # Check for cache miss for second long_callback
         dash_duo.find_element("#run-button2").click()
         dash_duo.wait_for_text_to_equal("#status2", "Progress 2/4", 20)
-        dash_duo.wait_for_text_to_equal("#status2", "Finished", 8)
-        dash_duo.wait_for_text_to_equal("#result2", "Result for 'aaa'", 4)
+        dash_duo.wait_for_text_to_equal("#status2", "Finished", 12)
+        dash_duo.wait_for_text_to_equal("#result2", "Result for 'aaa'", 8)
 
         assert not dash_duo.redux_state_is_loading
         assert dash_duo.get_logs() == []

From 4b7ec3c6ced1629288c63c274ef3d2525cf20659 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Mon, 16 Aug 2021 13:29:49 -0400
Subject: [PATCH 27/32] Don't fail on NoSuchProcess exception

---
 dash/long_callback/managers/diskcache_manager.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/dash/long_callback/managers/diskcache_manager.py b/dash/long_callback/managers/diskcache_manager.py
index ed23240881..00a4fe3178 100644
--- a/dash/long_callback/managers/diskcache_manager.py
+++ b/dash/long_callback/managers/diskcache_manager.py
@@ -55,9 +55,18 @@ def terminate_job(self, job):
         with self.handle.transact():
             if psutil.pid_exists(job):
                 process = psutil.Process(job)
+
                 for proc in process.children(recursive=True):
-                    proc.kill()
-                process.kill()
+                    try:
+                        proc.kill()
+                    except psutil.NoSuchProcess:
+                        pass
+
+                try:
+                    process.kill()
+                except psutil.NoSuchProcess:
+                    pass
+
                 try:
                     process.wait(0.5)
                 except psutil.TimeoutExpired:

From 5731edf033012bb82b9eec0b31910066581734f8 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Mon, 16 Aug 2021 13:37:24 -0400
Subject: [PATCH 28/32] Don't fail on NoSuchProcess exception (2)

---
 dash/long_callback/managers/diskcache_manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dash/long_callback/managers/diskcache_manager.py b/dash/long_callback/managers/diskcache_manager.py
index 00a4fe3178..5363929c99 100644
--- a/dash/long_callback/managers/diskcache_manager.py
+++ b/dash/long_callback/managers/diskcache_manager.py
@@ -69,7 +69,7 @@ def terminate_job(self, job):
 
                 try:
                     process.wait(0.5)
-                except psutil.TimeoutExpired:
+                except (psutil.TimeoutExpired, psutil.NoSuchProcess):
                     pass
 
     def terminate_unhealthy_job(self, job):

From 56ad571adc82e68360cf08c37075e5904e1f697c Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Mon, 16 Aug 2021 16:12:52 -0400
Subject: [PATCH 29/32] Add CHANGELOG entry

---
 CHANGELOG.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 97f1c471a6..ba0c8c2410 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,9 @@ This project adheres to [Semantic Versioning](https://semver.org/).
 
 ## Dash and Dash Renderer
 
+### Added
+ - [#1702](https://github.com/plotly/dash/pull/1702) Added a new `@app.long_callback` decorator to support callback functions that take a long time to run. See the PR and documentation for more information.
+
 ### Changed
 - [#1707](https://github.com/plotly/dash/pull/1707) Change the default value of the `compress` argument to the `dash.Dash` constructor to `False`. This change reduces CPU usage, and was made in recognition of the fact that many deployment platforms (e.g. Dash Enterprise) already apply their own compression. If deploying to an environment that does not already provide compression, the Dash 1 behavior may be restored by adding `compress=True` to the `dash.Dash` constructor. 
 

From 9134207ec700c2ffa5290883e8dff484d36250b7 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Tue, 17 Aug 2021 07:55:50 -0400
Subject: [PATCH 30/32] Add extra components to validation_layout and fix
 prevent_initial_call

Add test where the dependencies of long_callback are added dynamically, and validation is
handled using validation_layout and prevent_initial_call=True
---
 dash/dash.py                                  | 16 ++++-
 tests/integration/long_callback/app2.py       |  1 +
 tests/integration/long_callback/app3.py       |  1 +
 tests/integration/long_callback/app4.py       |  1 +
 tests/integration/long_callback/app6.py       |  1 +
 tests/integration/long_callback/app7.py       | 69 +++++++++++++++++++
 .../long_callback/test_basic_long_callback.py | 56 ++++++++++++++-
 7 files changed, 140 insertions(+), 5 deletions(-)
 create mode 100644 tests/integration/long_callback/app7.py

diff --git a/dash/dash.py b/dash/dash.py
index 25d4bd7bc1..85a2aa6e7a 100644
--- a/dash/dash.py
+++ b/dash/dash.py
@@ -571,6 +571,8 @@ def serve_layout(self):
         )
 
     def _config(self):
+        from dash_html_components import Div  # pylint: disable=import-outside-toplevel
+
         # pieces of config needed by the front end
         config = {
             "url_base_pathname": self.config.url_base_pathname,
@@ -588,7 +590,15 @@ def _config(self):
                 "max_retry": self._dev_tools.hot_reload_max_retry,
             }
         if self.validation_layout and not self.config.suppress_callback_exceptions:
-            config["validation_layout"] = self.validation_layout
+            validation_layout = self.validation_layout
+
+            # Add extra components
+            if self._extra_components:
+                validation_layout = Div(
+                    children=[validation_layout] + self._extra_components
+                )
+
+            config["validation_layout"] = validation_layout
 
         return config
 
@@ -1251,7 +1261,9 @@ def long_callback(self, *_args, **_kwargs):
         # Create Interval and Store for long callback and add them to the app's
         # _extra_components list
         interval_id = f"_long_callback_interval_{long_callback_id}"
-        interval_component = dcc.Interval(id=interval_id, interval=interval_time)
+        interval_component = dcc.Interval(
+            id=interval_id, interval=interval_time, disabled=prevent_initial_call
+        )
         store_id = f"_long_callback_store_{long_callback_id}"
         store_component = dcc.Store(id=store_id, data=dict())
         self._extra_components.extend([interval_component, store_component])
diff --git a/tests/integration/long_callback/app2.py b/tests/integration/long_callback/app2.py
index 206b502723..1f2066339b 100644
--- a/tests/integration/long_callback/app2.py
+++ b/tests/integration/long_callback/app2.py
@@ -23,6 +23,7 @@
     [Input("button-1", "n_clicks")],
     running=[(Output("status", "children"), "Running", "Finished")],
     interval=500,
+    prevent_initial_call=True,
 )
 def update_output(n_clicks):
     time.sleep(2)
diff --git a/tests/integration/long_callback/app3.py b/tests/integration/long_callback/app3.py
index 0b6681173f..2b376406de 100644
--- a/tests/integration/long_callback/app3.py
+++ b/tests/integration/long_callback/app3.py
@@ -28,6 +28,7 @@
     cancel=[Input("cancel-button", "n_clicks")],
     interval=500,
     manager=long_callback_manager,
+    prevent_initial_call=True,
 )
 def update_output(n_clicks, value):
     time.sleep(2)
diff --git a/tests/integration/long_callback/app4.py b/tests/integration/long_callback/app4.py
index 3f63f88bcb..41034fe29b 100644
--- a/tests/integration/long_callback/app4.py
+++ b/tests/integration/long_callback/app4.py
@@ -29,6 +29,7 @@
     progress_default="Finished",
     cancel=[Input("cancel-button", "n_clicks")],
     interval=500,
+    prevent_initial_call=True,
 )
 def update_output(set_progress, n_clicks, value):
     for i in range(4):
diff --git a/tests/integration/long_callback/app6.py b/tests/integration/long_callback/app6.py
index 06b4461446..da654d5959 100644
--- a/tests/integration/long_callback/app6.py
+++ b/tests/integration/long_callback/app6.py
@@ -59,6 +59,7 @@ def update_output1(set_progress, _n_clicks, value):
     progress_default="Finished",
     interval=500,
     cache_args_to_ignore=["button"],
+    prevent_initial_call=True,
 )
 def update_output2(set_progress, button, value):
     for i in range(4):
diff --git a/tests/integration/long_callback/app7.py b/tests/integration/long_callback/app7.py
new file mode 100644
index 0000000000..f34f5b8c9b
--- /dev/null
+++ b/tests/integration/long_callback/app7.py
@@ -0,0 +1,69 @@
+import dash
+from dash.dependencies import Input, State, Output
+import dash_html_components as html
+import dash_core_components as dcc
+import time
+
+from tests.integration.long_callback.utils import get_long_callback_manager
+
+long_callback_manager = get_long_callback_manager()
+handle = long_callback_manager.handle
+
+
+app = dash.Dash(__name__, long_callback_manager=long_callback_manager)
+app.layout = html.Div(
+    [
+        html.Button(id="show-layout-button", children="Show"),
+        html.Div(id="dynamic-layout"),
+    ]
+)
+
+app.validation_layout = html.Div(
+    [
+        html.Button(id="show-layout-button", children="Show"),
+        html.Div(id="dynamic-layout"),
+        dcc.Input(id="input", value="hello, world"),
+        html.Button(id="run-button", children="Run"),
+        html.Button(id="cancel-button", children="Cancel"),
+        html.Div(id="status", children="Finished"),
+        html.Div(id="result", children="No results"),
+    ]
+)
+
+
+@app.callback(
+    Output("dynamic-layout", "children"), Input("show-layout-button", "n_clicks")
+)
+def make_layout(n_clicks):
+    if n_clicks is not None:
+        return html.Div(
+            [
+                dcc.Input(id="input", value="hello, world"),
+                html.Button(id="run-button", children="Run"),
+                html.Button(id="cancel-button", children="Cancel"),
+                html.Div(id="status", children="Finished"),
+                html.Div(id="result", children="No results"),
+            ]
+        )
+    else:
+        return []
+
+
+@app.long_callback(
+    Output("result", "children"),
+    [Input("run-button", "n_clicks"), State("input", "value")],
+    progress=Output("status", "children"),
+    progress_default="Finished",
+    cancel=[Input("cancel-button", "n_clicks")],
+    interval=500,
+    prevent_initial_call=True,
+)
+def update_output(set_progress, n_clicks, value):
+    for i in range(4):
+        set_progress(f"Progress {i}/4")
+        time.sleep(1)
+    return f"Processed '{value}'"
+
+
+if __name__ == "__main__":
+    app.run_server(debug=True)
diff --git a/tests/integration/long_callback/test_basic_long_callback.py b/tests/integration/long_callback/test_basic_long_callback.py
index 2708fd87e0..1b492cfc94 100644
--- a/tests/integration/long_callback/test_basic_long_callback.py
+++ b/tests/integration/long_callback/test_basic_long_callback.py
@@ -110,7 +110,7 @@ def test_lcbc001_fast_input(dash_duo, manager):
 def test_lcbc002_long_callback_running(dash_duo, manager):
     with setup_long_callback_app(manager, "app2") as app:
         dash_duo.start_server(app)
-        dash_duo.wait_for_text_to_equal("#result", "Clicked 0 time(s)", 15)
+        dash_duo.wait_for_text_to_equal("#result", "Not clicked", 15)
         dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
 
         # Click button and check that status has changed to "Running"
@@ -139,6 +139,10 @@ def test_lcbc003_long_callback_running_cancel(dash_duo, manager):
 
     with setup_long_callback_app(manager, "app3") as app:
         dash_duo.start_server(app)
+        dash_duo.wait_for_text_to_equal("#result", "No results", 15)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 6)
+
+        dash_duo.find_element("#run-button").click()
         dash_duo.wait_for_text_to_equal("#result", "Processed 'initial value'", 15)
         dash_duo.wait_for_text_to_equal("#status", "Finished", 6)
 
@@ -173,8 +177,11 @@ def test_lcbc003_long_callback_running_cancel(dash_duo, manager):
 def test_lcbc004_long_callback_progress(dash_duo, manager):
     with setup_long_callback_app(manager, "app4") as app:
         dash_duo.start_server(app)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
+        dash_duo.wait_for_text_to_equal("#result", "No results", 8)
 
-        # check that status eventually cycles to 2/4
+        # click run and check that status eventually cycles to 2/4
+        dash_duo.find_element("#run-button").click()
         dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 15)
 
         # Then click Cancel button and make sure that the status changes to finish
@@ -277,7 +284,13 @@ def test_lcbc006_long_callback_caching_multi(dash_duo, manager):
         dash_duo.wait_for_text_to_equal("#result1", "Result for 'AAA'", 8)
 
         # Check initial status/output of second long_callback
-        dash_duo.wait_for_text_to_equal("#status2", "Finished", 15)
+        # prevent_initial_callback=True means no calculation should have run yet
+        dash_duo.wait_for_text_to_equal("#status2", "Finished", 8)
+        dash_duo.wait_for_text_to_equal("#result2", "No results", 8)
+
+        # Click second run button
+        dash_duo.find_element("#run-button2").click()
+        dash_duo.wait_for_text_to_equal("#status2", "Progress 2/4", 15)
         dash_duo.wait_for_text_to_equal("#result2", "Result for 'aaa'", 8)
 
         # Update input text box to BBB
@@ -368,3 +381,40 @@ def test_lcbc006_long_callback_caching_multi(dash_duo, manager):
 
         assert not dash_duo.redux_state_is_loading
         assert dash_duo.get_logs() == []
+
+
+def test_lcbc007_validation_layout(dash_duo, manager):
+    with setup_long_callback_app(manager, "app7") as app:
+        dash_duo.start_server(app)
+
+        # Show layout
+        dash_duo.find_element("#show-layout-button").click()
+
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
+        dash_duo.wait_for_text_to_equal("#result", "No results", 8)
+
+        # click run and check that status eventually cycles to 2/4
+        dash_duo.find_element("#run-button").click()
+        dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 15)
+
+        # Then click Cancel button and make sure that the status changes to finish
+        # without updating result
+        dash_duo.find_element("#cancel-button").click()
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 8)
+        dash_duo.wait_for_text_to_equal("#result", "No results", 8)
+
+        # Click run button and allow callback to finish
+        dash_duo.find_element("#run-button").click()
+        dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 15)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 15)
+        dash_duo.wait_for_text_to_equal("#result", "Processed 'hello, world'", 8)
+
+        # Click run button again with same input.
+        # without caching, this should rerun callback and display progress
+        dash_duo.find_element("#run-button").click()
+        dash_duo.wait_for_text_to_equal("#status", "Progress 2/4", 15)
+        dash_duo.wait_for_text_to_equal("#status", "Finished", 15)
+        dash_duo.wait_for_text_to_equal("#result", "Processed 'hello, world'", 8)
+
+    assert not dash_duo.redux_state_is_loading
+    assert dash_duo.get_logs() == []

From 92d7c045f2bad6128dcc09064992f43d0b976bcb Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Tue, 17 Aug 2021 08:17:06 -0400
Subject: [PATCH 31/32] Increase sleep time to allow final app state to settle

---
 tests/integration/long_callback/test_basic_long_callback.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/long_callback/test_basic_long_callback.py b/tests/integration/long_callback/test_basic_long_callback.py
index 1b492cfc94..5bfcb03b38 100644
--- a/tests/integration/long_callback/test_basic_long_callback.py
+++ b/tests/integration/long_callback/test_basic_long_callback.py
@@ -79,8 +79,8 @@ def setup_long_callback_app(manager_name, app_name):
             yield import_app(f"tests.integration.long_callback.{app_name}")
         finally:
             # Interval may run one more time after settling on final app state
-            # Sleep for 1 interval of time
-            time.sleep(0.5)
+            # Sleep for a couple of intervals
+            time.sleep(2.0)
             shutil.rmtree(cache_directory, ignore_errors=True)
             os.environ.pop("LONG_CALLBACK_MANAGER")
             os.environ.pop("DISKCACHE_DIR")

From 3892ecb3ea878561210db8a16176c42dd2097417 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Wed, 18 Aug 2021 11:25:48 -0400
Subject: [PATCH 32/32] Increase sleep time to allow final app state to settle

---
 dash/dash.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dash/dash.py b/dash/dash.py
index c0c755d137..93645dea94 100644
--- a/dash/dash.py
+++ b/dash/dash.py
@@ -263,7 +263,7 @@ class Dash(object):
     clientside callback.
 
     :param long_callback_manager: Long callback manager instance to support the
-    ``@app.long_callback`` decorator. Currently one of
+    ``@app.long_callback`` decorator. Currently an instance of one of
     ``DiskcacheLongCallbackManager`` or ``CeleryLongCallbackManager``
     """
 
@@ -1162,7 +1162,7 @@ def long_callback(self, *_args, **_kwargs):
 
         :Keyword Arguments:
             :param manager:
-                A long callback manager instance. Currently one of
+                A long callback manager instance. Currently an instance of one of
                 `DiskcacheLongCallbackManager` or `CeleryLongCallbackManager`.
                 Defaults to the `long_callback_manager` instance provided to the
                 `dash.Dash constructor`.