From af8630eae423e3ac2bb160c80e4c0966a015c899 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Tue, 2 Aug 2022 14:57:09 +0100 Subject: [PATCH] Check external event cache within deferred Still maintains local in memory lookup optimisation, but does any external lookup as part of the deferred that prevents duplicate lookups for the same event at once. This makes the assumption that fetching from an external cache is a non-zero load operation. --- .../storage/databases/main/events_worker.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index d5d855941194..f7aa75f985c0 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -600,7 +600,10 @@ async def _get_events_from_cache_or_db( Returns: map from event id to result """ - event_entry_map = await self._get_events_from_cache( + # Shortcut: check if we have any events in the *in memory* cache - this function + # may be called repeatedly for the same event so at this point we cannot reach + # out to any external cache for performance reasons. + event_entry_map = self._get_events_from_local_cache( event_ids, ) @@ -632,7 +635,7 @@ async def _get_events_from_cache_or_db( if missing_events_ids: - async def get_missing_events_from_db() -> Dict[str, EventCacheEntry]: + async def get_missing_events_from_cache_or_db() -> Dict[str, EventCacheEntry]: """Fetches the events in `missing_event_ids` from the database. Also creates entries in `self._current_event_fetches` to allow @@ -657,10 +660,18 @@ async def get_missing_events_from_db() -> Dict[str, EventCacheEntry]: # the events have been redacted, and if so pulling the redaction event # out of the database to check it. # + missing_events = {} try: - missing_events = await self._get_events_from_db( + # First fetch from the cache - including any external caches + cache_missing_events = await self._get_events_from_cache( missing_events_ids, ) + missing_events.update(cache_missing_events) + # Now actually fetch any remaining events from the DB + db_missing_events = await self._get_events_from_db( + missing_events_ids - set(cache_missing_events.keys()), + ) + missing_events.update(db_missing_events) except Exception as e: with PreserveLoggingContext(): fetching_deferred.errback(e) @@ -679,7 +690,7 @@ async def get_missing_events_from_db() -> Dict[str, EventCacheEntry]: # cancellations, since multiple `_get_events_from_cache_or_db` calls can # reuse the same fetch. missing_events: Dict[str, EventCacheEntry] = await delay_cancellation( - get_missing_events_from_db() + get_missing_events_from_cache_or_db() ) event_entry_map.update(missing_events)