feat: Support persisting SQLite DB, and data refresher

.github/workflows/openactive-test-suite.yml

@@ -55,7 +55,7 @@ jobs:
       # output
     - name: Run OpenActive.FakeDatabase.NET.Tests
       run: dotnet test ./Fakes/OpenActive.FakeDatabase.NET.Tests/OpenActive.FakeDatabase.NET.Tests.csproj --configuration Release --no-build --verbosity normal
-  
+
   core:
     # Specifies that this job depends on the successful completion of two other jobs: "test-server" and "test-fake-database"
     needs:
@@ -107,6 +107,7 @@ jobs:
 
     # runs `dotnet restore` to install the dependencies for the "OpenActive.Server.NET" project. It is conditional and
     # depends on the "profile" value not being 'no-auth' or 'single-seller'
+    # LW: Why?
     - name: Install OpenActive.Server.NET dependencies
       if: ${{ matrix.profile != 'no-auth' && matrix.profile != 'single-seller' }}
       run: dotnet restore ./server/

.gitignore

@@ -336,3 +336,6 @@ ASALocalRun/
 
 # Fake database
 *fakedatabase.db
+
+# Output path for app publishing
+/web-app-package/

Examples/BookingSystem.AspNetCore/BackgroundServices/FakeDataRefresherService.cs

@@ -0,0 +1,64 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using OpenActive.FakeDatabase.NET;
+using BookingSystem.AspNetCore.Services;
+
+
+namespace BookingSystem
+{
+    // Background task
+    // More information: https://docs.microsoft.com/en-us/dotnet/architecture/microservices/multi-container-microservice-net-applications/background-tasks-with-ihostedservice#implementing-ihostedservice-with-a-custom-hosted-service-class-deriving-from-the-backgroundservice-base-class
+    public class FakeDataRefresherService : BackgroundService
+    {
+        private readonly ILogger<FakeDataRefresherService> _logger;
+        private readonly AppSettings _settings;
+        private readonly FakeBookingSystem _bookingSystem;
+        private readonly DataRefresherStatusService _statusService;
+
+        public FakeDataRefresherService(
+            AppSettings settings, 
+            ILogger<FakeDataRefresherService> logger,
+            FakeBookingSystem bookingSystem,
+            DataRefresherStatusService statusService)
+        {
+            _settings = settings;
+            _logger = logger;
+            _bookingSystem = bookingSystem;
+            _statusService = statusService;
+
+            // Indicate that the refresher service is configured to run
+            _statusService.SetRefresherConfigured(true);
+        }
+
+        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+        {
+
+            stoppingToken.Register(() =>
+                _logger.LogInformation($"FakeDataRefresherService background task is stopping."));
+
+            while (!stoppingToken.IsCancellationRequested)
+            {
+                _logger.LogInformation($"FakeDataRefresherService is starting..");
+                var (numDeletedOccurrences, numDeletedSlots) = await _bookingSystem
+                  .Database
+                  .HardDeleteOldSoftDeletedOccurrencesAndSlots();
+                _logger.LogInformation($"FakeDataRefresherService hard deleted {numDeletedOccurrences} occurrences and {numDeletedSlots} slots that were previously old and soft-deleted.");
+
+                var (numRefreshedOccurrences, numRefreshedSlots) = await _bookingSystem
+                  .Database
+                  .SoftDeletePastOpportunitiesAndInsertNewAtEdgeOfWindow();
+                _logger.LogInformation($"FakeDataRefresherService, for {numRefreshedOccurrences} old occurrences and {numRefreshedSlots} old slots, inserted new copies into the future and soft-deleted the old ones.");
+
+                _logger.LogInformation($"FakeDataRefresherService is finished");
+
+                // Signal that a cycle has completed
+                _statusService.SignalCycleCompletion();
+
+                await Task.Delay(_settings.DataRefresherInterval, stoppingToken);
+            }
+        }
+    }
+}

Examples/BookingSystem.AspNetCore/Controllers/InitWaitController.cs

@@ -0,0 +1,71 @@
+using System;
+using System.Threading.Tasks;
+using BookingSystem.AspNetCore.Services;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+
+/// <summary>
+/// Endpoints which are used to wait for components within
+/// BookingSystem.AspNetCore to be initialized.
+/// </summary>
+namespace BookingSystem.AspNetCore.Controllers
+{
+    [ApiController]
+    [Route("init-wait")]
+    public class InitWaitController : ControllerBase
+    {
+        private readonly DataRefresherStatusService _statusService;
+        private readonly ILogger<InitWaitController> _logger;
+        private static readonly TimeSpan _defaultTimeout = TimeSpan.FromMinutes(5);
+
+        public InitWaitController(
+            DataRefresherStatusService statusService,
+            ILogger<InitWaitController> logger)
+        {
+            _statusService = statusService;
+            _logger = logger;
+        }
+
+        /// <summary>
+        /// Wait for the data refresher to complete its first cycle.
+        /// Returns 204 when the data refresher has completed its first cycle.
+        /// Returns 503 if the data refresher is not configured to run.
+        /// Returns 504 if the data refresher fails to complete a cycle within the default timeout.
+        /// </summary>
+        [HttpGet("data-refresher")]
+        public async Task<IActionResult> WaitForDataRefresher()
+        {
+            _logger.LogDebug("Received request to wait for data refresher completion");
+
+            // Check if the data refresher is configured to run
+            if (!_statusService.IsRefresherConfigured())
+            {
+                _logger.LogWarning("Data refresher is not configured to run");
+                return StatusCode(503, "Data refresher service is not configured to run");
+            }
+
+            // If it has already completed a cycle, return immediately
+            if (_statusService.HasCompletedCycle())
+            {
+                _logger.LogDebug("Data refresher has already completed a cycle");
+                return NoContent();
+            }
+
+            _logger.LogDebug("Waiting for data refresher to complete a cycle...");
+
+            // Wait for the cycle to complete, with a timeout
+            await _statusService.WaitForCycleCompletion(_defaultTimeout);
+
+            if (_statusService.HasCompletedCycle())
+            {
+                _logger.LogDebug("Data refresher completed a cycle, returning 204");
+                return NoContent();
+            }
+            else
+            {
+                _logger.LogWarning("Timed out waiting for data refresher to complete a cycle");
+                return StatusCode(504, "Timed out waiting for data refresher to complete a cycle");
+            }
+        }
+    }
+} 

Examples/BookingSystem.AspNetCore/Feeds/FacilitiesFeeds.cs

@@ -1,5 +1,5 @@
-using Bogus;
-using BookingSystem.AspNetCore.Helpers;
+using Bogus;
+using BookingSystem.AspNetCore.Helpers;
 using OpenActive.DatasetSite.NET;
 using OpenActive.FakeDatabase.NET;
 using OpenActive.NET;
@@ -26,6 +26,7 @@ public AcmeFacilityUseRpdeGenerator(AppSettings appSettings, FakeBookingSystem f
             this._fakeBookingSystem = fakeBookingSystem;
         }
 
+        // TODO this method should use async queries so as not to block the main thread
         protected override async Task<List<RpdeItem<FacilityUse>>> GetRpdeItems(long? afterTimestamp, long? afterId)
         {
             var facilityTypeId = Environment.GetEnvironmentVariable("FACILITY_TYPE_ID") ?? "https://openactive.io/facility-types#a1f82b7a-1258-4d9a-8dc5-bfc2ae961651";
@@ -204,6 +205,7 @@ public AcmeFacilityUseSlotRpdeGenerator(AppSettings appSettings, FakeBookingSyst
             this._fakeBookingSystem = fakeBookingSystem;
         }
 
+        // TODO this method should use async queries so as not to block the main thread
         protected override async Task<List<RpdeItem<Slot>>> GetRpdeItems(long? afterTimestamp, long? afterId)
         {
             using (var db = _fakeBookingSystem.Database.Mem.Database.Open())

Examples/BookingSystem.AspNetCore/Services/DataRefresherStatusService.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace BookingSystem.AspNetCore.Services
+{
+    public class DataRefresherStatusService
+    {
+        private readonly SemaphoreSlim _completionSemaphore = new SemaphoreSlim(0, 1);
+        private bool _isRefresherConfigured = false;
+        private bool _hasCompletedCycle = false;
+
+        public void SetRefresherConfigured(bool isConfigured)
+        {
+            _isRefresherConfigured = isConfigured;
+        }
+
+        public bool IsRefresherConfigured()
+        {
+            return _isRefresherConfigured;
+        }
+
+        public void SignalCycleCompletion()
+        {
+            _hasCompletedCycle = true;
+
+            // Release the semaphore if someone is waiting on it
+            if (_completionSemaphore.CurrentCount == 0)
+            {
+                _completionSemaphore.Release();
+            }
+        }
+
+        public bool HasCompletedCycle()
+        {
+            return _hasCompletedCycle;
+        }
+
+        public async Task WaitForCycleCompletion(TimeSpan timeout)
+        {
+            if (_hasCompletedCycle)
+            {
+                return;
+            }
+
+            await _completionSemaphore.WaitAsync(timeout);
+        }
+    }
+} 

Examples/BookingSystem.AspNetCore/Settings/AppSettings.cs

@@ -1,3 +1,5 @@
+using System;
+
 namespace BookingSystem
 {
     public class AppSettings
@@ -6,6 +8,7 @@ public class AppSettings
         public string OpenIdIssuerUrl { get; set; }
         public FeatureSettings FeatureFlags { get; set; }
         public PaymentSettings Payment { get; set; }
+        public TimeSpan DataRefresherInterval = TimeSpan.FromHours(6);
     }
 
     /**

Examples/BookingSystem.AspNetCore/Startup.cs

@@ -1,4 +1,5 @@
-using Microsoft.AspNetCore.Builder;
+using System;
+using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Configuration;
@@ -9,6 +10,8 @@
 using OpenActive.Server.NET.OpenBookingHelper;
 using Microsoft.AspNetCore.Authorization;
 using OpenActive.FakeDatabase.NET;
+using Microsoft.Extensions.Logging;
+using BookingSystem.AspNetCore.Services;
 
 namespace BookingSystem.AspNetCore
 {
@@ -20,22 +23,37 @@ public Startup(IConfiguration configuration)
             configuration.Bind(AppSettings);
 
             // Provide a simple way to disable token auth for some testing scenarios
-            if (System.Environment.GetEnvironmentVariable("DISABLE_TOKEN_AUTH") == "true")
+            var disableTokenAuthEnvVar = System.Environment.GetEnvironmentVariable("DISABLE_TOKEN_AUTH");
+            if (disableTokenAuthEnvVar == "true")
             {
                 AppSettings.FeatureFlags.EnableTokenAuth = false;
             }
+            else if (disableTokenAuthEnvVar == "false")
+            {
+                AppSettings.FeatureFlags.EnableTokenAuth = true;
+            }
 
             // Provide a simple way to enable FacilityUseHasSlots for some testing scenarios
-            if (System.Environment.GetEnvironmentVariable("FACILITY_USE_HAS_SLOTS") == "true")
+            var facilityUseHasSlotsEnvVar = System.Environment.GetEnvironmentVariable("FACILITY_USE_HAS_SLOTS");
+            if (facilityUseHasSlotsEnvVar == "true")
             {
                 AppSettings.FeatureFlags.FacilityUseHasSlots = true;
             }
+            else if (facilityUseHasSlotsEnvVar == "false")
+            {
+                AppSettings.FeatureFlags.FacilityUseHasSlots = false;
+            }
 
             // Provide a simple way to enable CI mode 
-            if (System.Environment.GetEnvironmentVariable("IS_LOREM_FITSUM_MODE") == "true")
+            var isLoremFitsumModeEnvVar = System.Environment.GetEnvironmentVariable("IS_LOREM_FITSUM_MODE");
+            if (isLoremFitsumModeEnvVar == "true")
             {
                 AppSettings.FeatureFlags.IsLoremFitsumMode = true;
             }
+            else if (isLoremFitsumModeEnvVar == "false")
+            {
+                AppSettings.FeatureFlags.IsLoremFitsumMode = false;
+            }
         }
 
         public AppSettings AppSettings { get; }
@@ -92,7 +110,32 @@ public void ConfigureServices(IServiceCollection services)
                 .AddControllers()
                 .AddMvcOptions(options => options.InputFormatters.Insert(0, new OpenBookingInputFormatter()));
 
-            services.AddSingleton<IBookingEngine>(sp => EngineConfig.CreateStoreBookingEngine(AppSettings, new FakeBookingSystem(AppSettings.FeatureFlags.FacilityUseHasSlots)));
+            services.AddSingleton(x => AppSettings);
+
+            services.AddSingleton(sp => new FakeBookingSystem(
+                AppSettings.FeatureFlags.FacilityUseHasSlots,
+                sp.GetRequiredService<ILogger<FakeBookingSystem>>()
+            ));
+
+            // Register our DataRefresherStatusService as a singleton
+            services.AddSingleton<DataRefresherStatusService>();
+
+            // Use the singleton FakeBookingSystem in IBookingEngine registration
+            services.AddSingleton<IBookingEngine>(sp => 
+                EngineConfig.CreateStoreBookingEngine(
+                    AppSettings, 
+                    sp.GetRequiredService<FakeBookingSystem>()
+                ));
+
+            var doRunDataRefresher = Environment.GetEnvironmentVariable("PERIODICALLY_REFRESH_DATA")?.ToLowerInvariant() == "true";
+            if (doRunDataRefresher) {
+                services.AddHostedService<FakeDataRefresherService>();
+            }
+            else {
+                // If data refresher is not configured to run, update the status service
+                var statusService = services.BuildServiceProvider().GetRequiredService<DataRefresherStatusService>();
+                statusService.SetRefresherConfigured(false);
+            }
         }
 
         // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.