Add a background job to prune outdated sync tokens

We remove all outdated sync tokens, based on their auto-incremented ID.

By default we only keep the last 10 000, but this can be configurable.

Signed-off-by: Thomas Citharel <tcit@tcit.fr>

apps/dav/appinfo/info.xml

@@ -24,6 +24,7 @@
 		<job>OCA\DAV\BackgroundJob\CleanupInvitationTokenJob</job>
 		<job>OCA\DAV\BackgroundJob\EventReminderJob</job>
 		<job>OCA\DAV\BackgroundJob\CalendarRetentionJob</job>
+		<job>OCA\DAV\BackgroundJob\PruneOutdatedSyncTokensJob</job>
 	</background-jobs>
 
 	<repair-steps>

apps/dav/composer/composer/autoload_classmap.php

@@ -18,6 +18,7 @@
     'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => $baseDir . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php',
     'OCA\\DAV\\BackgroundJob\\EventReminderJob' => $baseDir . '/../lib/BackgroundJob/EventReminderJob.php',
     'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => $baseDir . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',
+    'OCA\\DAV\\BackgroundJob\\PruneOutdatedSyncTokensJob' => $baseDir . '/../lib/BackgroundJob/PruneOutdatedSyncTokensJob.php',
     'OCA\\DAV\\BackgroundJob\\RefreshWebcalJob' => $baseDir . '/../lib/BackgroundJob/RefreshWebcalJob.php',
     'OCA\\DAV\\BackgroundJob\\RegisterRegenerateBirthdayCalendars' => $baseDir . '/../lib/BackgroundJob/RegisterRegenerateBirthdayCalendars.php',
     'OCA\\DAV\\BackgroundJob\\UpdateCalendarResourcesRoomsBackgroundJob' => $baseDir . '/../lib/BackgroundJob/UpdateCalendarResourcesRoomsBackgroundJob.php',

apps/dav/composer/composer/autoload_static.php

@@ -33,6 +33,7 @@ class ComposerStaticInitDAV
         'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php',
         'OCA\\DAV\\BackgroundJob\\EventReminderJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/EventReminderJob.php',
         'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',
+        'OCA\\DAV\\BackgroundJob\\PruneOutdatedSyncTokensJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/PruneOutdatedSyncTokensJob.php',
         'OCA\\DAV\\BackgroundJob\\RefreshWebcalJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/RefreshWebcalJob.php',
         'OCA\\DAV\\BackgroundJob\\RegisterRegenerateBirthdayCalendars' => __DIR__ . '/..' . '/../lib/BackgroundJob/RegisterRegenerateBirthdayCalendars.php',
         'OCA\\DAV\\BackgroundJob\\UpdateCalendarResourcesRoomsBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/UpdateCalendarResourcesRoomsBackgroundJob.php',

apps/dav/lib/BackgroundJob/PruneOutdatedSyncTokensJob.php

@@ -0,0 +1,64 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright 2022 Thomas Citharel <nextcloud@tcit.fr>
+ *
+ * @author Thomas Citharel <nextcloud@tcit.fr>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+namespace OCA\DAV\BackgroundJob;
+
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\TimedJob;
+use OCA\DAV\AppInfo\Application;
+use OCA\DAV\CalDAV\CalDavBackend;
+use OCA\DAV\CardDAV\CardDavBackend;
+use OCP\IConfig;
+use Psr\Log\LoggerInterface;
+
+class PruneOutdatedSyncTokensJob extends TimedJob {
+
+	private IConfig $config;
+	private LoggerInterface $logger;
+	private CardDavBackend $cardDavBackend;
+	private CalDavBackend $calDavBackend;
+
+	public function __construct(ITimeFactory $timeFactory, CalDavBackend $calDavBackend, CardDavBackend $cardDavBackend, IConfig $config, LoggerInterface $logger) {
+		parent::__construct($timeFactory);
+		$this->calDavBackend = $calDavBackend;
+		$this->cardDavBackend = $cardDavBackend;
+		$this->config = $config;
+		$this->logger = $logger;
+		$this->setInterval(60 * 60 * 24); // One day
+		$this->setTimeSensitivity(self::TIME_INSENSITIVE);
+	}
+
+	public function run($argument) {
+		$limit = max(1, (int) $this->config->getAppValue(Application::APP_ID, 'totalNumberOfSyncTokensToKeep', '10000'));
+
+		$prunedCalendarSyncTokens = $this->calDavBackend->pruneOutdatedSyncTokens($limit);
+		$prunedAddressBookSyncTokens = $this->cardDavBackend->pruneOutdatedSyncTokens($limit);
+
+		$this->logger->info('Pruned {calendarSyncTokensNumber} calendar sync tokens and {addressBooksSyncTokensNumber} address book sync tokens', [
+			'calendarSyncTokensNumber' => $prunedCalendarSyncTokens,
+			'addressBooksSyncTokensNumber' => $prunedAddressBookSyncTokens
+		]);
+	}
+}

apps/dav/lib/CalDAV/CalDavBackend.php

@@ -3104,6 +3104,20 @@ protected function getCalendarObjectId($calendarId, $uri, $calendarType):int {
 		return (int)$objectIds['id'];
 	}
 
+	/**
+	 * @throws \InvalidArgumentException
+	 */
+	public function pruneOutdatedSyncTokens(int $keep = 10_000): int {
+		if ($keep < 0) {
+			throw new \InvalidArgumentException();
+		}
+		$query = $this->db->getQueryBuilder();
+		$query->delete('calendarchanges')
+			->orderBy('id', 'DESC')
+			->setFirstResult($keep);
+		return $query->executeStatement();
+	}
+
 	/**
 	 * return legacy endpoint principal name to new principal name
 	 *

apps/dav/lib/CardDAV/CardDavBackend.php

@@ -1327,6 +1327,20 @@ public function applyShareAcl(int $addressBookId, array $acl): array {
 		return $this->sharingBackend->applyShareAcl($addressBookId, $acl);
 	}
 
+	/**
+	 * @throws \InvalidArgumentException
+	 */
+	public function pruneOutdatedSyncTokens(int $keep = 10_000): int {
+		if ($keep < 0) {
+			throw new \InvalidArgumentException();
+		}
+		$query = $this->db->getQueryBuilder();
+		$query->delete('addressbookchanges')
+			->orderBy('id', 'DESC')
+			->setFirstResult($keep);
+		return $query->executeStatement();
+	}
+
 	private function convertPrincipal(string $principalUri, bool $toV2): string {
 		if ($this->principalBackend->getPrincipalPrefix() === 'principals') {
 			[, $name] = \Sabre\Uri\split($principalUri);

apps/dav/tests/unit/BackgroundJob/CleanupInvitationTokenJobTest.php

@@ -44,7 +44,7 @@ class CleanupInvitationTokenJobTest extends TestCase {
 	/** @var ITimeFactory | \PHPUnit\Framework\MockObject\MockObject */
 	private $timeFactory;
 
-	/** @var \OCA\DAV\BackgroundJob\GenerateBirthdayCalendarBackgroundJob */
+	/** @var \OCA\DAV\BackgroundJob\CleanupInvitationTokenJob */
 	private $backgroundJob;
 
 	protected function setUp(): void {

apps/dav/tests/unit/BackgroundJob/PruneOutdatedSyncTokensJobTest.php

@@ -0,0 +1,104 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright 2018, Georg Ehrke <oc.list@georgehrke.com>
+ *
+ * @author Christoph Wurst <christoph@winzerhof-wurst.at>
+ * @author Georg Ehrke <oc.list@georgehrke.com>
+ * @author Joas Schilling <coding@schilljs.com>
+ * @author Morris Jobke <hey@morrisjobke.de>
+ * @author Roeland Jago Douma <roeland@famdouma.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+namespace OCA\DAV\Tests\unit\BackgroundJob;
+
+use OCA\DAV\AppInfo\Application;
+use OCA\DAV\BackgroundJob\PruneOutdatedSyncTokensJob;
+use OCA\DAV\CalDAV\CalDavBackend;
+use OCA\DAV\CardDAV\CardDavBackend;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\DB\Exception;
+use OCP\IConfig;
+use PHPUnit\Framework\MockObject\MockObject;
+use Psr\Log\LoggerInterface;
+use Test\TestCase;
+
+class PruneOutdatedSyncTokensJobTest extends TestCase {
+
+	/** @var CalDavBackend | MockObject */
+	private $calDavBackend;
+
+	/** @var CardDavBackend | MockObject */
+	private $cardDavBackend;
+
+	/** @var IConfig|MockObject */
+	private $config;
+
+	/** @var LoggerInterface|MockObject*/
+	private $logger;
+
+	/** @var PruneOutdatedSyncTokensJob */
+	private PruneOutdatedSyncTokensJob $backgroundJob;
+
+	protected function setUp(): void {
+		parent::setUp();
+
+		$this->timeFactory = $this->createMock(ITimeFactory::class);
+		$this->calDavBackend = $this->createMock(CalDavBackend::class);
+		$this->cardDavBackend = $this->createMock(CardDavBackend::class);
+		$this->config = $this->createMock(IConfig::class);
+		$this->logger = $this->createMock(LoggerInterface::class);
+
+		$this->backgroundJob = new PruneOutdatedSyncTokensJob($this->timeFactory, $this->calDavBackend, $this->cardDavBackend, $this->config, $this->logger);
+	}
+
+	/**
+	 * @dataProvider dataForTestRun
+	 */
+	public function testRun(string $configValue, int $actualLimit, int $deletedCalendarSyncTokens, int $deletedAddressBookSyncTokens) {
+		$this->config->expects($this->once())
+			->method('getAppValue')
+			->with(Application::APP_ID, 'totalNumberOfSyncTokensToKeep', '10000')
+			->willReturn($configValue);
+		$this->calDavBackend->expects($this->once())
+			->method('pruneOutdatedSyncTokens')
+			->with($actualLimit)
+			->willReturn($deletedCalendarSyncTokens);
+		$this->cardDavBackend->expects($this->once())
+			->method('pruneOutdatedSyncTokens')
+			->with($actualLimit)
+			->willReturn($deletedAddressBookSyncTokens);
+		$this->logger->expects($this->once())
+			->method('info')
+			->with('Pruned {calendarSyncTokensNumber} calendar sync tokens and {addressBooksSyncTokensNumber} address book sync tokens', [
+				'calendarSyncTokensNumber' => $deletedCalendarSyncTokens,
+				'addressBooksSyncTokensNumber' => $deletedAddressBookSyncTokens
+			]);
+
+		$this->backgroundJob->run(null);
+	}
+
+	public function dataForTestRun(): array {
+		return [
+			['100', 100, 2, 3],
+			['0', 1, 0, 0]
+		];
+	}
+}

apps/dav/tests/unit/CalDAV/CalDavBackendTest.php

@@ -1273,4 +1273,58 @@ public function testSearchPrincipal(): void {
 		$this->assertEquals($sharerPrivate, $sharerSearchResults[1]['calendardata']);
 		$this->assertEquals($sharerConfidential, $sharerSearchResults[2]['calendardata']);
 	}
+
+	/**
+	 * @throws \OCP\DB\Exception
+	 * @throws \Sabre\DAV\Exception\BadRequest
+	 */
+	public function testPruneOutdatedSyncTokens(): void {
+		$calendarId = $this->createTestCalendar();
+
+		$uri = static::getUniqueID('calobj');
+		$calData = <<<EOD
+BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:Nextcloud Calendar
+BEGIN:VEVENT
+CREATED;VALUE=DATE-TIME:20130910T125139Z
+UID:47d15e3ec8
+LAST-MODIFIED;VALUE=DATE-TIME:20130910T125139Z
+DTSTAMP;VALUE=DATE-TIME:20130910T125139Z
+SUMMARY:Test Event
+DTSTART;VALUE=DATE-TIME:20130912T130000Z
+DTEND;VALUE=DATE-TIME:20130912T140000Z
+CLASS:PUBLIC
+END:VEVENT
+END:VCALENDAR
+EOD;
+
+		$this->backend->createCalendarObject($calendarId, $uri, $calData);
+
+		// update the card
+		$calData = <<<'EOD'
+BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:Nextcloud Calendar
+BEGIN:VEVENT
+CREATED;VALUE=DATE-TIME:20130910T125139Z
+UID:47d15e3ec8
+LAST-MODIFIED;VALUE=DATE-TIME:20130910T125139Z
+DTSTAMP;VALUE=DATE-TIME:20130910T125139Z
+SUMMARY:123 Event 🙈
+DTSTART;VALUE=DATE-TIME:20130912T130000Z
+DTEND;VALUE=DATE-TIME:20130912T140000Z
+ATTENDEE;CN=test:mailto:foo@bar.com
+END:VEVENT
+END:VCALENDAR
+EOD;
+		$this->backend->updateCalendarObject($calendarId, $uri, $calData);
+		$deleted = $this->backend->pruneOutdatedSyncTokens(0);
+		// At least one from the object creation and one from the object update
+		$this->assertGreaterThanOrEqual(2, $deleted);
+		$changes = $this->backend->getChangesForCalendar($calendarId, '5', 1);
+		$this->assertEmpty($changes['added']);
+		$this->assertEmpty($changes['modified']);
+		$this->assertEmpty($changes['deleted']);
+	}
 }

apps/dav/tests/unit/CardDAV/CardDavBackendTest.php

@@ -845,4 +845,22 @@ public function testCollectCardProperties() {
 		$result = $this->backend->collectCardProperties(666, 'FN');
 		$this->assertEquals(['John Doe'], $result);
 	}
+
+	/**
+	 * @throws \OCP\DB\Exception
+	 * @throws \Sabre\DAV\Exception\BadRequest
+	 */
+	public function testPruneOutdatedSyncTokens(): void {
+		$addressBookId = $this->backend->createAddressBook(self::UNIT_TEST_USER, 'Example', []);
+		$uri = $this->getUniqueID('card');
+		$this->backend->createCard($addressBookId, $uri, $this->vcardTest0);
+		$this->backend->updateCard($addressBookId, $uri, $this->vcardTest1);
+		$deleted = $this->backend->pruneOutdatedSyncTokens(0);
+		// At least one from the object creation and one from the object update
+		$this->assertGreaterThanOrEqual(2, $deleted);
+		$changes = $this->backend->getChangesForAddressBook($addressBookId, '5', 1);
+		$this->assertEmpty($changes['added']);
+		$this->assertEmpty($changes['modified']);
+		$this->assertEmpty($changes['deleted']);
+	}
 }