[stable19] Handle limit offset and sorting in files search

lib/private/Files/Cache/Cache.php

@@ -188,10 +188,10 @@ public static function cacheEntryFromData($data, IMimeTypeLoader $mimetypeLoader
 		}
 		$data['permissions'] = (int)$data['permissions'];
 		if (isset($data['creation_time'])) {
-			$data['creation_time'] = (int) $data['creation_time'];
+			$data['creation_time'] = (int)$data['creation_time'];
 		}
 		if (isset($data['upload_time'])) {
-			$data['upload_time'] = (int) $data['upload_time'];
+			$data['upload_time'] = (int)$data['upload_time'];
 		}
 		return new CacheEntry($data);
 	}
@@ -788,14 +788,18 @@ public function searchQuery(ISearchQuery $searchQuery) {
 		$query->whereStorageId();
 
 		if ($this->querySearchHelper->shouldJoinTags($searchQuery->getSearchOperation())) {
+			$user = $searchQuery->getUser();
+			if ($user === null) {
+				throw new \InvalidArgumentException("Searching by tag requires the user to be set in the query");
+			}
 			$query
 				->innerJoin('file', 'vcategory_to_object', 'tagmap', $builder->expr()->eq('file.fileid', 'tagmap.objid'))
 				->innerJoin('tagmap', 'vcategory', 'tag', $builder->expr()->andX(
 					$builder->expr()->eq('tagmap.type', 'tag.type'),
 					$builder->expr()->eq('tagmap.categoryid', 'tag.id')
 				))
 				->andWhere($builder->expr()->eq('tag.type', $builder->createNamedParameter('files')))
-				->andWhere($builder->expr()->eq('tag.uid', $builder->createNamedParameter($searchQuery->getUser()->getUID())));
+				->andWhere($builder->expr()->eq('tag.uid', $builder->createNamedParameter($user->getUID())));
 		}
 
 		$searchExpr = $this->querySearchHelper->searchOperatorToDBExpr($builder, $searchQuery->getSearchOperation());

lib/private/Files/Node/Folder.php

@@ -31,14 +31,23 @@
 namespace OC\Files\Node;
 
 use OC\DB\QueryBuilder\Literal;
+use OC\Files\Search\SearchBinaryOperator;
+use OC\Files\Search\SearchComparison;
+use OC\Files\Search\SearchQuery;
+use OC\Files\Storage\Storage;
 use OCA\Files_Sharing\SharedStorage;
 use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\Files\Cache\ICacheEntry;
 use OCP\Files\Config\ICachedMountInfo;
 use OCP\Files\FileInfo;
 use OCP\Files\Mount\IMountPoint;
 use OCP\Files\NotFoundException;
 use OCP\Files\NotPermittedException;
+use OCP\Files\Search\ISearchBinaryOperator;
+use OCP\Files\Search\ISearchComparison;
+use OCP\Files\Search\ISearchOperator;
 use OCP\Files\Search\ISearchQuery;
+use OCP\IUserManager;
 
 class Folder extends Node implements \OCP\Files\Folder {
 	/**
@@ -94,8 +103,8 @@ public function isSubNode($node) {
 	/**
 	 * get the content of this directory
 	 *
-	 * @throws \OCP\Files\NotFoundException
 	 * @return Node[]
+	 * @throws \OCP\Files\NotFoundException
 	 */
 	public function getDirectoryListing() {
 		$folderContent = $this->view->getDirectoryContent($this->path);
@@ -198,6 +207,17 @@ public function newFile($path, $content = null) {
 		throw new NotPermittedException('No create permission for path');
 	}
 
+	private function queryFromOperator(ISearchOperator $operator, string $uid = null): ISearchQuery {
+		if ($uid === null) {
+			$user = null;
+		} else {
+			/** @var IUserManager $userManager */
+			$userManager = \OC::$server->query(IUserManager::class);
+			$user = $userManager->get($uid);
+		}
+		return new SearchQuery($operator, 0, 0, [], $user);
+	}
+
 	/**
 	 * search for files with the name matching $query
 	 *
@@ -206,45 +226,27 @@ public function newFile($path, $content = null) {
 	 */
 	public function search($query) {
 		if (is_string($query)) {
-			return $this->searchCommon('search', ['%' . $query . '%']);
-		} else {
-			return $this->searchCommon('searchQuery', [$query]);
+			$query = $this->queryFromOperator(new SearchComparison(ISearchComparison::COMPARE_LIKE, 'name', '%' . $query . '%'));
 		}
-	}
-
-	/**
-	 * search for files by mimetype
-	 *
-	 * @param string $mimetype
-	 * @return Node[]
-	 */
-	public function searchByMime($mimetype) {
-		return $this->searchCommon('searchByMime', [$mimetype]);
-	}
-
-	/**
-	 * search for files by tag
-	 *
-	 * @param string|int $tag name or tag id
-	 * @param string $userId owner of the tags
-	 * @return Node[]
-	 */
-	public function searchByTag($tag, $userId) {
-		return $this->searchCommon('searchByTag', [$tag, $userId]);
-	}
 
-	/**
-	 * @param string $method cache method
-	 * @param array $args call args
-	 * @return \OC\Files\Node\Node[]
-	 */
-	private function searchCommon($method, $args) {
-		$limitToHome = ($method === 'searchQuery')? $args[0]->limitToHome(): false;
+		// Limit+offset for queries with ordering
+		//
+		// Because we currently can't do ordering between the results from different storages in sql
+		// The only way to do ordering is requesting the $limit number of entries from all storages
+		// sorting them and returning the first $limit entries.
+		//
+		// For offset we have the same problem, we don't know how many entries from each storage should be skipped
+		// by a given $offset, so instead we query $offset + $limit from each storage and return entries $offset..($offset+$limit)
+		// after merging and sorting them.
+		//
+		// This is suboptimal but because limit and offset tend to be fairly small in real world use cases it should
+		// still be significantly better than disabling paging altogether
+
+		$limitToHome = $query->limitToHome();
 		if ($limitToHome && count(explode('/', $this->path)) !== 3) {
 			throw new \InvalidArgumentException('searching by owner is only allows on the users home folder');
 		}
 
-		$files = [];
 		$rootLength = strlen($this->path);
 		$mount = $this->root->getMount($this->path);
 		$storage = $mount->getStorage();
@@ -253,45 +255,106 @@ private function searchCommon($method, $args) {
 		if ($internalPath !== '') {
 			$internalPath = $internalPath . '/';
 		}
-		$internalRootLength = strlen($internalPath);
+
+		$subQueryLimit = $query->getLimit() > 0 ? $query->getLimit() + $query->getOffset() : 0;
+		$rootQuery = new SearchQuery(
+			new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
+				new SearchComparison(ISearchComparison::COMPARE_LIKE, 'path', $internalPath . '%'),
+				$query->getSearchOperation(),
+			]
+			),
+			$subQueryLimit,
+			0,
+			$query->getOrder(),
+			$query->getUser()
+		);
+
+		$files = [];
 
 		$cache = $storage->getCache('');
 
-		$results = call_user_func_array([$cache, $method], $args);
+		$results = $cache->searchQuery($rootQuery);
 		foreach ($results as $result) {
-			if ($internalRootLength === 0 or substr($result['path'], 0, $internalRootLength) === $internalPath) {
-				$result['internalPath'] = $result['path'];
-				$result['path'] = substr($result['path'], $internalRootLength);
-				$result['storage'] = $storage;
-				$files[] = new \OC\Files\FileInfo($this->path . '/' . $result['path'], $storage, $result['internalPath'], $result, $mount);
-			}
+			$files[] = $this->cacheEntryToFileInfo($mount, '', $internalPath, $result);
 		}
 
 		if (!$limitToHome) {
 			$mounts = $this->root->getMountsIn($this->path);
 			foreach ($mounts as $mount) {
+				$subQuery = new SearchQuery(
+					$query->getSearchOperation(),
+					$subQueryLimit,
+					0,
+					$query->getOrder(),
+					$query->getUser()
+				);
+
 				$storage = $mount->getStorage();
 				if ($storage) {
 					$cache = $storage->getCache('');
 
 					$relativeMountPoint = ltrim(substr($mount->getMountPoint(), $rootLength), '/');
-					$results = call_user_func_array([$cache, $method], $args);
+					$results = $cache->searchQuery($subQuery);
 					foreach ($results as $result) {
-						$result['internalPath'] = $result['path'];
-						$result['path'] = $relativeMountPoint . $result['path'];
-						$result['storage'] = $storage;
-						$files[] = new \OC\Files\FileInfo($this->path . '/' . $result['path'], $storage,
-							$result['internalPath'], $result, $mount);
+						$files[] = $this->cacheEntryToFileInfo($mount, $relativeMountPoint, '', $result);
 					}
 				}
 			}
 		}
 
+		$order = $query->getOrder();
+		if ($order) {
+			usort($files, function (FileInfo $a,FileInfo  $b) use ($order) {
+				foreach ($order as $orderField) {
+					$cmp = $orderField->sortFileInfo($a, $b);
+					if ($cmp !== 0) {
+						return $cmp;
+					}
+				}
+				return 0;
+			});
+		}
+		$files = array_values(array_slice($files, $query->getOffset(), $query->getLimit() > 0 ? $query->getLimit() : null));
+
 		return array_map(function (FileInfo $file) {
 			return $this->createNode($file->getPath(), $file);
 		}, $files);
 	}
 
+	private function cacheEntryToFileInfo(IMountPoint $mount, string $appendRoot, string $trimRoot, ICacheEntry $cacheEntry): FileInfo {
+		$trimLength = strlen($trimRoot);
+		$cacheEntry['internalPath'] = $cacheEntry['path'];
+		$cacheEntry['path'] = $appendRoot . substr($cacheEntry['path'], $trimLength);
+		return new \OC\Files\FileInfo($this->path . '/' . $cacheEntry['path'], $mount->getStorage(), $cacheEntry['internalPath'], $cacheEntry, $mount);
+	}
+
+	/**
+	 * search for files by mimetype
+	 *
+	 * @param string $mimetype
+	 * @return Node[]
+	 */
+	public function searchByMime($mimetype) {
+		if (strpos($mimetype, '/') === false) {
+			$query = $this->queryFromOperator(new SearchComparison(ISearchComparison::COMPARE_LIKE, 'mimetype', $mimetype . '/%'));
+		} else {
+			$query = $this->queryFromOperator(new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'mimetype', $mimetype));
+		}
+		return $this->search($query);
+	}
+
+	/**
+	 * search for files by tag
+	 *
+	 * @param string|int $tag name or tag id
+	 * @param string $userId owner of the tags
+	 * @return Node[]
+	 */
+	public function searchByTag($tag, $userId) {
+		$query = $this->queryFromOperator(new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'tagname', $tag), $userId);
+		return $this->search($query);
+	}
+
 	/**
 	 * @param int $id
 	 * @return \OC\Files\Node\Node[]
@@ -318,7 +381,7 @@ public function getById($id) {
 
 		if (count($mountsContainingFile) === 0) {
 			if ($user === $this->getAppDataDirectoryName()) {
-				return $this->getByIdInRootMount((int) $id);
+				return $this->getByIdInRootMount((int)$id);
 			}
 			return [];
 		}
@@ -381,11 +444,11 @@ protected function getByIdInRootMount(int $id): array {
 
 		return [$this->root->createNode(
 			$absolutePath, new \OC\Files\FileInfo(
-				$absolutePath,
-				$mount->getStorage(),
-				$cacheEntry->getPath(),
-				$cacheEntry,
-				$mount
+			$absolutePath,
+			$mount->getStorage(),
+			$cacheEntry->getPath(),
+			$cacheEntry,
+			$mount
 		))];
 	}
 

lib/private/Files/Search/SearchOrder.php

@@ -23,6 +23,7 @@
 
 namespace OC\Files\Search;
 
+use OCP\Files\FileInfo;
 use OCP\Files\Search\ISearchOrder;
 
 class SearchOrder implements ISearchOrder {
@@ -55,4 +56,28 @@ public function getDirection() {
 	public function getField() {
 		return $this->field;
 	}
+
+	public function sortFileInfo(FileInfo $a, FileInfo $b): int {
+		$cmp = $this->sortFileInfoNoDirection($a, $b);
+		return $cmp * ($this->direction === ISearchOrder::DIRECTION_ASCENDING ? 1 : -1);
+	}
+
+	private function sortFileInfoNoDirection(FileInfo $a, FileInfo $b): int {
+		switch ($this->field) {
+			case 'name':
+				return $a->getName() <=> $b->getName();
+			case 'mimetype':
+				return $a->getMimetype() <=> $b->getMimetype();
+			case 'mtime':
+				return $a->getMtime() <=> $b->getMtime();
+			case 'size':
+				return $a->getSize() <=> $b->getSize();
+			case 'fileid':
+				return $a->getId() <=> $b->getId();
+			case 'permissions':
+				return $a->getPermissions() <=> $b->getPermissions();
+			default:
+				return 0;
+		}
+	}
 }

lib/private/Files/Search/SearchQuery.php

@@ -37,7 +37,7 @@ class SearchQuery implements ISearchQuery {
 	private $offset;
 	/** @var  ISearchOrder[] */
 	private $order;
-	/** @var IUser */
+	/** @var ?IUser */
 	private $user;
 	private $limitToHome;
 
@@ -48,15 +48,15 @@ class SearchQuery implements ISearchQuery {
 	 * @param int $limit
 	 * @param int $offset
 	 * @param array $order
-	 * @param IUser $user
+	 * @param ?IUser $user
 	 * @param bool $limitToHome
 	 */
 	public function __construct(
 		ISearchOperator $searchOperation,
 		int $limit,
 		int $offset,
 		array $order,
-		IUser $user,
+		?IUser $user = null,
 		bool $limitToHome = false
 	) {
 		$this->searchOperation = $searchOperation;
@@ -96,7 +96,7 @@ public function getOrder() {
 	}
 
 	/**
-	 * @return IUser
+	 * @return ?IUser
 	 */
 	public function getUser() {
 		return $this->user;