Allow REGEX expressions with a dynamically computed regular expression (#1889)

Add support for the `REGEX` function in the case where the regular expression is dynamically computed by the query, e.g.
`REGEX(?input, ?regex)` or `REGEX(?input, CONCAT("bla", "*"))`. Previously, QLever only supported hardcoded regular expressions like `REGEX(?input, "bla*")`.
The implementation optimizes for the case, that the regex is stored in a variable with a low multiplicity, so for example for the case `REGEX(?input, ?regex)`, where the `?regex` only binds to a few distinct values by using a simple LRU cache in the `RegexValueGetter`.
Also clean up the code for the `Regex`, `Replace` and `Prefix` expressions.
Fixes #1821

Author: RobinTF

src/engine/sparqlExpressions/RegexExpression.cpp

@@ -4,78 +4,77 @@
 
 #pragma once
 
+#include <gtest/gtest_prod.h>
+#include <re2/re2.h>
+
 #include <string>
 
-#include "engine/sparqlExpressions/LiteralExpression.h"
 #include "engine/sparqlExpressions/SparqlExpression.h"
-#include "re2/re2.h"
 
 namespace sparqlExpression {
-// Class implementing the REGEX function, which takes two mandatory arguments
-// (an expression and a regex) and one optional argument (a string of flags).
-class RegexExpression : public SparqlExpression {
+// Class implementing a specialization of the REGEX function. This optimization
+// is possible if the regex is known in advance and is a simple prefix regex.
+class PrefixRegexExpression : public SparqlExpression {
  private:
-  SparqlExpression::Ptr child_;
-  // The reguar expression. It needs to be a `std::optional` because `RE2`
-  // objects do not have a default constructor.
-  std::optional<RE2> regex_;
-  // If this `std::optional` holds a string, we have a simple prefix regex
-  // (which translates to a range search) and this string holds the prefix.
-  std::optional<std::string> prefixRegex_;
-  // The regex as a string, used for the cache key.
-  std::string regexAsString_;
-
-  // True iff the expression is enclosed in `STR()`.
+  Ptr child_;
+  // A simple prefix regex (which translates to a range search) and this string
+  // holds the prefix.
+  std::string prefixRegex_;
+  // Holds the variable over which the regex is evaluated.
+  Variable variable_;
+  // If the variable is wrapped inside a `STR()` function, this is set to true.
   bool childIsStrExpression_ = false;
 
- public:
   // The `child` must be a `VariableExpression` and `regex` must be a
   // `LiteralExpression` that stores a string, otherwise an exception will be
   // thrown.
-  RegexExpression(SparqlExpression::Ptr child, SparqlExpression::Ptr regex,
-                  std::optional<SparqlExpression::Ptr> optionalFlags);
+  PrefixRegexExpression(Ptr child, std::string prefixRegex, Variable variable);
+
+ public:
+  PrefixRegexExpression(PrefixRegexExpression&&) noexcept = default;
+  PrefixRegexExpression& operator=(PrefixRegexExpression&&) noexcept = default;
+  PrefixRegexExpression(const PrefixRegexExpression&) noexcept = delete;
+  PrefixRegexExpression& operator=(const PrefixRegexExpression&) noexcept =
+      delete;
+
+  // Check if the children of this expression allow for the prefix regex
+  // optimization. If this is the case, a `PrefixRegexExpression` is returned,
+  // otherwise `std::nullopt`.
+  static std::optional<PrefixRegexExpression>
+  makePrefixRegexExpressionIfPossible(Ptr& string,
+                                      const SparqlExpression& regex);
 
+  // ___________________________________________________________________________
   ExpressionResult evaluate(EvaluationContext* context) const override;
 
-  // _________________________________________________________________________
-  [[nodiscard]] string getCacheKey(
+  // ___________________________________________________________________________
+  [[nodiscard]] std::string getCacheKey(
       const VariableToColumnMap& varColMap) const override;
 
-  // _________________________________________________________________________
-  [[nodiscard]] bool isPrefixExpression() const;
-
-  // _________________________________________________________________________
+  // ___________________________________________________________________________
   Estimates getEstimatesForFilterExpression(
       uint64_t inputSize,
       const std::optional<Variable>& firstSortedVariable) const override;
 
  private:
-  std::span<SparqlExpression::Ptr> childrenImpl() override;
-
-  // Evaluate for the special case, where the expression is a variable and we
-  // have a simple prefix regex (in which case the regex match translates to a
-  // simple range check).
-  ExpressionResult evaluatePrefixRegex(
-      const Variable& variable,
-      sparqlExpression::EvaluationContext* context) const;
-
-  // Evaluate for the general case.
-  CPP_template(typename T)(requires SingleExpressionResult<T>) ExpressionResult
-      evaluateGeneralCase(T&& input,
-                          sparqlExpression::EvaluationContext* context) const;
+  std::span<Ptr> childrenImpl() override;
 
   // Check if the `CancellationHandle` of `context` has been cancelled and throw
   // an exception if this is the case.
-  static void checkCancellation(
-      const sparqlExpression::EvaluationContext* context,
-      ad_utility::source_location location =
-          ad_utility::source_location::current());
+  static void checkCancellation(const EvaluationContext* context,
+                                ad_utility::source_location location =
+                                    ad_utility::source_location::current());
+
+  // Check if `regex` is a prefix regex which means that it starts with `^` and
+  // contains no other "special" regex characters like `*` or `.`. If this check
+  // succeeds, the prefix is returned without the leading `^` and with all
+  // escaping undone. Else, `std::nullopt` is returned.
+  static std::optional<std::string> getPrefixRegex(std::string regex);
+
+  FRIEND_TEST(RegexExpression, getPrefixRegex);
 };
-namespace detail {
-// Check if `regex` is a prefix regex which means that it starts with `^` and
-// contains no other "special" regex characters like `*` or `.`. If this check
-// succeeds, the prefix is returned without the leading `^` and with all
-// escaping undone. Else, `std::nullopt` is returned.
-std::optional<std::string> getPrefixRegex(std::string regex);
-}  // namespace detail
+
+SparqlExpression::Ptr makeRegexExpression(SparqlExpression::Ptr string,
+                                          SparqlExpression::Ptr regex,
+                                          SparqlExpression::Ptr flags);
 }  // namespace sparqlExpression

src/engine/sparqlExpressions/RegexExpression.h

@@ -124,7 +124,7 @@ LiteralValueGetterWithoutStrFunction::operator()(
 std::optional<std::string> ReplacementStringGetter::operator()(
     Id id, const EvaluationContext* context) const {
   std::optional<std::string> originalString =
-      StringValueGetter::operator()(id, context);
+      LiteralFromIdGetter{}(id, context);
   if (!originalString.has_value()) {
     return originalString;
   }

src/engine/sparqlExpressions/SparqlExpressionValueGetters.cpp

@@ -9,11 +9,11 @@
 #include <re2/re2.h>
 
 #include "engine/ExportQueryExecutionTrees.h"
-#include "engine/Result.h"
 #include "engine/sparqlExpressions/SparqlExpressionTypes.h"
 #include "global/Id.h"
 #include "parser/GeoPoint.h"
 #include "util/ConstexprSmallString.h"
+#include "util/LruCache.h"
 #include "util/TypeTraits.h"
 
 /// Several classes that can be used as the `ValueGetter` template
@@ -144,22 +144,6 @@ struct StringValueGetter : Mixin<StringValueGetter> {
     return std::string(asStringViewUnsafe(s.getContent()));
   }
 };
-// Similar to `StringValueGetter`, but correctly preprocesses strings so that
-// they can be used by re2 as replacement strings. So '$1 \abc \$' becomes
-// '\1 \\abc $', where the former variant is valid in the SPARQL standard and
-// the latter represents the format that re2 expects.
-struct ReplacementStringGetter : StringValueGetter,
-                                 Mixin<ReplacementStringGetter> {
-  using Mixin<ReplacementStringGetter>::operator();
-  std::optional<std::string> operator()(ValueId,
-                                        const EvaluationContext*) const;
-
-  std::optional<std::string> operator()(const LiteralOrIri& s,
-                                        const EvaluationContext*) const;
-
- private:
-  static std::string convertToReplacementString(std::string_view view);
-};
 
 // This class can be used as the `ValueGetter` argument of Expression
 // templates. It implicitly applies the STR() function. In particular,
@@ -295,19 +279,39 @@ struct LiteralFromIdGetter : Mixin<LiteralFromIdGetter> {
   }
 };
 
-// Convert the input into a `unique_ptr<RE2>`. Return nullptr if the input is
+// Similar to `LiteralFromIdGetter`, but correctly preprocesses strings so that
+// they can be used by re2 as replacement strings. So '$1 \abc \$' becomes
+// '\1 \\abc $', where the former variant is valid in the SPARQL standard and
+// the latter represents the format that re2 expects.
+struct ReplacementStringGetter : Mixin<ReplacementStringGetter> {
+  using Mixin<ReplacementStringGetter>::operator();
+  std::optional<std::string> operator()(ValueId,
+                                        const EvaluationContext*) const;
+
+  std::optional<std::string> operator()(const LiteralOrIri& s,
+                                        const EvaluationContext*) const;
+
+ private:
+  static std::string convertToReplacementString(std::string_view view);
+};
+
+// Convert the input into a `shared_ptr<RE2>`. Return nullptr if the input is
 // not convertible to a string.
 struct RegexValueGetter {
+  mutable ad_utility::util::LRUCache<std::string, std::shared_ptr<RE2>> cache_{
+      100};
   template <typename S>
-  auto operator()(S&& input, const EvaluationContext* context) const -> CPP_ret(
-      std::unique_ptr<re2::RE2>)(
-      requires SingleExpressionResult<S>&&
-          ranges::invocable<StringValueGetter, S&&, const EvaluationContext*>) {
-    auto str = StringValueGetter{}(AD_FWD(input), context);
+  auto operator()(S&& input, const EvaluationContext* context) const
+      -> CPP_ret(std::shared_ptr<RE2>)(
+          requires SingleExpressionResult<S>&& ranges::invocable<
+              LiteralFromIdGetter, S&&, const EvaluationContext*>) {
+    auto str = LiteralFromIdGetter{}(AD_FWD(input), context);
     if (!str.has_value()) {
       return nullptr;
     }
-    return std::make_unique<re2::RE2>(str.value(), re2::RE2::Quiet);
+    return cache_.getOrCompute(str.value(), [](const std::string& value) {
+      return std::make_shared<RE2>(value, RE2::Quiet);
+    });
   }
 };
 

src/engine/sparqlExpressions/SparqlExpressionValueGetters.h

@@ -6,6 +6,7 @@
 
 #include "engine/sparqlExpressions/LiteralExpression.h"
 #include "engine/sparqlExpressions/NaryExpressionImpl.h"
+#include "engine/sparqlExpressions/StringExpressionsHelper.h"
 #include "engine/sparqlExpressions/VariadicExpression.h"
 
 namespace sparqlExpression {
@@ -66,70 +67,6 @@ class StrExpression : public StrExpressionImpl {
   bool isStrExpression() const override { return true; }
 };
 
-// Template for an expression that works on string literals. The arguments are
-// the same as those to `NaryExpression` with the difference that the value
-// getter is deduced automatically. If the child of the expression is the
-// `STR()` expression, then the `StringValueGetter` will be used (which also
-// returns string values for IRIs, numeric literals, etc.), otherwise the
-// `LiteralFromIdGetter` is used (which returns `std::nullopt` for these cases).
-template <typename ValueGetterWithStr, typename ValueGetterWithoutStr, size_t N,
-          typename Function, typename... AdditionalNonStringValueGetters>
-class StringExpressionImplImpl : public SparqlExpression {
- private:
-  using ExpressionWithStr = NARY<
-      N, FV<Function, ValueGetterWithStr, AdditionalNonStringValueGetters...>>;
-  using ExpressionWithoutStr = NARY<N, FV<Function, ValueGetterWithoutStr,
-                                          AdditionalNonStringValueGetters...>>;
-
-  SparqlExpression::Ptr impl_;
-
- public:
-  CPP_template(typename... C)(
-      requires(concepts::same_as<C, SparqlExpression::Ptr>&&...) CPP_and(
-          sizeof...(C) + 1 ==
-          N)) explicit StringExpressionImplImpl(SparqlExpression::Ptr child,
-                                                C... children) {
-    AD_CORRECTNESS_CHECK(child != nullptr);
-    if (child->isStrExpression()) {
-      auto childrenOfStr = std::move(*child).moveChildrenOut();
-      AD_CORRECTNESS_CHECK(childrenOfStr.size() == 1);
-      impl_ = std::make_unique<ExpressionWithStr>(
-          std::move(childrenOfStr.at(0)), std::move(children)...);
-    } else {
-      impl_ = std::make_unique<ExpressionWithoutStr>(std::move(child),
-                                                     std::move(children)...);
-    }
-  }
-
-  ExpressionResult evaluate(EvaluationContext* context) const override {
-    return impl_->evaluate(context);
-  }
-  std::string getCacheKey(const VariableToColumnMap& varColMap) const override {
-    return impl_->getCacheKey(varColMap);
-  }
-
- private:
-  std::span<SparqlExpression::Ptr> childrenImpl() override {
-    return impl_->children();
-  }
-};
-
-// Impl class for expressions that work on plain strings.
-template <size_t N, typename Function,
-          typename... AdditionalNonStringValueGetters>
-using StringExpressionImpl =
-    StringExpressionImplImpl<StringValueGetter, LiteralFromIdGetter, N,
-                             Function, AdditionalNonStringValueGetters...>;
-
-// Impl class for expressions that work on literals with datatypes and language
-// tags.
-template <size_t N, typename Function,
-          typename... AdditionalNonStringValueGetters>
-using LiteralExpressionImpl =
-    StringExpressionImplImpl<LiteralValueGetterWithStrFunction,
-                             LiteralValueGetterWithoutStrFunction, N, Function,
-                             AdditionalNonStringValueGetters...>;
-
 // Lift a `Function` that takes one or multiple `std::string`s (possibly via
 // references) and returns an `Id` or `std::string` to a function that takes the
 // same number of `std::optional<std::string>` and returns `Id` or
@@ -409,7 +346,7 @@ using StrBeforeExpression =
   if (!flags.has_value() || !regex.has_value()) {
     return Id::makeUndefined();
   }
-  auto firstInvalidFlag = flags.value().find_first_not_of("imsu");
+  auto firstInvalidFlag = flags.value().find_first_not_of("imsU");
   if (firstInvalidFlag != std::string::npos) {
     return Id::makeUndefined();
   }
@@ -423,11 +360,10 @@ using StrBeforeExpression =
 };
 
 using MergeRegexPatternAndFlagsExpression =
-    StringExpressionImpl<2, decltype(mergeFlagsIntoRegex), StringValueGetter>;
+    StringExpressionImpl<2, decltype(mergeFlagsIntoRegex), LiteralFromIdGetter>;
 
 [[maybe_unused]] auto replaceImpl =
-    [](std::optional<std::string> input,
-       const std::unique_ptr<re2::RE2>& pattern,
+    [](std::optional<std::string> input, const std::shared_ptr<RE2>& pattern,
        const std::optional<std::string>& replacement) -> IdOrLiteralOrIri {
   if (!input.has_value() || !pattern || !replacement.has_value()) {
     return Id::makeUndefined();
@@ -439,7 +375,7 @@ using MergeRegexPatternAndFlagsExpression =
     return Id::makeUndefined();
   }
   const auto& repl = replacement.value();
-  re2::RE2::GlobalReplace(&in, pat, repl);
+  RE2::GlobalReplace(&in, pat, repl);
   return toLiteral(in);
 };