Merge pull request #3480 from sourcenetwork/release/0.16.0

Release v0.16.0

.github/workflows/lint.yml

@@ -50,7 +50,7 @@ jobs:
           # Required: the version of golangci-lint is required.
           # Note: The version should not pick the patch version as the latest patch
           #  version is what will always be used.
-          version: v1.61
+          version: v1.64
 
           # Optional: working directory, useful for monorepos or if we wanted to run this
           #  on a non-root directory.

.github/workflows/release.yml

@@ -56,7 +56,7 @@ jobs:
         uses: goreleaser/goreleaser-action@v6
         with:
           distribution: goreleaser-pro
-          version: 2.3.2
+          version: 2.6.1
           args: release --clean --split ${{ env.flags }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -67,7 +67,7 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
         uses: actions/cache/save@v4
         with:
-          path: dist/linux_amd64_v1
+          path: dist/linux_amd64
           key: linux-${{ env.sha_short }}
 
       - name: Save cache on MacOS
@@ -81,7 +81,7 @@ jobs:
         if: matrix.os == 'windows-latest'
         uses: actions/cache/save@v4
         with:
-          path: dist/windows_amd64_v1
+          path: dist/windows_amd64
           key: windows-${{ env.sha_short }}
           enableCrossOsArchive: true
 
@@ -113,7 +113,7 @@ jobs:
         id: restore-linux
         uses: actions/cache/restore@v4
         with:
-          path: dist/linux_amd64_v1
+          path: dist/linux_amd64
           key: linux-${{ env.sha_short }}
           fail-on-cache-miss: true
 
@@ -129,7 +129,7 @@ jobs:
         id: restore-windows
         uses: actions/cache/restore@v4
         with:
-          path: dist/windows_amd64_v1
+          path: dist/windows_amd64
           key: windows-${{ env.sha_short }}
           fail-on-cache-miss: true
           enableCrossOsArchive: true

.github/workflows/test-coverage.yml

@@ -37,6 +37,8 @@ env:
 
   DEFRA_BADGER_ENCRYPTION: false
 
+  DEFRA_VECTOR_EMBEDDING: false
+
   DEFRA_MUTATION_TYPE: collection-save
   DEFRA_LENS_TYPE: wasm-time
   DEFRA_ACP_TYPE: local
@@ -72,6 +74,7 @@ jobs:
       DEFRA_BADGER_MEMORY: ${{ matrix.database-type == 'memory' }}
       DEFRA_BADGER_FILE: ${{ matrix.database-type == 'file' }}
       DEFRA_MUTATION_TYPE: ${{ matrix.mutation-type }}
+      DEFRA_VECTOR_EMBEDDING: true
 
     steps:
       - name: Checkout code into the directory
@@ -80,6 +83,15 @@ jobs:
       - name: Setup defradb
         uses: ./.github/composites/setup-defradb
 
+      - name: Install Ollama
+        run: make deps:ollama
+
+      - name: Run Ollama
+        run: make ollama
+
+      - name: Pull LLM model
+        run: make ollama:nomic
+
       - name: Test coverage & save coverage report in an artifact
         uses: ./.github/composites/test-coverage-with-artifact
         with:
@@ -236,6 +248,27 @@ jobs:
           coverage-artifact-name: "coverage_encryption"
           coverage-path: coverage.txt
 
+  # This job runs the database with telemetry tests using default configuration, on linux.
+  test-telemetry:
+    name: Test telemetry job
+
+    runs-on: ubuntu-latest
+
+    env:
+      GOFLAGS: -tags=telemetry
+
+    steps:
+      - name: Checkout code into the directory
+        uses: actions/checkout@v4
+
+      - name: Setup defradb
+        uses: ./.github/composites/setup-defradb
+
+      - name: Test coverage & save coverage report in an artifact
+        uses: ./.github/composites/test-coverage-with-artifact
+        with:
+          coverage-artifact-name: "coverage_telemetry"
+          coverage-path: coverage.txt
 
   ## This job gathers all the coverage reports and uploads them to code-cov
   upload-coverage:
@@ -248,6 +281,7 @@ jobs:
       - test-lens         #  2 test(s)
       - test-view         #  1 test(s)
       - test-encryption   #  1 test(s)
+      - test-telemetry    #  1 test(s)
 
     # Important to know:
     # - We didn't use `if: always()` here, so this job doesn't run if we manually canceled.

.goreleaser.yaml

@@ -6,7 +6,8 @@ before:
     - make deps:playground
 
 builds:
-  - id: "defradb"
+  # Minimal build with no extras
+  - id: "defradb_lite"
     main: ./cmd/defradb
     goos:
       - linux
@@ -15,11 +16,11 @@ builds:
     goarch:
       - amd64
       - arm64
-  # A build with the playground included.
-  - id: "defradb_playground"
+  # Default build with playground and telemetry
+  - id: "defradb"
     main: ./cmd/defradb
     flags:
-      - -tags=playground
+      - -tags=playground,telemetry
     goos:
       - linux
       - windows
@@ -32,20 +33,20 @@ partial:
   by: target
 
 archives:
-  - id: defradb_playground
+  - id: defradb_lite
     builds:
-      - defradb_playground
-    format: binary
+      - defradb_lite
+    formats: [binary]
     # this name template makes the OS and Arch compatible with the results of `uname`.
     name_template: >-
-      {{ .Binary }}_playground_{{ .Version }}_{{ .Os }}_
+      {{ .Binary }}_lite_{{ .Version }}_{{ .Os }}_
       {{- if eq .Arch "amd64" }}x86_64
       {{- else }}{{ .Arch }}{{ end }}
       {{- if .Arm }}v{{ .Arm }}{{ end }}
   - id: defradb
     builds:
       - defradb
-    format: binary
+    formats: [binary]
     # this name template makes the OS and Arch compatible with the results of `uname`.
     name_template: >-
       {{ .Binary }}_{{ .Version }}_{{ .Os }}_

CHANGELOG.md

@@ -1,3 +1,63 @@
+<a name="v0.16.0"></a>
+## [v0.16.0](https://github.com/sourcenetwork/defradb/compare/v0.15.0...v0.16.0)
+
+> 2025-02-21
+
+DefraDB v0.16 is a major pre-production release. Until the stable version 1.0 is reached, the SemVer minor patch number will denote notable releases, which will give the project freedom to experiment and explore potentially breaking changes.
+
+To get a full outline of the changes, we invite you to review the official changelog below. This release does include a Breaking Change to existing v0.14.x databases. If you need help migrating an existing deployment, reach out at [hello@source.network](mailto:hello@source.network) or join our Discord at https://discord.gg/w7jYQVJ/.
+
+### Features
+
+* Add cosine similarity query ([#3464](https://github.com/sourcenetwork/defradb/issues/3464))
+* Add contraints and embedding directives ([#3405](https://github.com/sourcenetwork/defradb/issues/3405))
+* Add open telemetry metrics and traces ([#3404](https://github.com/sourcenetwork/defradb/issues/3404))
+* Enable secondary index for compound filter conditions ([#3417](https://github.com/sourcenetwork/defradb/issues/3417))
+* Add special encoding of json array elements ([#3368](https://github.com/sourcenetwork/defradb/issues/3368))
+* P2P ACP ([#3317](https://github.com/sourcenetwork/defradb/issues/3317))
+* Multiple schema file support in schema add ([#3352](https://github.com/sourcenetwork/defradb/issues/3352))
+* Purge ACP state on DB purge ([#3363](https://github.com/sourcenetwork/defradb/issues/3363))
+* Sec. index on json ([#3330](https://github.com/sourcenetwork/defradb/issues/3330))
+* Add 'keyring list' command to CLI ([#3331](https://github.com/sourcenetwork/defradb/issues/3331))
+
+### Fixes
+
+* Consistent version query semantics ([#3477](https://github.com/sourcenetwork/defradb/issues/3477))
+* Updating of document via cli ([#3474](https://github.com/sourcenetwork/defradb/issues/3474))
+* Make --no-encryption flag work without keyring ([#3466](https://github.com/sourcenetwork/defradb/issues/3466))
+* Ensure no ACP doc check when action is node specific ([#3387](https://github.com/sourcenetwork/defradb/issues/3387))
+* Maintain indexes across schema versions ([#3367](https://github.com/sourcenetwork/defradb/issues/3367))
+* Maintain indexes across schema versions ([#3366](https://github.com/sourcenetwork/defradb/issues/3366))
+
+### Performance
+
+* Remove compound condition from 1-1 validation filter ([#3390](https://github.com/sourcenetwork/defradb/issues/3390))
+
+### Chore
+
+* Bump to GoLang v1.23 ([#3444](https://github.com/sourcenetwork/defradb/issues/3444))
+* Bump acp_core version ([#3392](https://github.com/sourcenetwork/defradb/issues/3392))
+* Import defradb/errors for Join ([#3343](https://github.com/sourcenetwork/defradb/issues/3343))
+
+### Bot
+
+* Update dependencies (bulk dependabot PRs) 2025-02-17 ([#3458](https://github.com/sourcenetwork/defradb/issues/3458))
+* Update dependencies (bulk dependabot PRs) 2025-02-10 ([#3441](https://github.com/sourcenetwork/defradb/issues/3441))
+* Bump github.com/cometbft/cometbft from 0.38.12 to 0.38.17 ([#3423](https://github.com/sourcenetwork/defradb/issues/3423))
+* Bump github.com/getkin/kin-openapi from 0.128.0 to 0.129.0 ([#3418](https://github.com/sourcenetwork/defradb/issues/3418))
+* Bump github.com/golang/glog from 1.2.3 to 1.2.4 ([#3425](https://github.com/sourcenetwork/defradb/issues/3425))
+* Update dependencies (bulk dependabot PRs) 03-02-2025 ([#3424](https://github.com/sourcenetwork/defradb/issues/3424))
+* Update dependencies (bulk dependabot PRs) 27-01-2025 ([#3415](https://github.com/sourcenetwork/defradb/issues/3415))
+* Update dependencies (bulk dependabot PRs) 20-01-2025 ([#3397](https://github.com/sourcenetwork/defradb/issues/3397))
+* Update dependencies (bulk dependabot PRs) 14-01-2025 ([#3382](https://github.com/sourcenetwork/defradb/issues/3382))
+* Update dependencies (bulk dependabot PRs) 13-01-2025 ([#3379](https://github.com/sourcenetwork/defradb/issues/3379))
+* Bump vite from 6.0.6 to 6.0.7 in /playground ([#3355](https://github.com/sourcenetwork/defradb/issues/3355))
+* Bump github.com/ipfs/boxo from 0.24.3 to 0.26.0 ([#3336](https://github.com/sourcenetwork/defradb/issues/3336))
+* Bump google.golang.org/grpc from 1.68.1 to 1.69.2 ([#3339](https://github.com/sourcenetwork/defradb/issues/3339))
+* Bump [@typescript](https://github.com/typescript)-eslint/eslint-plugin from 8.18.2 to 8.19.0 in /playground ([#3348](https://github.com/sourcenetwork/defradb/issues/3348))
+* Update dependencies (bulk dependabot PRs) 30-12-2024 ([#3347](https://github.com/sourcenetwork/defradb/issues/3347))
+* Update dependencies (bulk dependabot PRs) 18-12-2024 ([#3333](https://github.com/sourcenetwork/defradb/issues/3333))
+
 <a name="v0.15.0"></a>
 ## [v0.15.0](https://github.com/sourcenetwork/defradb/compare/v0.14.0...v0.15.0)
 

Makefile

@@ -124,7 +124,7 @@ client\:add-schema:
 
 .PHONY: deps\:lint-go
 deps\:lint-go:
-	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.61
+	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64
 
 .PHONY: deps\:lint-yaml
 deps\:lint-yaml:
@@ -170,6 +170,16 @@ deps\:mocks:
 deps\:playground:
 	cd $(PLAYGROUND_DIRECTORY) && npm install --legacy-peer-deps && npm run build
 
+.PHONY: deps\:ollama
+deps\:ollama:
+ifeq ($(OS_GENERAL),Linux)
+	curl -fsSL https://ollama.com/install.sh | sh
+else ifeq ($(OS_GENERAL),Darwin)
+	brew install ollama
+else
+	@echo "Makefile installation of Ollama is not supported for your system. Please install manually."
+endif
+
 .PHONY: deps
 deps:
 	@$(MAKE) deps:modules && \
@@ -185,6 +195,17 @@ mocks:
 	@$(MAKE) deps:mocks
 	mockery --config="tools/configs/mockery.yaml"
 
+.PHONY: ollama
+ollama:
+# run ollama in the background
+	nohup ollama serve > ollama.log 2>&1 &
+
+.PHONY: ollama\:nomic
+ollama\:nomic:
+# make sure ollama is running before continuing
+	time curl --retry 5 --retry-connrefused --retry-delay 0 -sf http://localhost:11434
+	ollama pull nomic-embed-text
+
 .PHONY: dev\:start
 dev\:start:
 	@$(MAKE) build

README.md

@@ -34,6 +34,7 @@ Read the documentation on [docs.source.network](https://docs.source.network/).
    * [Access Control System](#access-control-system)
    * [Supporting CORS](#supporting-cors)
    * [Backing up and restoring](#backing-up-and-restoring)
+   * [Telemetry](#telemetry)
    * [Community](#community)
    * [Licensing](#licensing)
    * [Contributors](#contributors)
@@ -473,6 +474,14 @@ To restore the data, run the following command:
 defradb client backup import path/to/backup.json
 ```
 
+## Telemetry
+
+DefraDB has no telemetry reporting by default. To enable OpenTelemetry in DefraDB you must build with the `telemetry` tag set. To configure the HTTP exporters use the environment variables in the links below.
+
+[Metric exporter documentation](https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp)
+
+[Trace exporter documentation](https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp)
+
 ## Community
 
 Discuss on [Discord](https://discord.gg/w7jYQVJ) or [Github Discussions](https://github.com/sourcenetwork/defradb/discussions). The Source project is on [Twitter](https://twitter.com/sourcenetwrk).

acp/acp.go

@@ -43,6 +43,10 @@ type ACP interface {
 	// Close closes the resources in use by acp.
 	Close() error
 
+	// ResetState purges the entire ACP state.
+	// Resetting will close the ACP engine, purge the state, then restart it
+	ResetState(ctx context.Context) error
+
 	// AddPolicy attempts to add the given policy. Detects the format of the policy automatically
 	// by assuming YAML format if JSON validation fails. Upon success a policyID is returned,
 	// otherwise returns error.

acp/acp_local.go

@@ -12,7 +12,8 @@ package acp
 
 import (
 	"context"
-	"errors"
+
+	"os"
 
 	protoTypes "github.com/cosmos/gogoproto/types"
 	"github.com/sourcenetwork/acp_core/pkg/auth"
@@ -22,6 +23,7 @@ import (
 	"github.com/sourcenetwork/immutable"
 
 	"github.com/sourcenetwork/defradb/acp/identity"
+	"github.com/sourcenetwork/defradb/errors"
 )
 
 const localACPStoreName = "local_acp"
@@ -31,6 +33,7 @@ type ACPLocal struct {
 	pathToStore immutable.Option[string]
 	engine      types.ACPEngineServer
 	manager     runtime.RuntimeManager
+	closed      bool
 }
 
 var _ sourceHubClient = (*ACPLocal)(nil)
@@ -82,6 +85,7 @@ func (l *ACPLocal) Start(ctx context.Context) error {
 	var opts []runtime.Opt
 	var storeLocation string
 
+	l.closed = false
 	if !l.pathToStore.HasValue() { // Use a non-persistent, i.e. in memory store.
 		storeLocation = "in-memory"
 		opts = append(opts, runtime.WithMemKV())
@@ -102,7 +106,48 @@ func (l *ACPLocal) Start(ctx context.Context) error {
 }
 
 func (l *ACPLocal) Close() error {
-	return l.manager.Terminate()
+	if !l.closed {
+		err := l.manager.Terminate()
+		if err != nil {
+			return err
+		}
+		l.closed = true
+	}
+	return nil
+}
+
+func (l *ACPLocal) ResetState(ctx context.Context) error {
+	err := l.Close()
+	if err != nil {
+		return err
+	}
+
+	// delete state (applicable to persistent store)
+	if l.pathToStore.HasValue() {
+		storeLocation := l.pathToStore.Value()
+		path := storeLocation + "/" + localACPStoreName
+		info, err := os.Stat(path)
+		if os.IsNotExist(err) {
+			return errors.Join(ErrACPResetState, err)
+		} else if err != nil {
+			return errors.Join(ErrACPResetState, err)
+		}
+
+		if info.IsDir() {
+			// remove dir
+			if err := os.RemoveAll(path); err != nil {
+				return errors.Join(ErrACPResetState, err)
+			}
+		} else {
+			// remove file
+			if err := os.Remove(path); err != nil {
+				return errors.Join(ErrACPResetState, err)
+			}
+		}
+	}
+
+	// Start again
+	return l.Start(ctx)
 }
 
 func (l *ACPLocal) AddPolicy(