feat: add trace retention policy

Author: ItsNotGoodName

README.md

@@ -106,15 +106,18 @@ healthcheck:
   # Run on startup
   startup: false
 
-# Retention policy for envelopes and attachment files
+# Retention policy will delete resources that match the configured policy
 retention:
-  # Retention policy for envelopes in database
+  # Envelopes in database
   envelope_count: # (0, 100, 250, ...)
   envelope_age: # (5m, 5h45m, ...)
 
-  # Retention policy for attachment files on disk
+  # Attachment files in file store
   attachment_size: # (100 MB, 1 GB, ...)
 
+  # Traces in database
+  trace_age: 168h # 7 days (5m, 5h45m, ...)
+
 # HTTP server
 http:
   disable: false # (false, true)

config/config.go

@@ -64,6 +64,7 @@ type Raw struct {
 	RetentionEnvelopeCount  string `koanf:"retention.envelope_count"`
 	RetentionEnvelopeAge    string `koanf:"retention.envelope_age"`
 	RetentionAttachmentSize string `koanf:"retention.attachment_size"`
+	RetentionTraceAge       string `koanf:"retention.trace_age"`
 	HealthcheckURL          string `koanf:"healthcheck.url"`
 	HealthcheckInterval     string `koanf:"healthcheck.interval"`
 	HealthcheckStartup      bool   `koanf:"healthcheck.startup"`
@@ -111,13 +112,15 @@ var RawDefault = struct {
 	SMTPPort            uint16 `koanf:"smtp.port"`
 	SMTPMaxPayloadSize  string `koanf:"smtp.max_payload_size"`
 	HTTPPort            uint16 `koanf:"http.port"`
+	RetentionTraceAge   string `koanf:"retention.trace_age"`
 	// IMAPPort         uint16 `koanf:"imap.port"`
 }{
 	HealthcheckInterval: "5m",
 	TimeFormat:          timeFormat12H,
 	SMTPMaxPayloadSize:  "25 MB",
 	DataDirectory:       "smtpbridge_data",
 	PythonExecutable:    "python3",
+	RetentionTraceAge:   "168h",
 	SMTPPort:            1025,
 	HTTPPort:            8080,
 	// IMAPPort:         10143,
@@ -226,13 +229,22 @@ func (p Parser) Parse(raw Raw) (Config, error) {
 			}
 			envelopeAge = &age
 		}
+		var traceAge *time.Duration
+		if raw.RetentionTraceAge != "" {
+			age, err := time.ParseDuration(raw.RetentionTraceAge)
+			if err != nil {
+				return Config{}, err
+			}
+			traceAge = &age
+		}
 
 		config = &models.Config{
 			RetentionPolicy: models.ConfigRetentionPolicy{
+				MinAge:         5 * time.Minute,
 				EnvelopeCount:  envelopeCount,
 				AttachmentSize: attachmentsSize,
 				EnvelopeAge:    envelopeAge,
-				MinAge:         5 * time.Minute,
+				TraceAge:       traceAge,
 			},
 			AuthSMTP: auth.New(
 				raw.SMTPUsername,

internal/app/app.go

@@ -354,7 +354,8 @@ func (a App) TraceList(ctx context.Context, page pagination.Page, req models.DTO
 }
 
 func (a App) TraceDrop(ctx context.Context) error {
-	return repo.TraceDrop(ctx, a.db)
+	_, err := repo.TraceDrop(ctx, a.db)
+	return err
 }
 
 func (a App) RetentionPolicyGet(ctx context.Context) models.ConfigRetentionPolicy {
@@ -385,6 +386,11 @@ func (a App) RetentionPolicyRun(ctx context.Context, tracer trace.Tracer) error
 		return err
 	}
 
+	_, err = retention.DeleteTraceByAge(ctx, tracer, a.db, a.config.RetentionPolicy)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 

internal/models/models.go

@@ -21,6 +21,7 @@ type ConfigRetentionPolicy struct {
 	EnvelopeCount  *int
 	EnvelopeAge    *time.Duration
 	AttachmentSize *int64
+	TraceAge       *time.Duration
 }
 
 func (p ConfigRetentionPolicy) EnvelopeAgeTime() time.Time {

internal/repo/trace.go

@@ -3,6 +3,7 @@ package repo
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/ItsNotGoodName/smtpbridge/internal/database"
 	. "github.com/ItsNotGoodName/smtpbridge/internal/jet/table"
@@ -103,10 +104,25 @@ func TraceList(ctx context.Context, db database.Querier, page pagination.Page, r
 	}, nil
 }
 
-func TraceDrop(ctx context.Context, db database.Querier) error {
-	_, err := Traces.
+func TraceDrop(ctx context.Context, db database.Querier) (int64, error) {
+	res, err := Traces.
 		DELETE().
 		WHERE(RawBool("1=1")).
 		ExecContext(ctx, db)
-	return err
+	if err != nil {
+		return 0, err
+	}
+
+	return res.RowsAffected()
+}
+
+func TraceTrim(ctx context.Context, db database.Querier, age time.Time) (int64, error) {
+	res, err := Traces.
+		DELETE().
+		WHERE(Traces.CreatedAt.LT(RawTimestamp(muhTypeAffinity(models.NewTime(age))))).
+		ExecContext(ctx, db)
+	if err != nil {
+		return 0, err
+	}
+	return res.RowsAffected()
 }

internal/retention/retention.go

@@ -115,3 +115,19 @@ func DeleteOrphanAttachments(ctx context.Context, tracer trace.Tracer, db databa
 		tracer.Trace(ctx, "retention.attachment.orphan.delete", trace.WithKV("count", len(atts)))
 	}
 }
+
+func DeleteTraceByAge(ctx context.Context, tracer trace.Tracer, db database.Querier, policy models.ConfigRetentionPolicy) (int64, error) {
+	if policy.TraceAge == nil {
+		return 0, nil
+	}
+
+	age := time.Now().Add(-*policy.TraceAge)
+	count, err := repo.TraceTrim(ctx, db, age)
+	if err != nil {
+		return 0, err
+	}
+
+	tracer.Trace(ctx, "retention.trace.age.delete", trace.WithKV("count", count))
+
+	return count, nil
+}

web/pages/index.templ

@@ -66,6 +66,12 @@ templ indexView(m meta.Meta, props indexViewProps) {
                       <td>{ helpers.BytesHumanize(*props.RetentionPolicy.AttachmentSize)  }</td>
                     </tr>
                   }
+                  if props.RetentionPolicy.TraceAge != nil {
+                    <tr>
+                      <th>Maximum Trace Age</th>
+                      <td>{ props.RetentionPolicy.TraceAge.String() }</td>
+                    </tr>
+                  }
                 </tbody>
               </table>
             </div>