From 5f006ef892106f084cd8a1ee1e47ba8954536323 Mon Sep 17 00:00:00 2001 From: James Pickett Date: Fri, 29 Mar 2024 15:24:17 -0700 Subject: [PATCH 1/7] post PR with notes on windows test issues --- pkg/osquery/runtime/osqueryinstance.go | 7 ++ pkg/osquery/runtime/runtime_all_test.go | 131 ++++++++++++++++++++++++ pkg/osquery/runtime/runtime_test.go | 106 ------------------- 3 files changed, 138 insertions(+), 106 deletions(-) create mode 100644 pkg/osquery/runtime/runtime_all_test.go diff --git a/pkg/osquery/runtime/osqueryinstance.go b/pkg/osquery/runtime/osqueryinstance.go index 632a40a0b..149b309c9 100644 --- a/pkg/osquery/runtime/osqueryinstance.go +++ b/pkg/osquery/runtime/osqueryinstance.go @@ -592,6 +592,13 @@ func (o *OsqueryInstance) StartOsqueryExtensionManagerServer(name string, socket "err", err, "extension_name", name, ) + // PR Note, remove after test working + // On windows, during test this always returns error + // "status 1 registering extension: Failed adding registry: SQLITE_CANTOPEN" + // the source of the error is in osquery-go code + // https://github.com/osquery/osquery-go/blob/master/server.go#L217 + // coming out of + // https://github.com/osquery/osquery-go/blob/master/gen/osquery/osquery.go#L2225 return fmt.Errorf("running extension server: %w", err) } return errors.New("extension manager server exited") diff --git a/pkg/osquery/runtime/runtime_all_test.go b/pkg/osquery/runtime/runtime_all_test.go new file mode 100644 index 000000000..58c7b7488 --- /dev/null +++ b/pkg/osquery/runtime/runtime_all_test.go @@ -0,0 +1,131 @@ +package runtime + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/apache/thrift/lib/go/thrift" + "github.com/kolide/kit/fsutil" + "github.com/kolide/launcher/ee/agent/storage" + storageci "github.com/kolide/launcher/ee/agent/storage/ci" + typesMocks "github.com/kolide/launcher/ee/agent/types/mocks" + "github.com/kolide/launcher/pkg/backoff" + "github.com/kolide/launcher/pkg/log/multislogger" + "github.com/kolide/launcher/pkg/osquery/runtime/history" + "github.com/kolide/launcher/pkg/packaging" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +var testOsqueryBinaryDirectory string + +// TestMain overrides the default test main function. This allows us to share setup/teardown. +func TestMain(m *testing.M) { + binDirectory, rmBinDirectory, err := osqueryTempDir() + if err != nil { + fmt.Println("Failed to make temp dir for test binaries") + os.Exit(1) + } + defer rmBinDirectory() + + s, err := storageci.NewStore(nil, multislogger.NewNopLogger(), storage.OsqueryHistoryInstanceStore.String()) + if err != nil { + fmt.Println("Failed to make new store") + os.Exit(1) + } + if err := history.InitHistory(s); err != nil { + fmt.Println("Failed to init history") + os.Exit(1) + } + + testOsqueryBinaryDirectory = filepath.Join(binDirectory, "osqueryd") + + thrift.ServerConnectivityCheckInterval = 100 * time.Millisecond + + if err := downloadOsqueryInBinDir(binDirectory); err != nil { + fmt.Printf("Failed to download osquery: %v\n", err) + os.Exit(1) + } + + // Run the tests! + retCode := m.Run() + os.Exit(retCode) +} + +// downloadOsqueryInBinDir downloads osqueryd. This allows the test +// suite to run on hosts lacking osqueryd. We could consider moving this into a deps step. +func downloadOsqueryInBinDir(binDirectory string) error { + target := packaging.Target{} + if err := target.PlatformFromString(runtime.GOOS); err != nil { + return fmt.Errorf("Error parsing platform: %s: %w", runtime.GOOS, err) + } + + outputFile := filepath.Join(binDirectory, "osqueryd") + if runtime.GOOS == "windows" { + outputFile += ".exe" + } + + cacheDir := "/tmp" + if runtime.GOOS == "windows" { + cacheDir = os.Getenv("TEMP") + } + + path, err := packaging.FetchBinary(context.TODO(), cacheDir, "osqueryd", target.PlatformBinaryName("osqueryd"), "stable", target) + if err != nil { + return fmt.Errorf("An error occurred fetching the osqueryd binary: %w", err) + } + + if err := fsutil.CopyFile(path, outputFile); err != nil { + return fmt.Errorf("Couldn't copy file to %s: %w", outputFile, err) + } + + return nil +} + +// waitHealthy expects the instance to be healthy within 30 seconds, or else +// fatals the test +func waitHealthy(t *testing.T, runner *Runner) { + require.NoError(t, backoff.WaitFor(func() error { + if runner.Healthy() == nil { + return nil + } + return fmt.Errorf("instance not healthy") + }, 120*time.Second, 5*time.Second)) +} + +func TestSimplePath(t *testing.T) { + t.Parallel() + rootDirectory, rmRootDirectory, err := osqueryTempDir() + require.NoError(t, err) + defer rmRootDirectory() + + k := typesMocks.NewKnapsack(t) + k.On("OsqueryHealthcheckStartupDelay").Return(0 * time.Second).Maybe() + k.On("WatchdogEnabled").Return(false) + k.On("RegisterChangeObserver", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) + k.On("Slogger").Return(multislogger.NewNopLogger()) + k.On("PinnedOsquerydVersion").Return("") + + runner := New( + k, + WithKnapsack(k), + WithRootDirectory(rootDirectory), + WithOsquerydBinary(testOsqueryBinaryDirectory), + ) + + go runner.Run() + + // never gets health on windows + waitHealthy(t, runner) + + require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be added to instance stats on start up") + require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be added to instance stats on start up") + + require.NoError(t, runner.Shutdown()) +} diff --git a/pkg/osquery/runtime/runtime_test.go b/pkg/osquery/runtime/runtime_test.go index 1dd1d3be0..801991ce2 100644 --- a/pkg/osquery/runtime/runtime_test.go +++ b/pkg/osquery/runtime/runtime_test.go @@ -4,29 +4,20 @@ package runtime import ( - "context" "errors" "fmt" "log/slog" "os" "os/exec" "path/filepath" - "runtime" "strings" "syscall" "testing" "time" - "github.com/apache/thrift/lib/go/thrift" - "github.com/kolide/kit/fsutil" - "github.com/kolide/kit/testutil" "github.com/kolide/launcher/ee/agent/flags/keys" - "github.com/kolide/launcher/ee/agent/storage" - storageci "github.com/kolide/launcher/ee/agent/storage/ci" typesMocks "github.com/kolide/launcher/ee/agent/types/mocks" "github.com/kolide/launcher/pkg/log/multislogger" - "github.com/kolide/launcher/pkg/osquery/runtime/history" - "github.com/kolide/launcher/pkg/packaging" "github.com/kolide/launcher/pkg/threadsafebuffer" osquery "github.com/osquery/osquery-go" @@ -35,41 +26,6 @@ import ( "github.com/stretchr/testify/require" ) -var testOsqueryBinaryDirectory string - -// TestMain overrides the default test main function. This allows us to share setup/teardown. -func TestMain(m *testing.M) { - binDirectory, rmBinDirectory, err := osqueryTempDir() - if err != nil { - fmt.Println("Failed to make temp dir for test binaries") - os.Exit(1) - } - defer rmBinDirectory() - - s, err := storageci.NewStore(nil, multislogger.NewNopLogger(), storage.OsqueryHistoryInstanceStore.String()) - if err != nil { - fmt.Println("Failed to make new store") - os.Exit(1) - } - if err := history.InitHistory(s); err != nil { - fmt.Println("Failed to init history") - os.Exit(1) - } - - testOsqueryBinaryDirectory = filepath.Join(binDirectory, "osqueryd") - - thrift.ServerConnectivityCheckInterval = 100 * time.Millisecond - - if err := downloadOsqueryInBinDir(binDirectory); err != nil { - fmt.Printf("Failed to download osquery: %v\n", err) - os.Exit(1) - } - - // Run the tests! - retCode := m.Run() - os.Exit(retCode) -} - // getBinDir finds the directory of the currently running binary (where we will // look for the osquery extension) func getBinDir() (string, error) { @@ -266,29 +222,6 @@ func TestCreateOsqueryCommand_SetsDisabledWatchdogSettingsAppropriately(t *testi k.AssertExpectations(t) } -// downloadOsqueryInBinDir downloads osqueryd. This allows the test -// suite to run on hosts lacking osqueryd. We could consider moving this into a deps step. -func downloadOsqueryInBinDir(binDirectory string) error { - target := packaging.Target{} - if err := target.PlatformFromString(runtime.GOOS); err != nil { - return fmt.Errorf("Error parsing platform: %s: %w", runtime.GOOS, err) - } - - outputFile := filepath.Join(binDirectory, "osqueryd") - cacheDir := "/tmp" - - path, err := packaging.FetchBinary(context.TODO(), cacheDir, "osqueryd", target.PlatformBinaryName("osqueryd"), "stable", target) - if err != nil { - return fmt.Errorf("An error occurred fetching the osqueryd binary: %w", err) - } - - if err := fsutil.CopyFile(path, outputFile); err != nil { - return fmt.Errorf("Couldn't copy file to %s: %w", outputFile, err) - } - - return nil -} - func TestBadBinaryPath(t *testing.T) { t.Parallel() rootDirectory, rmRootDirectory, err := osqueryTempDir() @@ -429,45 +362,6 @@ func TestFlagsChanged(t *testing.T) { runner.Interrupt(errors.New("test error")) } -// waitHealthy expects the instance to be healthy within 30 seconds, or else -// fatals the test -func waitHealthy(t *testing.T, runner *Runner) { - testutil.FatalAfterFunc(t, 30*time.Second, func() { - for runner.Healthy() != nil { - time.Sleep(500 * time.Millisecond) - } - }) -} - -func TestSimplePath(t *testing.T) { - t.Parallel() - rootDirectory, rmRootDirectory, err := osqueryTempDir() - require.NoError(t, err) - defer rmRootDirectory() - - k := typesMocks.NewKnapsack(t) - k.On("OsqueryHealthcheckStartupDelay").Return(0 * time.Second).Maybe() - k.On("WatchdogEnabled").Return(false) - k.On("RegisterChangeObserver", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) - k.On("Slogger").Return(multislogger.NewNopLogger()) - k.On("PinnedOsquerydVersion").Return("") - - runner := New( - k, - WithKnapsack(k), - WithRootDirectory(rootDirectory), - WithOsquerydBinary(testOsqueryBinaryDirectory), - ) - go runner.Run() - - waitHealthy(t, runner) - - require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be added to instance stats on start up") - require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be added to instance stats on start up") - - require.NoError(t, runner.Shutdown()) -} - func TestMultipleShutdowns(t *testing.T) { t.Parallel() rootDirectory, rmRootDirectory, err := osqueryTempDir() From eea89affc915bf88461c3b59a2698368e27add34 Mon Sep 17 00:00:00 2001 From: James Pickett Date: Mon, 1 Apr 2024 10:08:35 -0700 Subject: [PATCH 2/7] enable tests on windows, move posix tests to own file --- pkg/osquery/runtime/osqueryinstance.go | 7 - pkg/osquery/runtime/runtime_all_test.go | 131 --------- pkg/osquery/runtime/runtime_posix_test.go | 210 +++++++++++++++ pkg/osquery/runtime/runtime_test.go | 308 ++++++++-------------- 4 files changed, 327 insertions(+), 329 deletions(-) delete mode 100644 pkg/osquery/runtime/runtime_all_test.go create mode 100644 pkg/osquery/runtime/runtime_posix_test.go diff --git a/pkg/osquery/runtime/osqueryinstance.go b/pkg/osquery/runtime/osqueryinstance.go index 149b309c9..632a40a0b 100644 --- a/pkg/osquery/runtime/osqueryinstance.go +++ b/pkg/osquery/runtime/osqueryinstance.go @@ -592,13 +592,6 @@ func (o *OsqueryInstance) StartOsqueryExtensionManagerServer(name string, socket "err", err, "extension_name", name, ) - // PR Note, remove after test working - // On windows, during test this always returns error - // "status 1 registering extension: Failed adding registry: SQLITE_CANTOPEN" - // the source of the error is in osquery-go code - // https://github.com/osquery/osquery-go/blob/master/server.go#L217 - // coming out of - // https://github.com/osquery/osquery-go/blob/master/gen/osquery/osquery.go#L2225 return fmt.Errorf("running extension server: %w", err) } return errors.New("extension manager server exited") diff --git a/pkg/osquery/runtime/runtime_all_test.go b/pkg/osquery/runtime/runtime_all_test.go deleted file mode 100644 index 58c7b7488..000000000 --- a/pkg/osquery/runtime/runtime_all_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package runtime - -import ( - "context" - "fmt" - "os" - "path/filepath" - "runtime" - "testing" - "time" - - "github.com/apache/thrift/lib/go/thrift" - "github.com/kolide/kit/fsutil" - "github.com/kolide/launcher/ee/agent/storage" - storageci "github.com/kolide/launcher/ee/agent/storage/ci" - typesMocks "github.com/kolide/launcher/ee/agent/types/mocks" - "github.com/kolide/launcher/pkg/backoff" - "github.com/kolide/launcher/pkg/log/multislogger" - "github.com/kolide/launcher/pkg/osquery/runtime/history" - "github.com/kolide/launcher/pkg/packaging" - - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" -) - -var testOsqueryBinaryDirectory string - -// TestMain overrides the default test main function. This allows us to share setup/teardown. -func TestMain(m *testing.M) { - binDirectory, rmBinDirectory, err := osqueryTempDir() - if err != nil { - fmt.Println("Failed to make temp dir for test binaries") - os.Exit(1) - } - defer rmBinDirectory() - - s, err := storageci.NewStore(nil, multislogger.NewNopLogger(), storage.OsqueryHistoryInstanceStore.String()) - if err != nil { - fmt.Println("Failed to make new store") - os.Exit(1) - } - if err := history.InitHistory(s); err != nil { - fmt.Println("Failed to init history") - os.Exit(1) - } - - testOsqueryBinaryDirectory = filepath.Join(binDirectory, "osqueryd") - - thrift.ServerConnectivityCheckInterval = 100 * time.Millisecond - - if err := downloadOsqueryInBinDir(binDirectory); err != nil { - fmt.Printf("Failed to download osquery: %v\n", err) - os.Exit(1) - } - - // Run the tests! - retCode := m.Run() - os.Exit(retCode) -} - -// downloadOsqueryInBinDir downloads osqueryd. This allows the test -// suite to run on hosts lacking osqueryd. We could consider moving this into a deps step. -func downloadOsqueryInBinDir(binDirectory string) error { - target := packaging.Target{} - if err := target.PlatformFromString(runtime.GOOS); err != nil { - return fmt.Errorf("Error parsing platform: %s: %w", runtime.GOOS, err) - } - - outputFile := filepath.Join(binDirectory, "osqueryd") - if runtime.GOOS == "windows" { - outputFile += ".exe" - } - - cacheDir := "/tmp" - if runtime.GOOS == "windows" { - cacheDir = os.Getenv("TEMP") - } - - path, err := packaging.FetchBinary(context.TODO(), cacheDir, "osqueryd", target.PlatformBinaryName("osqueryd"), "stable", target) - if err != nil { - return fmt.Errorf("An error occurred fetching the osqueryd binary: %w", err) - } - - if err := fsutil.CopyFile(path, outputFile); err != nil { - return fmt.Errorf("Couldn't copy file to %s: %w", outputFile, err) - } - - return nil -} - -// waitHealthy expects the instance to be healthy within 30 seconds, or else -// fatals the test -func waitHealthy(t *testing.T, runner *Runner) { - require.NoError(t, backoff.WaitFor(func() error { - if runner.Healthy() == nil { - return nil - } - return fmt.Errorf("instance not healthy") - }, 120*time.Second, 5*time.Second)) -} - -func TestSimplePath(t *testing.T) { - t.Parallel() - rootDirectory, rmRootDirectory, err := osqueryTempDir() - require.NoError(t, err) - defer rmRootDirectory() - - k := typesMocks.NewKnapsack(t) - k.On("OsqueryHealthcheckStartupDelay").Return(0 * time.Second).Maybe() - k.On("WatchdogEnabled").Return(false) - k.On("RegisterChangeObserver", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) - k.On("Slogger").Return(multislogger.NewNopLogger()) - k.On("PinnedOsquerydVersion").Return("") - - runner := New( - k, - WithKnapsack(k), - WithRootDirectory(rootDirectory), - WithOsquerydBinary(testOsqueryBinaryDirectory), - ) - - go runner.Run() - - // never gets health on windows - waitHealthy(t, runner) - - require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be added to instance stats on start up") - require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be added to instance stats on start up") - - require.NoError(t, runner.Shutdown()) -} diff --git a/pkg/osquery/runtime/runtime_posix_test.go b/pkg/osquery/runtime/runtime_posix_test.go new file mode 100644 index 000000000..f75342040 --- /dev/null +++ b/pkg/osquery/runtime/runtime_posix_test.go @@ -0,0 +1,210 @@ +//go:build !windows +// +build !windows + +package runtime + +import ( + "fmt" + "log/slog" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" + + typesMocks "github.com/kolide/launcher/ee/agent/types/mocks" + "github.com/kolide/launcher/pkg/log/multislogger" + "github.com/kolide/launcher/pkg/threadsafebuffer" + "github.com/osquery/osquery-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestRestart(t *testing.T) { + t.Parallel() + runner, teardown := setupOsqueryInstanceForTests(t) + defer teardown() + + previousStats := runner.instance.stats + + require.NoError(t, runner.Restart()) + waitHealthy(t, runner) + + require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be set on latest instance stats after restart") + require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be set on latest instance stats after restart") + + require.NotEmpty(t, previousStats.ExitTime, "exit time should be set on last instance stats when restarted") + require.NotEmpty(t, previousStats.Error, "stats instance should have an error on restart") + + previousStats = runner.instance.stats + + require.NoError(t, runner.Restart()) + waitHealthy(t, runner) + + require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be added to latest instance stats after restart") + require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be added to latest instance stats after restart") + + require.NotEmpty(t, previousStats.ExitTime, "exit time should be set on instance stats when restarted") + require.NotEmpty(t, previousStats.Error, "stats instance should have an error on restart") +} + +// TestExtensionIsCleanedUp tests that the osquery extension cleans +// itself up. Unfortunately, this test has proved very flakey on +// circle-ci, but just fine on laptops. +func TestExtensionIsCleanedUp(t *testing.T) { + t.Skip("https://github.com/kolide/launcher/issues/478") + t.Parallel() + + runner, teardown := setupOsqueryInstanceForTests(t) + defer teardown() + + osqueryPID := runner.instance.cmd.Process.Pid + + pgid, err := syscall.Getpgid(osqueryPID) + require.NoError(t, err) + require.Equal(t, pgid, osqueryPID, "pgid must be set") + + require.NoError(t, err) + + // kill the current osquery process but not the extension + err = syscall.Kill(osqueryPID, syscall.SIGKILL) + require.NoError(t, err) + + // We need to (a) let the runner restart osquery, and (b) wait for + // the extension to die. Both of these may take up to 30s. We'll + // start a clock, wait for the respawn, and after 32s, test that the + // extension process is no longer running. See + // https://github.com/kolide/launcher/pull/342 and associated for + // background. + timer1 := time.NewTimer(35 * time.Second) + + // Wait for osquery to respawn + waitHealthy(t, runner) + + // Ensure we've waited at least 32s + <-timer1.C +} + +func TestOsquerySlowStart(t *testing.T) { + t.Parallel() + rootDirectory, rmRootDirectory, err := osqueryTempDir() + require.NoError(t, err) + defer rmRootDirectory() + + var logBytes threadsafebuffer.ThreadSafeBuffer + + k := typesMocks.NewKnapsack(t) + k.On("OsqueryHealthcheckStartupDelay").Return(0 * time.Second).Maybe() + k.On("WatchdogEnabled").Return(false) + k.On("RegisterChangeObserver", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) + slogger := multislogger.New(slog.NewJSONHandler(&logBytes, &slog.HandlerOptions{Level: slog.LevelDebug})) + k.On("Slogger").Return(slogger.Logger) + k.On("PinnedOsquerydVersion").Return("") + + runner := New( + k, + WithKnapsack(k), + WithRootDirectory(rootDirectory), + WithOsquerydBinary(testOsqueryBinaryDirectory), + WithSlogger(slogger.Logger), + WithStartFunc(func(cmd *exec.Cmd) error { + err := cmd.Start() + if err != nil { + return fmt.Errorf("unexpected error starting command: %w", err) + } + // suspend the process right away + cmd.Process.Signal(syscall.SIGTSTP) + go func() { + // wait a while before resuming the process + time.Sleep(3 * time.Second) + cmd.Process.Signal(syscall.SIGCONT) + }() + return nil + }), + ) + go runner.Run() + waitHealthy(t, runner) + + // ensure that we actually had to wait on the socket + require.Contains(t, logBytes.String(), "osquery extension socket not created yet") + require.NoError(t, runner.Shutdown()) +} + +func TestExtensionSocketPath(t *testing.T) { + t.Parallel() + + rootDirectory, rmRootDirectory, err := osqueryTempDir() + require.NoError(t, err) + defer rmRootDirectory() + + k := typesMocks.NewKnapsack(t) + k.On("OsqueryHealthcheckStartupDelay").Return(0 * time.Second).Maybe() + k.On("WatchdogEnabled").Return(false) + k.On("RegisterChangeObserver", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) + k.On("Slogger").Return(multislogger.NewNopLogger()) + k.On("PinnedOsquerydVersion").Return("") + + extensionSocketPath := filepath.Join(rootDirectory, "sock") + runner := New( + k, + WithKnapsack(k), + WithRootDirectory(rootDirectory), + WithExtensionSocketPath(extensionSocketPath), + WithOsquerydBinary(testOsqueryBinaryDirectory), + ) + go runner.Run() + + waitHealthy(t, runner) + + // wait for the launcher-provided extension to register + time.Sleep(2 * time.Second) + + client, err := osquery.NewClient(extensionSocketPath, 5*time.Second, osquery.DefaultWaitTime(1*time.Second), osquery.MaxWaitTime(1*time.Minute)) + require.NoError(t, err) + defer client.Close() + + resp, err := client.Query("select * from launcher_gc_info") + require.NoError(t, err) + assert.Equal(t, int32(0), resp.Status.Code) + assert.Equal(t, "OK", resp.Status.Message) + + require.NoError(t, runner.Shutdown()) +} + +// sets up an osquery instance with a running extension to be used in tests. +func setupOsqueryInstanceForTests(t *testing.T) (runner *Runner, teardown func()) { + rootDirectory, rmRootDirectory, err := osqueryTempDir() + require.NoError(t, err) + + k := typesMocks.NewKnapsack(t) + k.On("OsqueryHealthcheckStartupDelay").Return(0 * time.Second).Maybe() + k.On("WatchdogEnabled").Return(true) + k.On("WatchdogMemoryLimitMB").Return(150) + k.On("WatchdogUtilizationLimitPercent").Return(20) + k.On("WatchdogDelaySec").Return(120) + k.On("RegisterChangeObserver", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Maybe() + k.On("Slogger").Return(multislogger.NewNopLogger()) + k.On("PinnedOsquerydVersion").Return("") + + runner = New( + k, + WithKnapsack(k), + WithRootDirectory(rootDirectory), + WithOsquerydBinary(testOsqueryBinaryDirectory), + ) + go runner.Run() + waitHealthy(t, runner) + + osqueryPID := runner.instance.cmd.Process.Pid + + pgid, err := syscall.Getpgid(osqueryPID) + require.NoError(t, err) + require.Equal(t, pgid, osqueryPID) + + teardown = func() { + defer rmRootDirectory() + require.NoError(t, runner.Shutdown()) + } + return runner, teardown +} diff --git a/pkg/osquery/runtime/runtime_test.go b/pkg/osquery/runtime/runtime_test.go index 801991ce2..a750bc04a 100644 --- a/pkg/osquery/runtime/runtime_test.go +++ b/pkg/osquery/runtime/runtime_test.go @@ -1,31 +1,70 @@ -//go:build !windows -// +build !windows - package runtime +// these tests have to be run as admin on windows + import ( + "context" "errors" "fmt" - "log/slog" "os" "os/exec" "path/filepath" + "runtime" "strings" - "syscall" "testing" "time" + "github.com/apache/thrift/lib/go/thrift" + "github.com/kolide/kit/fsutil" "github.com/kolide/launcher/ee/agent/flags/keys" + "github.com/kolide/launcher/ee/agent/storage" + storageci "github.com/kolide/launcher/ee/agent/storage/ci" typesMocks "github.com/kolide/launcher/ee/agent/types/mocks" + "github.com/kolide/launcher/pkg/backoff" "github.com/kolide/launcher/pkg/log/multislogger" - "github.com/kolide/launcher/pkg/threadsafebuffer" - osquery "github.com/osquery/osquery-go" + "github.com/kolide/launcher/pkg/osquery/runtime/history" + "github.com/kolide/launcher/pkg/packaging" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) +var testOsqueryBinaryDirectory string + +// TestMain overrides the default test main function. This allows us to share setup/teardown. +func TestMain(m *testing.M) { + binDirectory, rmBinDirectory, err := osqueryTempDir() + if err != nil { + fmt.Println("Failed to make temp dir for test binaries") + os.Exit(1) + } + defer rmBinDirectory() + + s, err := storageci.NewStore(nil, multislogger.NewNopLogger(), storage.OsqueryHistoryInstanceStore.String()) + if err != nil { + fmt.Println("Failed to make new store") + os.Exit(1) + } + if err := history.InitHistory(s); err != nil { + fmt.Println("Failed to init history") + os.Exit(1) + } + + testOsqueryBinaryDirectory = filepath.Join(binDirectory, "osqueryd") + + thrift.ServerConnectivityCheckInterval = 100 * time.Millisecond + + if err := downloadOsqueryInBinDir(binDirectory); err != nil { + fmt.Printf("Failed to download osquery: %v\n", err) + os.Exit(1) + } + + // Run the tests! + retCode := m.Run() + os.Exit(retCode) +} + // getBinDir finds the directory of the currently running binary (where we will // look for the osquery extension) func getBinDir() (string, error) { @@ -52,7 +91,12 @@ func TestCalculateOsqueryPaths(t *testing.T) { // dictated require.Equal(t, binDir, filepath.Dir(paths.pidfilePath)) require.Equal(t, binDir, filepath.Dir(paths.databasePath)) - require.Equal(t, binDir, filepath.Dir(paths.extensionSocketPath)) + + // socket path on windows includes semi-random ulid + if runtime.GOOS != "windows" { + require.Equal(t, binDir, filepath.Dir(paths.extensionSocketPath)) + } + require.Equal(t, binDir, filepath.Dir(paths.extensionAutoloadPath)) } @@ -222,6 +266,36 @@ func TestCreateOsqueryCommand_SetsDisabledWatchdogSettingsAppropriately(t *testi k.AssertExpectations(t) } +// downloadOsqueryInBinDir downloads osqueryd. This allows the test +// suite to run on hosts lacking osqueryd. We could consider moving this into a deps step. +func downloadOsqueryInBinDir(binDirectory string) error { + target := packaging.Target{} + if err := target.PlatformFromString(runtime.GOOS); err != nil { + return fmt.Errorf("Error parsing platform: %s: %w", runtime.GOOS, err) + } + + outputFile := filepath.Join(binDirectory, "osqueryd") + if runtime.GOOS == "windows" { + outputFile += ".exe" + } + + cacheDir := "/tmp" + if runtime.GOOS == "windows" { + cacheDir = os.Getenv("TEMP") + } + + path, err := packaging.FetchBinary(context.TODO(), cacheDir, "osqueryd", target.PlatformBinaryName("osqueryd"), "stable", target) + if err != nil { + return fmt.Errorf("An error occurred fetching the osqueryd binary: %w", err) + } + + if err := fsutil.CopyFile(path, outputFile); err != nil { + return fmt.Errorf("Couldn't copy file to %s: %w", outputFile, err) + } + + return nil +} + func TestBadBinaryPath(t *testing.T) { t.Parallel() rootDirectory, rmRootDirectory, err := osqueryTempDir() @@ -362,7 +436,18 @@ func TestFlagsChanged(t *testing.T) { runner.Interrupt(errors.New("test error")) } -func TestMultipleShutdowns(t *testing.T) { +// waitHealthy expects the instance to be healthy within 30 seconds, or else +// fatals the test +func waitHealthy(t *testing.T, runner *Runner) { + require.NoError(t, backoff.WaitFor(func() error { + if runner.Healthy() == nil { + return nil + } + return fmt.Errorf("instance not healthy") + }, 120*time.Second, 1*time.Second)) +} + +func TestSimplePath(t *testing.T) { t.Parallel() rootDirectory, rmRootDirectory, err := osqueryTempDir() require.NoError(t, err) @@ -385,40 +470,13 @@ func TestMultipleShutdowns(t *testing.T) { waitHealthy(t, runner) - for i := 0; i < 3; i += 1 { - require.NoError(t, runner.Shutdown(), "expected no error on calling shutdown but received error on attempt: ", i) - } -} - -func TestRestart(t *testing.T) { - t.Parallel() - runner, teardown := setupOsqueryInstanceForTests(t) - defer teardown() - - previousStats := runner.instance.stats - - require.NoError(t, runner.Restart()) - waitHealthy(t, runner) - - require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be set on latest instance stats after restart") - require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be set on latest instance stats after restart") - - require.NotEmpty(t, previousStats.ExitTime, "exit time should be set on last instance stats when restarted") - require.NotEmpty(t, previousStats.Error, "stats instance should have an error on restart") - - previousStats = runner.instance.stats - - require.NoError(t, runner.Restart()) - waitHealthy(t, runner) + require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be added to instance stats on start up") + require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be added to instance stats on start up") - require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be added to latest instance stats after restart") - require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be added to latest instance stats after restart") - - require.NotEmpty(t, previousStats.ExitTime, "exit time should be set on instance stats when restarted") - require.NotEmpty(t, previousStats.Error, "stats instance should have an error on restart") + require.NoError(t, runner.Shutdown()) } -func TestOsqueryDies(t *testing.T) { +func TestMultipleShutdowns(t *testing.T) { t.Parallel() rootDirectory, rmRootDirectory, err := osqueryTempDir() require.NoError(t, err) @@ -438,80 +496,16 @@ func TestOsqueryDies(t *testing.T) { WithOsquerydBinary(testOsqueryBinaryDirectory), ) go runner.Run() - require.NoError(t, err) - - waitHealthy(t, runner) - - previousStats := runner.instance.stats - - // Simulate the osquery process unexpectedly dying - runner.instanceLock.Lock() - require.NoError(t, killProcessGroup(runner.instance.cmd)) - runner.instance.errgroup.Wait() - runner.instanceLock.Unlock() - - waitHealthy(t, runner) - require.NotEmpty(t, previousStats.Error, "error should be added to stats when unexpected shutdown") - require.NotEmpty(t, previousStats.ExitTime, "exit time should be added to instance when unexpected shutdown") - - require.NoError(t, runner.Shutdown()) -} - -func TestNotStarted(t *testing.T) { - t.Parallel() - rootDirectory, rmRootDirectory, err := osqueryTempDir() - require.NoError(t, err) - defer rmRootDirectory() - - k := typesMocks.NewKnapsack(t) - k.On("OsqueryHealthcheckStartupDelay").Return(0 * time.Second).Maybe() - runner := newRunner(WithKnapsack(k), WithRootDirectory(rootDirectory)) - require.NoError(t, err) - - assert.Error(t, runner.Healthy()) - assert.NoError(t, runner.Shutdown()) -} - -// TestExtensionIsCleanedUp tests that the osquery extension cleans -// itself up. Unfortunately, this test has proved very flakey on -// circle-ci, but just fine on laptops. -func TestExtensionIsCleanedUp(t *testing.T) { - t.Skip("https://github.com/kolide/launcher/issues/478") - t.Parallel() - - runner, teardown := setupOsqueryInstanceForTests(t) - defer teardown() - - osqueryPID := runner.instance.cmd.Process.Pid - pgid, err := syscall.Getpgid(osqueryPID) - require.NoError(t, err) - require.Equal(t, pgid, osqueryPID, "pgid must be set") - - require.NoError(t, err) - - // kill the current osquery process but not the extension - err = syscall.Kill(osqueryPID, syscall.SIGKILL) - require.NoError(t, err) - - // We need to (a) let the runner restart osquery, and (b) wait for - // the extension to die. Both of these may take up to 30s. We'll - // start a clock, wait for the respawn, and after 32s, test that the - // extension process is no longer running. See - // https://github.com/kolide/launcher/pull/342 and associated for - // background. - timer1 := time.NewTimer(35 * time.Second) - - // Wait for osquery to respawn waitHealthy(t, runner) - // Ensure we've waited at least 32s - <-timer1.C + for i := 0; i < 3; i += 1 { + require.NoError(t, runner.Shutdown(), "expected no error on calling shutdown but received error on attempt: ", i) + } } -func TestExtensionSocketPath(t *testing.T) { +func TestOsqueryDies(t *testing.T) { t.Parallel() - rootDirectory, rmRootDirectory, err := osqueryTempDir() require.NoError(t, err) defer rmRootDirectory() @@ -523,76 +517,45 @@ func TestExtensionSocketPath(t *testing.T) { k.On("Slogger").Return(multislogger.NewNopLogger()) k.On("PinnedOsquerydVersion").Return("") - extensionSocketPath := filepath.Join(rootDirectory, "sock") runner := New( k, WithKnapsack(k), WithRootDirectory(rootDirectory), - WithExtensionSocketPath(extensionSocketPath), WithOsquerydBinary(testOsqueryBinaryDirectory), ) go runner.Run() + require.NoError(t, err) waitHealthy(t, runner) - // wait for the launcher-provided extension to register - time.Sleep(2 * time.Second) + previousStats := runner.instance.stats - client, err := osquery.NewClient(extensionSocketPath, 5*time.Second, osquery.DefaultWaitTime(1*time.Second), osquery.MaxWaitTime(1*time.Minute)) - require.NoError(t, err) - defer client.Close() + // Simulate the osquery process unexpectedly dying + runner.instanceLock.Lock() + require.NoError(t, killProcessGroup(runner.instance.cmd)) + runner.instance.errgroup.Wait() + runner.instanceLock.Unlock() - resp, err := client.Query("select * from launcher_gc_info") - require.NoError(t, err) - assert.Equal(t, int32(0), resp.Status.Code) - assert.Equal(t, "OK", resp.Status.Message) + waitHealthy(t, runner) + require.NotEmpty(t, previousStats.Error, "error should be added to stats when unexpected shutdown") + require.NotEmpty(t, previousStats.ExitTime, "exit time should be added to instance when unexpected shutdown") require.NoError(t, runner.Shutdown()) } -func TestOsquerySlowStart(t *testing.T) { +func TestNotStarted(t *testing.T) { t.Parallel() rootDirectory, rmRootDirectory, err := osqueryTempDir() require.NoError(t, err) defer rmRootDirectory() - var logBytes threadsafebuffer.ThreadSafeBuffer - k := typesMocks.NewKnapsack(t) k.On("OsqueryHealthcheckStartupDelay").Return(0 * time.Second).Maybe() - k.On("WatchdogEnabled").Return(false) - k.On("RegisterChangeObserver", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) - slogger := multislogger.New(slog.NewJSONHandler(&logBytes, &slog.HandlerOptions{Level: slog.LevelDebug})) - k.On("Slogger").Return(slogger.Logger) - k.On("PinnedOsquerydVersion").Return("") - - runner := New( - k, - WithKnapsack(k), - WithRootDirectory(rootDirectory), - WithOsquerydBinary(testOsqueryBinaryDirectory), - WithSlogger(slogger.Logger), - WithStartFunc(func(cmd *exec.Cmd) error { - err := cmd.Start() - if err != nil { - return fmt.Errorf("unexpected error starting command: %w", err) - } - // suspend the process right away - cmd.Process.Signal(syscall.SIGTSTP) - go func() { - // wait a while before resuming the process - time.Sleep(3 * time.Second) - cmd.Process.Signal(syscall.SIGCONT) - }() - return nil - }), - ) - go runner.Run() - waitHealthy(t, runner) + runner := newRunner(WithKnapsack(k), WithRootDirectory(rootDirectory)) + require.NoError(t, err) - // ensure that we actually had to wait on the socket - require.Contains(t, logBytes.String(), "osquery extension socket not created yet") - require.NoError(t, runner.Shutdown()) + assert.Error(t, runner.Healthy()) + assert.NoError(t, runner.Shutdown()) } // WithStartFunc defines the function that will be used to exeute the osqueryd @@ -603,40 +566,3 @@ func WithStartFunc(f func(cmd *exec.Cmd) error) OsqueryInstanceOption { i.startFunc = f } } - -// sets up an osquery instance with a running extension to be used in tests. -func setupOsqueryInstanceForTests(t *testing.T) (runner *Runner, teardown func()) { - rootDirectory, rmRootDirectory, err := osqueryTempDir() - require.NoError(t, err) - - k := typesMocks.NewKnapsack(t) - k.On("OsqueryHealthcheckStartupDelay").Return(0 * time.Second).Maybe() - k.On("WatchdogEnabled").Return(true) - k.On("WatchdogMemoryLimitMB").Return(150) - k.On("WatchdogUtilizationLimitPercent").Return(20) - k.On("WatchdogDelaySec").Return(120) - k.On("RegisterChangeObserver", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Maybe() - k.On("Slogger").Return(multislogger.NewNopLogger()) - k.On("PinnedOsquerydVersion").Return("") - - runner = New( - k, - WithKnapsack(k), - WithRootDirectory(rootDirectory), - WithOsquerydBinary(testOsqueryBinaryDirectory), - ) - go runner.Run() - waitHealthy(t, runner) - - osqueryPID := runner.instance.cmd.Process.Pid - - pgid, err := syscall.Getpgid(osqueryPID) - require.NoError(t, err) - require.Equal(t, pgid, osqueryPID) - - teardown = func() { - defer rmRootDirectory() - require.NoError(t, runner.Shutdown()) - } - return runner, teardown -} From 7afaa41f617505808635bde981a0c3725447a8d7 Mon Sep 17 00:00:00 2001 From: James Pickett Date: Mon, 1 Apr 2024 10:53:48 -0700 Subject: [PATCH 3/7] enable more tests on windows --- pkg/osquery/runtime/runtime_posix_test.go | 107 ++------------------ pkg/osquery/runtime/runtime_test.go | 98 ++++++++++++++++++ pkg/osquery/runtime/runtime_windows_test.go | 10 ++ 3 files changed, 116 insertions(+), 99 deletions(-) create mode 100644 pkg/osquery/runtime/runtime_windows_test.go diff --git a/pkg/osquery/runtime/runtime_posix_test.go b/pkg/osquery/runtime/runtime_posix_test.go index f75342040..4c73578fe 100644 --- a/pkg/osquery/runtime/runtime_posix_test.go +++ b/pkg/osquery/runtime/runtime_posix_test.go @@ -21,71 +21,15 @@ import ( "github.com/stretchr/testify/require" ) -func TestRestart(t *testing.T) { - t.Parallel() - runner, teardown := setupOsqueryInstanceForTests(t) - defer teardown() - - previousStats := runner.instance.stats - - require.NoError(t, runner.Restart()) - waitHealthy(t, runner) - - require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be set on latest instance stats after restart") - require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be set on latest instance stats after restart") - - require.NotEmpty(t, previousStats.ExitTime, "exit time should be set on last instance stats when restarted") - require.NotEmpty(t, previousStats.Error, "stats instance should have an error on restart") - - previousStats = runner.instance.stats - - require.NoError(t, runner.Restart()) - waitHealthy(t, runner) - - require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be added to latest instance stats after restart") - require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be added to latest instance stats after restart") - - require.NotEmpty(t, previousStats.ExitTime, "exit time should be set on instance stats when restarted") - require.NotEmpty(t, previousStats.Error, "stats instance should have an error on restart") -} - -// TestExtensionIsCleanedUp tests that the osquery extension cleans -// itself up. Unfortunately, this test has proved very flakey on -// circle-ci, but just fine on laptops. -func TestExtensionIsCleanedUp(t *testing.T) { - t.Skip("https://github.com/kolide/launcher/issues/478") - t.Parallel() - - runner, teardown := setupOsqueryInstanceForTests(t) - defer teardown() - - osqueryPID := runner.instance.cmd.Process.Pid - - pgid, err := syscall.Getpgid(osqueryPID) +func requirePgidMatch(t *testing.T, pid int) { + pgid, err := syscall.Getpgid(pid) require.NoError(t, err) - require.Equal(t, pgid, osqueryPID, "pgid must be set") - - require.NoError(t, err) - - // kill the current osquery process but not the extension - err = syscall.Kill(osqueryPID, syscall.SIGKILL) - require.NoError(t, err) - - // We need to (a) let the runner restart osquery, and (b) wait for - // the extension to die. Both of these may take up to 30s. We'll - // start a clock, wait for the respawn, and after 32s, test that the - // extension process is no longer running. See - // https://github.com/kolide/launcher/pull/342 and associated for - // background. - timer1 := time.NewTimer(35 * time.Second) - - // Wait for osquery to respawn - waitHealthy(t, runner) - - // Ensure we've waited at least 32s - <-timer1.C + require.Equal(t, pgid, pid) } +// TestOsquerySlowStart tests that the launcher can handle a slow-starting osqueryd process. +// This this is only enabled on non-Windows platforms because suspending we have not yet +// figured out how to suspend a process on windows via golang. func TestOsquerySlowStart(t *testing.T) { t.Parallel() rootDirectory, rmRootDirectory, err := osqueryTempDir() @@ -131,6 +75,8 @@ func TestOsquerySlowStart(t *testing.T) { require.NoError(t, runner.Shutdown()) } +// TestExtensionSocketPath tests that the launcher can start osqueryd with a custom extension socket path. +// This is only run on non-windows platforms because the extension socket path is semi random on windows. func TestExtensionSocketPath(t *testing.T) { t.Parallel() @@ -171,40 +117,3 @@ func TestExtensionSocketPath(t *testing.T) { require.NoError(t, runner.Shutdown()) } - -// sets up an osquery instance with a running extension to be used in tests. -func setupOsqueryInstanceForTests(t *testing.T) (runner *Runner, teardown func()) { - rootDirectory, rmRootDirectory, err := osqueryTempDir() - require.NoError(t, err) - - k := typesMocks.NewKnapsack(t) - k.On("OsqueryHealthcheckStartupDelay").Return(0 * time.Second).Maybe() - k.On("WatchdogEnabled").Return(true) - k.On("WatchdogMemoryLimitMB").Return(150) - k.On("WatchdogUtilizationLimitPercent").Return(20) - k.On("WatchdogDelaySec").Return(120) - k.On("RegisterChangeObserver", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Maybe() - k.On("Slogger").Return(multislogger.NewNopLogger()) - k.On("PinnedOsquerydVersion").Return("") - - runner = New( - k, - WithKnapsack(k), - WithRootDirectory(rootDirectory), - WithOsquerydBinary(testOsqueryBinaryDirectory), - ) - go runner.Run() - waitHealthy(t, runner) - - osqueryPID := runner.instance.cmd.Process.Pid - - pgid, err := syscall.Getpgid(osqueryPID) - require.NoError(t, err) - require.Equal(t, pgid, osqueryPID) - - teardown = func() { - defer rmRootDirectory() - require.NoError(t, runner.Shutdown()) - } - return runner, teardown -} diff --git a/pkg/osquery/runtime/runtime_test.go b/pkg/osquery/runtime/runtime_test.go index a750bc04a..d534bff89 100644 --- a/pkg/osquery/runtime/runtime_test.go +++ b/pkg/osquery/runtime/runtime_test.go @@ -566,3 +566,101 @@ func WithStartFunc(f func(cmd *exec.Cmd) error) OsqueryInstanceOption { i.startFunc = f } } + +func TestRestart(t *testing.T) { + t.Parallel() + runner, teardown := setupOsqueryInstanceForTests(t) + defer teardown() + + previousStats := runner.instance.stats + + require.NoError(t, runner.Restart()) + waitHealthy(t, runner) + + require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be set on latest instance stats after restart") + require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be set on latest instance stats after restart") + + require.NotEmpty(t, previousStats.ExitTime, "exit time should be set on last instance stats when restarted") + require.NotEmpty(t, previousStats.Error, "stats instance should have an error on restart") + + previousStats = runner.instance.stats + + require.NoError(t, runner.Restart()) + waitHealthy(t, runner) + + require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be added to latest instance stats after restart") + require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be added to latest instance stats after restart") + + require.NotEmpty(t, previousStats.ExitTime, "exit time should be set on instance stats when restarted") + require.NotEmpty(t, previousStats.Error, "stats instance should have an error on restart") +} + +// TestExtensionIsCleanedUp tests that the osquery extension cleans +// itself up. Unfortunately, this test has proved very flakey on +// circle-ci, but just fine on laptops. +func TestExtensionIsCleanedUp(t *testing.T) { + t.Skip("https://github.com/kolide/launcher/issues/478") + t.Parallel() + + runner, teardown := setupOsqueryInstanceForTests(t) + defer teardown() + + osqueryPID := runner.instance.cmd.Process.Pid + + requirePgidMatch(t, osqueryPID) + + require.NoError(t, runner.instance.cmd.Process.Kill()) + + // kill the current osquery process but not the extension + // err = syscall.Kill(osqueryPID, syscall.SIGKILL) + // require.NoError(t, err) + + // We need to (a) let the runner restart osquery, and (b) wait for + // the extension to die. Both of these may take up to 30s. We'll + // start a clock, wait for the respawn, and after 32s, test that the + // extension process is no longer running. See + // https://github.com/kolide/launcher/pull/342 and associated for + // background. + timer1 := time.NewTimer(35 * time.Second) + + // Wait for osquery to respawn + waitHealthy(t, runner) + + // Ensure we've waited at least 32s + <-timer1.C +} + +// sets up an osquery instance with a running extension to be used in tests. +func setupOsqueryInstanceForTests(t *testing.T) (runner *Runner, teardown func()) { + rootDirectory, rmRootDirectory, err := osqueryTempDir() + require.NoError(t, err) + + k := typesMocks.NewKnapsack(t) + k.On("OsqueryHealthcheckStartupDelay").Return(0 * time.Second).Maybe() + k.On("WatchdogEnabled").Return(true) + k.On("WatchdogMemoryLimitMB").Return(150) + k.On("WatchdogUtilizationLimitPercent").Return(20) + k.On("WatchdogDelaySec").Return(120) + k.On("RegisterChangeObserver", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Maybe() + k.On("Slogger").Return(multislogger.NewNopLogger()) + k.On("PinnedOsquerydVersion").Return("") + + runner = New( + k, + WithKnapsack(k), + WithRootDirectory(rootDirectory), + WithOsquerydBinary(testOsqueryBinaryDirectory), + ) + go runner.Run() + waitHealthy(t, runner) + + osqueryPID := runner.instance.cmd.Process.Pid + + requirePgidMatch(t, osqueryPID) + + teardown = func() { + defer rmRootDirectory() + require.NoError(t, runner.Shutdown()) + } + return runner, teardown +} diff --git a/pkg/osquery/runtime/runtime_windows_test.go b/pkg/osquery/runtime/runtime_windows_test.go new file mode 100644 index 000000000..f9a0b7824 --- /dev/null +++ b/pkg/osquery/runtime/runtime_windows_test.go @@ -0,0 +1,10 @@ +//go:build windows +// +build windows + +package runtime + +import ( + "testing" +) + +func requirePgidMatch(_ *testing.T, _ int) {} From 752bb4ec1f3ec898b54e2d0c249604af96914f26 Mon Sep 17 00:00:00 2001 From: James Pickett Date: Mon, 1 Apr 2024 10:55:19 -0700 Subject: [PATCH 4/7] rearrainge --- pkg/osquery/runtime/runtime_test.go | 56 ++++++++++++++--------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/pkg/osquery/runtime/runtime_test.go b/pkg/osquery/runtime/runtime_test.go index d534bff89..33a1fb775 100644 --- a/pkg/osquery/runtime/runtime_test.go +++ b/pkg/osquery/runtime/runtime_test.go @@ -504,6 +504,34 @@ func TestMultipleShutdowns(t *testing.T) { } } +func TestRestart(t *testing.T) { + t.Parallel() + runner, teardown := setupOsqueryInstanceForTests(t) + defer teardown() + + previousStats := runner.instance.stats + + require.NoError(t, runner.Restart()) + waitHealthy(t, runner) + + require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be set on latest instance stats after restart") + require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be set on latest instance stats after restart") + + require.NotEmpty(t, previousStats.ExitTime, "exit time should be set on last instance stats when restarted") + require.NotEmpty(t, previousStats.Error, "stats instance should have an error on restart") + + previousStats = runner.instance.stats + + require.NoError(t, runner.Restart()) + waitHealthy(t, runner) + + require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be added to latest instance stats after restart") + require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be added to latest instance stats after restart") + + require.NotEmpty(t, previousStats.ExitTime, "exit time should be set on instance stats when restarted") + require.NotEmpty(t, previousStats.Error, "stats instance should have an error on restart") +} + func TestOsqueryDies(t *testing.T) { t.Parallel() rootDirectory, rmRootDirectory, err := osqueryTempDir() @@ -567,34 +595,6 @@ func WithStartFunc(f func(cmd *exec.Cmd) error) OsqueryInstanceOption { } } -func TestRestart(t *testing.T) { - t.Parallel() - runner, teardown := setupOsqueryInstanceForTests(t) - defer teardown() - - previousStats := runner.instance.stats - - require.NoError(t, runner.Restart()) - waitHealthy(t, runner) - - require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be set on latest instance stats after restart") - require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be set on latest instance stats after restart") - - require.NotEmpty(t, previousStats.ExitTime, "exit time should be set on last instance stats when restarted") - require.NotEmpty(t, previousStats.Error, "stats instance should have an error on restart") - - previousStats = runner.instance.stats - - require.NoError(t, runner.Restart()) - waitHealthy(t, runner) - - require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be added to latest instance stats after restart") - require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be added to latest instance stats after restart") - - require.NotEmpty(t, previousStats.ExitTime, "exit time should be set on instance stats when restarted") - require.NotEmpty(t, previousStats.Error, "stats instance should have an error on restart") -} - // TestExtensionIsCleanedUp tests that the osquery extension cleans // itself up. Unfortunately, this test has proved very flakey on // circle-ci, but just fine on laptops. From 566d4379e46b9f865fd737910bbb7b4e3c1289da Mon Sep 17 00:00:00 2001 From: James Pickett Date: Mon, 1 Apr 2024 10:58:04 -0700 Subject: [PATCH 5/7] clean up --- pkg/osquery/runtime/runtime_test.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/pkg/osquery/runtime/runtime_test.go b/pkg/osquery/runtime/runtime_test.go index 33a1fb775..0f8a7f390 100644 --- a/pkg/osquery/runtime/runtime_test.go +++ b/pkg/osquery/runtime/runtime_test.go @@ -605,15 +605,10 @@ func TestExtensionIsCleanedUp(t *testing.T) { runner, teardown := setupOsqueryInstanceForTests(t) defer teardown() - osqueryPID := runner.instance.cmd.Process.Pid - - requirePgidMatch(t, osqueryPID) - - require.NoError(t, runner.instance.cmd.Process.Kill()) + requirePgidMatch(t, runner.instance.cmd.Process.Pid) // kill the current osquery process but not the extension - // err = syscall.Kill(osqueryPID, syscall.SIGKILL) - // require.NoError(t, err) + require.NoError(t, runner.instance.cmd.Process.Kill()) // We need to (a) let the runner restart osquery, and (b) wait for // the extension to die. Both of these may take up to 30s. We'll @@ -654,9 +649,7 @@ func setupOsqueryInstanceForTests(t *testing.T) (runner *Runner, teardown func() go runner.Run() waitHealthy(t, runner) - osqueryPID := runner.instance.cmd.Process.Pid - - requirePgidMatch(t, osqueryPID) + requirePgidMatch(t, runner.instance.cmd.Process.Pid) teardown = func() { defer rmRootDirectory() From b92989d50d7f9ea4eaff41ef55c3b300214fb1d6 Mon Sep 17 00:00:00 2001 From: James Pickett Date: Mon, 1 Apr 2024 14:58:49 -0700 Subject: [PATCH 6/7] put restart test for posix only --- pkg/osquery/runtime/runtime_posix_test.go | 31 +++++++++++++++++++++++ pkg/osquery/runtime/runtime_test.go | 30 +--------------------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/pkg/osquery/runtime/runtime_posix_test.go b/pkg/osquery/runtime/runtime_posix_test.go index 4c73578fe..311162136 100644 --- a/pkg/osquery/runtime/runtime_posix_test.go +++ b/pkg/osquery/runtime/runtime_posix_test.go @@ -117,3 +117,34 @@ func TestExtensionSocketPath(t *testing.T) { require.NoError(t, runner.Shutdown()) } + +// TestRestart tests that the launcher can restart the osqueryd process. +// This test causes time outs on windows, so it is only run on non-windows platforms. +// Should investigate why this is the case. +func TestRestart(t *testing.T) { + t.Parallel() + runner, teardown := setupOsqueryInstanceForTests(t) + defer teardown() + + previousStats := runner.instance.stats + + require.NoError(t, runner.Restart()) + waitHealthy(t, runner) + + require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be set on latest instance stats after restart") + require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be set on latest instance stats after restart") + + require.NotEmpty(t, previousStats.ExitTime, "exit time should be set on last instance stats when restarted") + require.NotEmpty(t, previousStats.Error, "stats instance should have an error on restart") + + previousStats = runner.instance.stats + + require.NoError(t, runner.Restart()) + waitHealthy(t, runner) + + require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be added to latest instance stats after restart") + require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be added to latest instance stats after restart") + + require.NotEmpty(t, previousStats.ExitTime, "exit time should be set on instance stats when restarted") + require.NotEmpty(t, previousStats.Error, "stats instance should have an error on restart") +} diff --git a/pkg/osquery/runtime/runtime_test.go b/pkg/osquery/runtime/runtime_test.go index 0f8a7f390..7f3fef4d2 100644 --- a/pkg/osquery/runtime/runtime_test.go +++ b/pkg/osquery/runtime/runtime_test.go @@ -444,7 +444,7 @@ func waitHealthy(t *testing.T, runner *Runner) { return nil } return fmt.Errorf("instance not healthy") - }, 120*time.Second, 1*time.Second)) + }, 30*time.Second, 1*time.Second)) } func TestSimplePath(t *testing.T) { @@ -504,34 +504,6 @@ func TestMultipleShutdowns(t *testing.T) { } } -func TestRestart(t *testing.T) { - t.Parallel() - runner, teardown := setupOsqueryInstanceForTests(t) - defer teardown() - - previousStats := runner.instance.stats - - require.NoError(t, runner.Restart()) - waitHealthy(t, runner) - - require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be set on latest instance stats after restart") - require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be set on latest instance stats after restart") - - require.NotEmpty(t, previousStats.ExitTime, "exit time should be set on last instance stats when restarted") - require.NotEmpty(t, previousStats.Error, "stats instance should have an error on restart") - - previousStats = runner.instance.stats - - require.NoError(t, runner.Restart()) - waitHealthy(t, runner) - - require.NotEmpty(t, runner.instance.stats.StartTime, "start time should be added to latest instance stats after restart") - require.NotEmpty(t, runner.instance.stats.ConnectTime, "connect time should be added to latest instance stats after restart") - - require.NotEmpty(t, previousStats.ExitTime, "exit time should be set on instance stats when restarted") - require.NotEmpty(t, previousStats.Error, "stats instance should have an error on restart") -} - func TestOsqueryDies(t *testing.T) { t.Parallel() rootDirectory, rmRootDirectory, err := osqueryTempDir() From d1dadc7dea46edbb9f3eb758ed3288b28acaceed Mon Sep 17 00:00:00 2001 From: James Pickett Date: Tue, 2 Apr 2024 08:13:04 -0700 Subject: [PATCH 7/7] dont run tests without elevated permissions on windows, fix comments --- pkg/osquery/runtime/runtime_posix_test.go | 12 +++++++++--- pkg/osquery/runtime/runtime_test.go | 5 +++++ pkg/osquery/runtime/runtime_windows_test.go | 8 ++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/pkg/osquery/runtime/runtime_posix_test.go b/pkg/osquery/runtime/runtime_posix_test.go index 311162136..615829dc3 100644 --- a/pkg/osquery/runtime/runtime_posix_test.go +++ b/pkg/osquery/runtime/runtime_posix_test.go @@ -27,9 +27,15 @@ func requirePgidMatch(t *testing.T, pid int) { require.Equal(t, pgid, pid) } -// TestOsquerySlowStart tests that the launcher can handle a slow-starting osqueryd process. -// This this is only enabled on non-Windows platforms because suspending we have not yet -// figured out how to suspend a process on windows via golang. +// hasPermissionsToRunTest always return true for non-windows platforms since +// elveated permissions are not required to run the tests +func hasPermissionsToRunTest() bool { + return true +} + +// TestOsquerySlowStart tests that launcher can handle a slow-starting osqueryd process. +// This this is only enabled on non-Windows platforms because we have not yet figured +// out how to suspend and resume a process on Windows via golang. func TestOsquerySlowStart(t *testing.T) { t.Parallel() rootDirectory, rmRootDirectory, err := osqueryTempDir() diff --git a/pkg/osquery/runtime/runtime_test.go b/pkg/osquery/runtime/runtime_test.go index 7f3fef4d2..74680d2e4 100644 --- a/pkg/osquery/runtime/runtime_test.go +++ b/pkg/osquery/runtime/runtime_test.go @@ -34,6 +34,11 @@ var testOsqueryBinaryDirectory string // TestMain overrides the default test main function. This allows us to share setup/teardown. func TestMain(m *testing.M) { + if !hasPermissionsToRunTest() { + fmt.Println("these tests must be run as an administrator on windows") + return + } + binDirectory, rmBinDirectory, err := osqueryTempDir() if err != nil { fmt.Println("Failed to make temp dir for test binaries") diff --git a/pkg/osquery/runtime/runtime_windows_test.go b/pkg/osquery/runtime/runtime_windows_test.go index f9a0b7824..22730625b 100644 --- a/pkg/osquery/runtime/runtime_windows_test.go +++ b/pkg/osquery/runtime/runtime_windows_test.go @@ -5,6 +5,14 @@ package runtime import ( "testing" + + "golang.org/x/sys/windows" ) func requirePgidMatch(_ *testing.T, _ int) {} + +// hasPermissionsToRunTest return true if the current process has elevated permissions (administrator), +// this is required to run tests on windows +func hasPermissionsToRunTest() bool { + return windows.GetCurrentProcessToken().IsElevated() +}