Skip to content

Commit

Permalink
F/2117/gke job run (#205)
Browse files Browse the repository at this point in the history
  • Loading branch information
BSick7 authored Jan 6, 2025
1 parent a613933 commit 2ace6b3
Show file tree
Hide file tree
Showing 12 changed files with 315 additions and 8 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# 0.0.135 (Unreleased)
* Added `nullstone run` command that allows you to a start a new job/task.
* Added support for `nullstone run` to ECS/Fargate tasks and GKE jobs.

# 0.0.134 (Dec 17, 2024)
* Improve reliability of streaming deploy logs when performing a deployment.

Expand Down
11 changes: 11 additions & 0 deletions admin/remoter.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,22 @@ type RemoteOptions struct {
LogEmitter app.LogEmitter
}

type RunOptions struct {
// Container represents the specific container name to execute against in the k8s pod/ecs task
Container string
Username string
LogStreamer app.LogStreamer
LogEmitter app.LogEmitter
}

type Remoter interface {
// Exec allows a user to execute a command (usually tunneling) into a running service
// This only makes sense for container-based providers
Exec(ctx context.Context, options RemoteOptions, cmd []string) error

// Ssh allows a user to SSH into a running service
Ssh(ctx context.Context, options RemoteOptions) error

// Run starts a new job/task and executes a command
Run(ctx context.Context, options RunOptions, cmd []string) error
}
1 change: 1 addition & 0 deletions app/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ func Build() *cli.App {
cmd.Status(adminProviders),
cmd.Exec(appProviders, adminProviders),
cmd.Ssh(adminProviders),
cmd.Run(appProviders, adminProviders),
cmd.Profile,
}
sort.Sort(cli.CommandsByName(cliApp.Commands))
Expand Down
8 changes: 8 additions & 0 deletions aws/beanstalk/remoter.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ import (
"gopkg.in/nullstone-io/nullstone.v0/aws/ssm"
)

var (
_ admin.Remoter = Remoter{}
)

func NewRemoter(ctx context.Context, osWriters logging.OsWriters, source outputs.RetrieverSource, appDetails app.Details) (admin.Remoter, error) {
outs, err := outputs.Retrieve[Outputs](ctx, source, appDetails.Workspace, appDetails.WorkspaceConfig)
if err != nil {
Expand Down Expand Up @@ -52,6 +56,10 @@ func (r Remoter) Ssh(ctx context.Context, options admin.RemoteOptions) error {
return ssm.StartEc2Session(ctx, r.Infra.AdminerConfig(), r.Infra.Region, instanceId, parameters)
}

func (r Remoter) Run(ctx context.Context, options admin.RunOptions, cmd []string) error {
return fmt.Errorf("`run` is not supported for Beanstalk yet")
}

func (r Remoter) getInstanceId(ctx context.Context, options admin.RemoteOptions) (string, error) {
if options.Instance == "" {
if instanceId, err := GetRandomInstance(ctx, r.Infra); err != nil {
Expand Down
9 changes: 9 additions & 0 deletions aws/ec2/remoter.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,18 @@ package ec2

import (
"context"
"fmt"
"github.com/nullstone-io/deployment-sdk/app"
"github.com/nullstone-io/deployment-sdk/logging"
"github.com/nullstone-io/deployment-sdk/outputs"
"gopkg.in/nullstone-io/nullstone.v0/admin"
"gopkg.in/nullstone-io/nullstone.v0/aws/ssm"
)

var (
_ admin.Remoter = Remoter{}
)

func NewRemoter(ctx context.Context, osWriters logging.OsWriters, source outputs.RetrieverSource, appDetails app.Details) (admin.Remoter, error) {
outs, err := outputs.Retrieve[Outputs](ctx, source, appDetails.Workspace, appDetails.WorkspaceConfig)
if err != nil {
Expand Down Expand Up @@ -42,3 +47,7 @@ func (r Remoter) Ssh(ctx context.Context, options admin.RemoteOptions) error {

return ssm.StartEc2Session(ctx, r.Infra.AdminerConfig(), r.Infra.Region, r.Infra.InstanceId, parameters)
}

func (r Remoter) Run(ctx context.Context, options admin.RunOptions, cmd []string) error {
return fmt.Errorf("`run` is not supported for EC2 yet")
}
11 changes: 11 additions & 0 deletions aws/ecs/remoter.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ import (
"gopkg.in/nullstone-io/nullstone.v0/admin"
)

var (
_ admin.Remoter = Remoter{}
)

func NewRemoter(ctx context.Context, osWriters logging.OsWriters, source outputs.RetrieverSource, appDetails app.Details) (admin.Remoter, error) {
outs, err := outputs.Retrieve[Outputs](ctx, source, appDetails.Workspace, appDetails.WorkspaceConfig)
if err != nil {
Expand Down Expand Up @@ -60,6 +64,13 @@ func (r Remoter) Ssh(ctx context.Context, options admin.RemoteOptions) error {
return ExecCommand(ctx, r.Infra, taskId, options.Container, []string{"/bin/sh"}, nil)
}

func (r Remoter) Run(ctx context.Context, options admin.RunOptions, cmd []string) error {
if r.Infra.ServiceName != "" {
return fmt.Errorf("cannot use `run` with a long-running application, use `exec` instead")
}
return RunTask(ctx, r.Infra, options.Container, options.Username, cmd, options.LogStreamer, options.LogEmitter)
}

func (r Remoter) getTaskId(ctx context.Context, options admin.RemoteOptions) (string, error) {
if options.Task == "" {
if taskId, err := GetRandomTask(ctx, r.Infra); err != nil {
Expand Down
64 changes: 64 additions & 0 deletions cmd/run.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package cmd

import (
"context"
"fmt"
"github.com/nullstone-io/deployment-sdk/app"
"github.com/nullstone-io/deployment-sdk/logging"
"github.com/nullstone-io/deployment-sdk/outputs"
"github.com/urfave/cli/v2"
"gopkg.in/nullstone-io/go-api-client.v0"
"gopkg.in/nullstone-io/nullstone.v0/admin"
"os"
)

var Run = func(appProviders app.Providers, providers admin.Providers) *cli.Command {
return &cli.Command{
Name: "run",
Description: "Starts a new container/serverless for the given Nullstone job/task. ",
Usage: "Starts a new job/task",
UsageText: "nullstone run [--stack=<stack-name>] --app=<app-name> --env=<env-name> [options] [command]",
Flags: []cli.Flag{
StackFlag,
AppFlag,
EnvFlag,
ContainerFlag,
},
Action: func(c *cli.Context) error {
var cmd []string
if c.Args().Present() {
cmd = c.Args().Slice()
}

return AppWorkspaceAction(c, func(ctx context.Context, cfg api.Config, appDetails app.Details) error {
client := api.Client{Config: cfg}
user, err := client.CurrentUser().Get(ctx)
if err != nil {
return fmt.Errorf("unable to fetch the current user")
}
if user == nil {
return fmt.Errorf("unable to load the current user info")
}

source := outputs.ApiRetrieverSource{Config: cfg}

logStreamer, err := appProviders.FindLogStreamer(ctx, logging.StandardOsWriters{}, source, appDetails)
if err != nil {
return err
}

remoter, err := providers.FindRemoter(ctx, logging.StandardOsWriters{}, source, appDetails)
if err != nil {
return err
}
options := admin.RunOptions{
Container: c.String("container"),
Username: user.Name,
LogStreamer: logStreamer,
LogEmitter: app.NewWriterLogEmitter(os.Stdout),
}
return remoter.Run(ctx, options, cmd)
})
},
}
}
3 changes: 3 additions & 0 deletions gcp/gke/outputs.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ type Outputs struct {
ImageRepoUrl docker.ImageUrl `ns:"image_repo_url,optional"`
Deployer gcp.ServiceAccount `ns:"deployer"`
MainContainerName string `ns:"main_container_name,optional"`
// JobDefinitionName is only specified for a job/task
// It refers to a Kubernetes ConfigMap containing the job definition in the "template" field
JobDefinitionName string `ns:"job_definition_name,optional"`

ClusterNamespace ClusterNamespaceOutputs `ns:",connectionContract:cluster-namespace/gcp/k8s:gke"`
}
Expand Down
27 changes: 27 additions & 0 deletions gcp/gke/remoter.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,19 @@ import (
"context"
"fmt"
"github.com/nullstone-io/deployment-sdk/app"
"github.com/nullstone-io/deployment-sdk/gcp/gke"
"github.com/nullstone-io/deployment-sdk/logging"
"github.com/nullstone-io/deployment-sdk/outputs"
"gopkg.in/nullstone-io/nullstone.v0/admin"
"gopkg.in/nullstone-io/nullstone.v0/k8s"
"k8s.io/client-go/rest"
"os"
)

var (
_ admin.Remoter = Remoter{}
)

func NewRemoter(ctx context.Context, osWriters logging.OsWriters, source outputs.RetrieverSource, appDetails app.Details) (admin.Remoter, error) {
outs, err := outputs.Retrieve[Outputs](ctx, source, appDetails.Workspace, appDetails.WorkspaceConfig)
if err != nil {
Expand All @@ -32,6 +38,10 @@ type Remoter struct {
}

func (r Remoter) Exec(ctx context.Context, options admin.RemoteOptions, cmd []string) error {
if r.Infra.ServiceName == "" {
return fmt.Errorf("cannot `exec` unless you have a long-running service, use `run` for a job/task")
}

opts := &k8s.ExecOptions{
In: os.Stdin,
Out: r.OsWriters.Stdout(),
Expand All @@ -55,3 +65,20 @@ func (r Remoter) Ssh(ctx context.Context, options admin.RemoteOptions) error {

return ExecCommand(ctx, r.Infra, options.Pod, options.Container, []string{"/bin/sh"}, opts)
}

func (r Remoter) Run(ctx context.Context, options admin.RunOptions, cmd []string) error {
if r.Infra.ServiceName != "" {
return fmt.Errorf("cannot use `run` for a long-running service, use `exec` instead")
}

runner := k8s.JobRunner{
Namespace: r.Infra.ServiceNamespace,
AppName: r.Details.App.Name,
MainContainerName: r.Infra.MainContainerName,
JobDefinitionName: r.Infra.JobDefinitionName,
NewConfigFn: func(ctx context.Context) (*rest.Config, error) {
return gke.CreateKubeConfig(ctx, r.Infra.ClusterNamespace, r.Infra.Deployer)
},
}
return runner.Run(ctx, options, cmd)
}
8 changes: 4 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
module gopkg.in/nullstone-io/nullstone.v0

go 1.22.0
go 1.23.0

toolchain go1.22.8
toolchain go1.23.3

require (
github.com/aws/aws-sdk-go-v2 v1.26.0
Expand All @@ -13,15 +13,15 @@ require (
github.com/go-git/go-git/v5 v5.4.2
github.com/gosuri/uilive v0.0.4
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db
github.com/nullstone-io/deployment-sdk v0.0.0-20241211140744-31a2fc4195ae
github.com/nullstone-io/deployment-sdk v0.0.0-20250106193300-6ff1ee19f6e7
github.com/nullstone-io/iac v0.0.0-20241105141304-4427f5e31e88
github.com/nullstone-io/module v0.2.9
github.com/ryanuber/columnize v2.1.2+incompatible
github.com/stretchr/testify v1.9.0
github.com/urfave/cli/v2 v2.3.0
golang.org/x/crypto v0.28.0
golang.org/x/sync v0.8.0
gopkg.in/nullstone-io/go-api-client.v0 v0.0.0-20241126225657-182d3304f2df
gopkg.in/nullstone-io/go-api-client.v0 v0.0.0-20241213224916-66a525701805
k8s.io/api v0.27.2
k8s.io/apimachinery v0.27.2
k8s.io/client-go v0.27.2
Expand Down
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -820,8 +820,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nullstone-io/deployment-sdk v0.0.0-20241211140744-31a2fc4195ae h1:vKD8ufJlr+DblrdoXLftebLGrobRwaedYMVPMxA7Dew=
github.com/nullstone-io/deployment-sdk v0.0.0-20241211140744-31a2fc4195ae/go.mod h1:kqj/vI7njyeluNT59Faz0IGhD/h2AZLe+KDczi4XS84=
github.com/nullstone-io/deployment-sdk v0.0.0-20250106193300-6ff1ee19f6e7 h1:m4nhAsZv4G+DLalvjVUvLuzF2zehfgKb7tvZJocc8pk=
github.com/nullstone-io/deployment-sdk v0.0.0-20250106193300-6ff1ee19f6e7/go.mod h1:44JrdCw01w4meAG+S4kKIZQxatCJa1fiBUxyo+FBzoA=
github.com/nullstone-io/iac v0.0.0-20241105141304-4427f5e31e88 h1:/mXYlqXFwPf0lqBGosvYzregrSL8N0FPaS5dG/Jzar4=
github.com/nullstone-io/iac v0.0.0-20241105141304-4427f5e31e88/go.mod h1:tRZPp425TN0XVAkrkICbZ1EAkmmhNb7ueq7xBLX2goM=
github.com/nullstone-io/module v0.2.9 h1:PcYhPEemBbc+RdP+Q/DF0+XlwJkkNb5R17Hfv8qaYyc=
Expand Down Expand Up @@ -1534,8 +1534,8 @@ gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKW
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
gopkg.in/nullstone-io/go-api-client.v0 v0.0.0-20241126225657-182d3304f2df h1:cun7quo+kgvhOvThH2pV9G+E6S78UWri2oufnI1dDaI=
gopkg.in/nullstone-io/go-api-client.v0 v0.0.0-20241126225657-182d3304f2df/go.mod h1:m/JNiW4XSXjRLAedd7LYOO0l/FfCiKffRFolkKpfpgA=
gopkg.in/nullstone-io/go-api-client.v0 v0.0.0-20241213224916-66a525701805 h1:aGnIBuBJTSE1RQf1YOFVIOUXjpIrUe3AsynUN5TOAD8=
gopkg.in/nullstone-io/go-api-client.v0 v0.0.0-20241213224916-66a525701805/go.mod h1:m/JNiW4XSXjRLAedd7LYOO0l/FfCiKffRFolkKpfpgA=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1 h1:d4KQkxAaAiRY2h5Zqis161Pv91A37uZyJOx73duwUwM=
gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1/go.mod h1:WbjuEoo1oadwzQ4apSDU+JTvmllEHtsNHS6y7vFc7iw=
Expand Down
Loading

0 comments on commit 2ace6b3

Please sign in to comment.