FIPS Build

.buildkite/integration.pipeline.yml

@@ -20,6 +20,19 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
 
+      - label: "Packaging: Ubuntu x86_64 FIPS"
+        key: "packaging-ubuntu-x86-64-fips"
+        env:
+          PACKAGES: "tar.gz"
+          PLATFORMS: "linux/amd64"
+          FIPS: "true"
+        command: ".buildkite/scripts/steps/integration-package.sh"
+        artifact_paths:
+        - build/distributions/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-4"
+
       - label: "Packaging: Ubuntu arm64"
         key: "packaging-ubuntu-arm64"
         env:
@@ -32,6 +45,19 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
 
+      - label: "Packaging: Ubuntu arm64 FIPS"
+        key: "packaging-ubuntu-arm64-fips"
+        env:
+          PACKAGES: "tar.gz"
+          PLATFORMS: "linux/arm64"
+          FIPS: "true"
+        command: ".buildkite/scripts/steps/integration-package.sh"
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-4"
+
       - label: "Packaging: Windows"
         key: "packaging-windows"
         env:

dev-tools/mage/build.go

@@ -11,6 +11,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/josephspurrier/goversioninfo"
@@ -34,6 +35,39 @@ type BuildArgs struct {
 	WinMetadata bool // Add resource metadata to Windows binaries (like add the version number to the .exe properties).
 }
 
+// buildTagRE is a regexp to match strings like "-tags=abcd"
+// but does not match "-tags= "
+var buildTagRE = regexp.MustCompile(`-tags=([\S]+)?`)
+
+// ParseBuildTags returns the ExtraFlags param where all flags that are go build tags are joined by a comma.
+//
+// For example if given -someflag=val1 -tags=buildtag1 -tags=buildtag2
+// It will return -someflag=val1 -tags=buildtag1,buildtag2
+func (b BuildArgs) ParseBuildTags() []string {
+	flags := make([]string, 0)
+	if len(b.ExtraFlags) == 0 {
+		return flags
+	}
+
+	buildTags := make([]string, 0)
+	for _, flag := range b.ExtraFlags {
+		if buildTagRE.MatchString(flag) {
+			arr := buildTagRE.FindStringSubmatch(flag)
+			if len(arr) != 2 || arr[1] == "" {
+				log.Printf("Unexpected format found for buildargs.ExtraFlags, ignoring value  %q", flag)
+				continue
+			}
+			buildTags = append(buildTags, arr[1])
+		} else {
+			flags = append(flags, flag)
+		}
+	}
+	if len(buildTags) > 0 {
+		flags = append(flags, "-tags="+strings.Join(buildTags, ","))
+	}
+	return flags
+}
+
 // DefaultBuildArgs returns the default BuildArgs for use in builds.
 func DefaultBuildArgs() BuildArgs {
 	args := BuildArgs{
@@ -53,6 +87,11 @@ func DefaultBuildArgs() BuildArgs {
 		args.ExtraFlags = append(args.ExtraFlags, "-buildmode", "pie")
 	}
 
+	if FIPSBuild {
+		args.ExtraFlags = append(args.ExtraFlags, "-tags=requirefips")
+		args.CGO = true
+	}
+
 	if DevBuild {
 		// Disable optimizations (-N) and inlining (-l) for debugging.
 		args.ExtraFlags = append(args.ExtraFlags, `-gcflags=all=-N -l`)
@@ -151,6 +190,12 @@ func Build(params BuildArgs) error {
 	if params.CGO {
 		cgoEnabled = "1"
 	}
+
+	if FIPSBuild {
+		cgoEnabled = "1"
+		env["GOEXPERIMENT"] = "systemcrypto"
+	}
+
 	env["CGO_ENABLED"] = cgoEnabled
 
 	// Spec
@@ -159,7 +204,7 @@ func Build(params BuildArgs) error {
 		"-o",
 		filepath.Join(params.OutputDir, binaryName),
 	}
-	args = append(args, params.ExtraFlags...)
+	args = append(args, params.ParseBuildTags()...)
 
 	// ldflags
 	ldflags := params.LDFlags

dev-tools/mage/build_test.go

@@ -0,0 +1,59 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+package mage
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_BuildArgs_ParseBuildTags(t *testing.T) {
+	tests := []struct {
+		name   string
+		input  []string
+		expect []string
+	}{{
+		name:   "no flags",
+		input:  nil,
+		expect: []string{},
+	}, {
+		name:   "multiple flags with no tags",
+		input:  []string{"-a", "-b", "-key=value"},
+		expect: []string{"-a", "-b", "-key=value"},
+	}, {
+		name:   "one build tag",
+		input:  []string{"-tags=example"},
+		expect: []string{"-tags=example"},
+	}, {
+		name:   "multiple build tags",
+		input:  []string{"-tags=example", "-tags=test"},
+		expect: []string{"-tags=example,test"},
+	}, {
+		name:   "joined build tags",
+		input:  []string{"-tags=example,test"},
+		expect: []string{"-tags=example,test"},
+	}, {
+		name:   "multiple build tags with other flags",
+		input:  []string{"-tags=example", "-tags=test", "-key=value", "-a"},
+		expect: []string{"-key=value", "-a", "-tags=example,test"},
+	}, {
+		name:   "incorrectly formatted tag",
+		input:  []string{"-tags= example"},
+		expect: []string{},
+	}, {
+		name:   "incorrectly formatted tag with valid tag",
+		input:  []string{"-tags= example", "-tags=test"},
+		expect: []string{"-tags=test"},
+	}}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			args := BuildArgs{ExtraFlags: tc.input}
+			flags := args.ParseBuildTags()
+			assert.EqualValues(t, tc.expect, flags)
+		})
+	}
+}

dev-tools/mage/crossbuild.go

@@ -249,6 +249,10 @@ func CrossBuildImage(platform string) (string, error) {
 		return "", err
 	}
 
+	if FIPSBuild {
+		tagSuffix += "-fips"
+	}
+
 	return BeatsCrossBuildImage + ":" + goVersion + "-" + tagSuffix, nil
 }
 
@@ -332,6 +336,7 @@ func (b GolangCrossBuilder) Build() error {
 		"--env", fmt.Sprintf("SNAPSHOT=%v", Snapshot),
 		"--env", fmt.Sprintf("DEV=%v", DevBuild),
 		"--env", fmt.Sprintf("EXTERNAL=%v", ExternalBuild),
+		"--env", fmt.Sprintf("FIPS=%v", FIPSBuild),
 		"-v", repoInfo.RootDir+":"+mountPoint,
 		"-w", workDir,
 		image,

dev-tools/mage/dockerbuilder.go

@@ -182,6 +182,9 @@ func (b *dockerBuilder) dockerBuild() (string, error) {
 	if b.Snapshot {
 		tag = tag + "-SNAPSHOT"
 	}
+	if b.FIPS {
+		tag = tag + "-fips"
+	}
 	if repository := b.ExtraVars["repository"]; repository != "" {
 		tag = fmt.Sprintf("%s/%s", repository, tag)
 	}

dev-tools/mage/gotest.go

@@ -187,9 +187,9 @@ func GoTestBuild(ctx context.Context, params GoTestArgs) error {
 	args := []string{"test", "-c", "-o", params.OutputFile}
 
 	if len(params.Tags) > 0 {
-		params := strings.Join(params.Tags, " ")
+		params := strings.Join(params.Tags, ",")
 		if params != "" {
-			args = append(args, "-tags", params)
+			args = append(args, "-tags="+params)
 		}
 	}
 

dev-tools/mage/pkg.go

@@ -80,6 +80,7 @@ func Package() error {
 				spec.OS = target.GOOS()
 				spec.Arch = packageArch
 				spec.Snapshot = Snapshot
+				spec.FIPS = FIPSBuild
 				spec.evalContext = map[string]interface{}{
 					"GOOS":          target.GOOS(),
 					"GOARCH":        target.GOARCH(),
@@ -148,11 +149,11 @@ type packageBuilder struct {
 }
 
 func (b packageBuilder) Build() error {
-	fmt.Printf(">> package: Building %v type=%v for platform=%v\n", b.Spec.Name, b.Type, b.Platform.Name)
+	fmt.Printf(">> package: Building %v type=%v for platform=%v fips=%v\n", b.Spec.Name, b.Type, b.Platform.Name, b.Spec.FIPS)
 	log.Printf("Package spec: %+v", b.Spec)
 	if err := b.Type.Build(b.Spec); err != nil {
-		return fmt.Errorf("failed building %v type=%v for platform=%v : %w",
-			b.Spec.Name, b.Type, b.Platform.Name, err)
+		return fmt.Errorf("failed building %v type=%v for platform=%v fips=%v : %w",
+			b.Spec.Name, b.Type, b.Platform.Name, b.Spec.FIPS, err)
 	}
 	return nil
 }
@@ -247,6 +248,10 @@ func TestPackages(options ...TestPackagesOption) error {
 		args = append(args, "-root-owner")
 	}
 
+	if FIPSBuild {
+		args = append(args, "-fips")
+	}
+
 	args = append(args, "-files", MustExpand("{{.PWD}}/build/distributions/*"))
 
 	if out, err := goTest(args...); err != nil {

dev-tools/mage/pkgtypes.go

@@ -39,13 +39,13 @@ const (
 	packageStagingDir = "build/package"
 
 	// defaultBinaryName specifies the output file for zip and tar.gz.
-	defaultBinaryName = "{{.Name}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}"
+	defaultBinaryName = "{{.Name}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}{{if .FIPS}}-fips{{end}}"
 
 	// defaultRootDir is the default name of the root directory contained inside of zip and
 	// tar.gz packages.
 	// NOTE: This uses .BeatName instead of .Name because we wanted the internal
 	// directory to not include "-oss".
-	defaultRootDir = "{{.BeatName}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}"
+	defaultRootDir = "{{.BeatName}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}{{if .FIPS}}-fips{{end}}"
 
 	componentConfigMode os.FileMode = 0600
 
@@ -92,6 +92,7 @@ type PackageSpec struct {
 	Arch              string                 `yaml:"arch,omitempty"`
 	Vendor            string                 `yaml:"vendor,omitempty"`
 	Snapshot          bool                   `yaml:"snapshot"`
+	FIPS              bool                   `yaml:"fips"`
 	Version           string                 `yaml:"version,omitempty"`
 	License           string                 `yaml:"license,omitempty"`
 	URL               string                 `yaml:"url,omitempty"`
@@ -732,7 +733,7 @@ func runFPM(spec PackageSpec, packageType PackageType) error {
 	}
 	defer os.Remove(inputTar)
 
-	outputFile, err := spec.Expand("{{.Name}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.Arch}}")
+	outputFile, err := spec.Expand("{{.Name}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.Arch}}{{if .FIPS}}-fips{{end}}")
 	if err != nil {
 		return err
 	}
@@ -962,7 +963,7 @@ func addFileToTar(ar *tar.Writer, baseDir string, pkgFile PackageFile) error {
 		}
 
 		if mg.Verbose() {
-			log.Println("Adding", os.FileMode(header.Mode), header.Name)
+			log.Println("Adding", os.FileMode(header.Mode), header.Name) //nolint:gosec // we don't care about an int overflow in a log line
 		}
 		if err := ar.WriteHeader(header); err != nil {
 			return err
@@ -1030,7 +1031,7 @@ func addSymlinkToTar(tmpdir string, ar *tar.Writer, baseDir string, pkgFile Pack
 		header.Typeflag = tar.TypeSymlink
 
 		if mg.Verbose() {
-			log.Println("Adding", os.FileMode(header.Mode), header.Name)
+			log.Println("Adding", os.FileMode(header.Mode), header.Name) //nolint:gosec // we don't care about an int overflow in a log line
 		}
 		if err := ar.WriteHeader(header); err != nil {
 			return err

dev-tools/mage/settings.go

@@ -88,6 +88,7 @@ var (
 	Snapshot      bool
 	DevBuild      bool
 	ExternalBuild bool
+	FIPSBuild     bool
 
 	versionQualified bool
 	versionQualifier string
@@ -153,6 +154,11 @@ func initGlobals() {
 		panic(fmt.Errorf("failed to parse EXTERNAL env value: %w", err))
 	}
 
+	FIPSBuild, err = strconv.ParseBool(EnvOr("FIPS", "false"))
+	if err != nil {
+		panic(fmt.Errorf("failed to parse FIPS env value: %w", err))
+	}
+
 	versionQualifier, versionQualified = os.LookupEnv("VERSION_QUALIFIER")
 
 	agentPackageVersion = EnvOr(agentPackageVersionEnvVar, "")
@@ -210,6 +216,7 @@ func varMap(args ...map[string]interface{}) map[string]interface{} {
 		"Snapshot":        Snapshot,
 		"DEV":             DevBuild,
 		"EXTERNAL":        ExternalBuild,
+		"FIPS":            FIPSBuild,
 		"Qualifier":       versionQualifier,
 		"CI":              CI,
 	}