DEVPROD-8937 Set a max number of intent hosts (#8820)

Author: hadjri

model/host/host.go

@@ -3136,6 +3136,17 @@ func CountSpawnhostsWithNoExpirationByUser(ctx context.Context, user string) (in
 	return Count(ctx, query)
 }
 
+// CountIntentHosts counts the number of intent hosts Evergreen will soon
+// attempt to create.
+func CountIntentHosts(ctx context.Context) (int, error) {
+	query := bson.M{
+		IdKey:        bson.M{"$regex": "^evg.*"},
+		StartedByKey: evergreen.User,
+		StatusKey:    bson.M{"$in": evergreen.UpHostStatus},
+	}
+	return Count(ctx, query)
+}
+
 // FindSpawnhostsWithNoExpirationToExtend returns all hosts that are set to never
 // expire but have their expiration time within the next day and are still up.
 func FindSpawnhostsWithNoExpirationToExtend(ctx context.Context) ([]Host, error) {

model/host/host_test.go

@@ -4622,6 +4622,44 @@ func TestCountSpawnhostsWithNoExpirationByUser(t *testing.T) {
 	assert.Equal(t, 0, count)
 }
 
+func TestCountIntentHosts(t *testing.T) {
+
+	require.NoError(t, db.ClearCollections(Collection))
+	hosts := []Host{
+		{
+			Id:        "evg-host-1",
+			Status:    evergreen.HostRunning,
+			StartedBy: evergreen.User,
+		},
+		{
+			Id:        "evg-host-2",
+			Status:    evergreen.HostRunning,
+			StartedBy: evergreen.User,
+		},
+		{
+			Id:        "evg-host-3",
+			Status:    evergreen.HostTerminated,
+			StartedBy: evergreen.User,
+		},
+		{
+			Id:        "host-4",
+			Status:    evergreen.HostRunning,
+			StartedBy: evergreen.User,
+		},
+		{
+			Id:        "host-5",
+			Status:    evergreen.HostStarting,
+			StartedBy: evergreen.User,
+		},
+	}
+	for _, h := range hosts {
+		assert.NoError(t, h.Insert(t.Context()))
+	}
+	count, err := CountIntentHosts(t.Context())
+	assert.NoError(t, err)
+	assert.Equal(t, 2, count)
+}
+
 func TestFindSpawnhostsWithNoExpirationToExtend(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

units/host_allocator.go

@@ -27,6 +27,9 @@ const (
 	hostAllocatorJobName         = "host-allocator"
 	hostAllocatorAttributePrefix = "evergreen.host_allocator"
 	maxHostAllocatorJobTime      = 10 * time.Minute
+	// maxIntentHosts represents the maximum number of intent hosts we can
+	// be processing at once, in order to prevent over-logging
+	maxIntentHosts = 5000
 )
 
 func init() {
@@ -198,6 +201,25 @@ func (j *hostAllocatorJob) Run(ctx context.Context) {
 	// host-spawning phase
 	//////////////////////
 
+	numIntentHosts, err := host.CountIntentHosts(ctx)
+	grip.Error(message.WrapError(err, message.Fields{
+		"runner":   hostAllocatorJobName,
+		"instance": j.ID(),
+		"distro":   j.DistroID,
+		"message":  "failed to count intent hosts",
+	}))
+
+	if numIntentHosts > maxIntentHosts {
+		grip.Info(message.Fields{
+			"runner":    hostAllocatorJobName,
+			"instance":  j.ID(),
+			"distro":    j.DistroID,
+			"message":   "too many intent hosts, skipping host allocation",
+			"max_hosts": maxIntentHosts,
+		})
+		return
+	}
+
 	hostSpawningBegins := time.Now()
 	// Number of new hosts to be allocated
 	hostsSpawned, err := scheduler.SpawnHosts(ctx, *distro, nHosts, containerPool)