Skip to content

Commit e480c0e

Browse files
jschwinger233jschwinger233
authored
cpu bind affinity during realloc (#293)
* realloc cpu affinity in scheduler * fix unittest * realloc ApplyChangesOnNode with index 0 * new interface ReselectCPUNodes
1 parent 3c409cd commit e480c0e

File tree

11 files changed

+350
-11
lines changed

11 files changed

+350
-11
lines changed

‎cluster/calcium/realloc.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ func (c *Calcium) ReallocResource(ctx context.Context, opts *types.ReallocOption
2020
CPUQuotaRequest: workload.CPUQuotaRequest + opts.ResourceOpts.CPUQuotaRequest,
2121
CPUQuotaLimit: workload.CPUQuotaLimit + opts.ResourceOpts.CPUQuotaLimit,
2222
CPUBind: types.ParseTriOption(opts.CPUBindOpts, len(workload.CPU) > 0),
23+
CPU: workload.CPU,
2324
MemoryRequest: workload.MemoryRequest + opts.ResourceOpts.MemoryRequest,
2425
MemoryLimit: workload.MemoryLimit + opts.ResourceOpts.MemoryLimit,
2526
StorageRequest: workload.StorageRequest + opts.ResourceOpts.StorageRequest,
@@ -68,7 +69,7 @@ func (c *Calcium) doReallocOnNode(ctx context.Context, nodename string, workload
6869
// then commit changes
6970
func(ctx context.Context) error {
7071
for _, plan := range plans {
71-
plan.ApplyChangesOnNode(node, 1)
72+
plan.ApplyChangesOnNode(node, 0)
7273
}
7374
return errors.WithStack(c.store.UpdateNodes(ctx, node))
7475
},

‎cluster/calcium/realloc_test.go

+7-6
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
enginemocks "github.com/projecteru2/core/engine/mocks"
99
enginetypes "github.com/projecteru2/core/engine/types"
1010
lockmocks "github.com/projecteru2/core/lock/mocks"
11+
resourcetypes "github.com/projecteru2/core/resources/types"
1112
"github.com/projecteru2/core/scheduler"
1213
complexscheduler "github.com/projecteru2/core/scheduler/complex"
1314
schedulermocks "github.com/projecteru2/core/scheduler/mocks"
@@ -120,14 +121,14 @@ func TestRealloc(t *testing.T) {
120121
simpleMockScheduler := &schedulermocks.Scheduler{}
121122
scheduler.InitSchedulerV1(simpleMockScheduler)
122123
c.scheduler = simpleMockScheduler
123-
simpleMockScheduler.On("SelectCPUNodes", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil, 0, types.ErrInsufficientMEM).Once()
124+
simpleMockScheduler.On("ReselectCPUNodes", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(resourcetypes.ScheduleInfo{}, nil, 0, types.ErrInsufficientMEM).Once()
124125
err = c.ReallocResource(ctx, newReallocOptions("c1", 0.1, 2*int64(units.MiB), nil, types.TriKeep, types.TriKeep))
125126
assert.EqualError(t, err, "cannot alloc a plan, not enough memory")
126127
store.AssertExpectations(t)
127128
simpleMockScheduler.AssertExpectations(t)
128129

129130
// failed by wrong total
130-
simpleMockScheduler.On("SelectCPUNodes", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil, 0, nil).Once()
131+
simpleMockScheduler.On("ReselectCPUNodes", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(resourcetypes.ScheduleInfo{}, nil, 0, nil).Once()
131132
simpleMockScheduler.On("SelectStorageNodes", mock.Anything, mock.Anything, mock.Anything).Return(nil, 100, nil)
132133
nodeVolumePlans := map[string][]types.VolumePlan{
133134
"node1": {{types.MustToVolumeBinding("AUTO:/data:rw:50"): types.VolumeMap{"/dir0": 50}}},
@@ -168,7 +169,7 @@ func TestRealloc(t *testing.T) {
168169
store.AssertExpectations(t)
169170

170171
// failed by update workload
171-
simpleMockScheduler.On("SelectCPUNodes", mock.Anything, mock.Anything, mock.Anything).Return(nil, nodeCPUPlans, 2, nil).Once()
172+
simpleMockScheduler.On("ReselectCPUNodes", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(resourcetypes.ScheduleInfo{}, nodeCPUPlans, 2, nil).Once()
172173
simpleMockScheduler.On("SelectVolumeNodes", mock.Anything, mock.Anything).Return(nil, nil, 100, nil).Once()
173174
engine.On("VirtualizationUpdateResource", mock.Anything, mock.Anything, mock.Anything).Return(nil)
174175
store.On("UpdateWorkload", mock.Anything, mock.Anything).Return(types.ErrBadWorkloadID).Times(1)
@@ -186,15 +187,15 @@ func TestRealloc(t *testing.T) {
186187
{types.MustToVolumeBinding("AUTO:/data:rw:100"): types.VolumeMap{"/dir4": 100}},
187188
},
188189
}
189-
simpleMockScheduler.On("SelectCPUNodes", mock.Anything, mock.Anything, mock.Anything).Return(nil, nodeCPUPlans, 2, nil).Once()
190+
simpleMockScheduler.On("ReselectCPUNodes", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(resourcetypes.ScheduleInfo{}, nodeCPUPlans, 2, nil).Once()
190191
simpleMockScheduler.On("SelectVolumeNodes", mock.Anything, mock.Anything).Return(nil, nodeVolumePlans, 4, nil).Once()
191192
err = c.ReallocResource(ctx, newReallocOptions("c1", 0.1, int64(units.MiB), types.MustToVolumeBindings([]string{"AUTO:/data:rw:50"}), types.TriKeep, types.TriKeep))
192193
assert.EqualError(t, err, "incompatible volume plans: cannot alloc a plan, not enough volume")
193194
simpleMockScheduler.AssertExpectations(t)
194195
store.AssertExpectations(t)
195196

196197
// failed by volume schedule error
197-
simpleMockScheduler.On("SelectCPUNodes", mock.Anything, mock.Anything, mock.Anything).Return(nil, nodeCPUPlans, 2, nil).Once()
198+
simpleMockScheduler.On("ReselectCPUNodes", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(resourcetypes.ScheduleInfo{}, nodeCPUPlans, 2, nil).Once()
198199
simpleMockScheduler.On("SelectVolumeNodes", mock.Anything, mock.Anything).Return(nil, nil, 0, types.ErrInsufficientVolume).Once()
199200
err = c.ReallocResource(ctx, newReallocOptions("c1", 0.1, int64(units.MiB), types.MustToVolumeBindings([]string{"AUTO:/data:rw:1"}), types.TriKeep, types.TriKeep))
200201
assert.EqualError(t, err, "cannot alloc a plan, not enough volume")
@@ -257,7 +258,7 @@ func TestRealloc(t *testing.T) {
257258
},
258259
},
259260
}
260-
simpleMockScheduler.On("SelectCPUNodes", mock.Anything, mock.Anything, mock.Anything).Return(nil, nodeCPUPlans, 2, nil)
261+
simpleMockScheduler.On("ReselectCPUNodes", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(resourcetypes.ScheduleInfo{}, nodeCPUPlans, 2, nil)
261262
simpleMockScheduler.On("SelectVolumeNodes", mock.Anything, mock.Anything).Return(nil, nodeVolumePlans, 2, nil)
262263
store.On("GetNode", mock.Anything, "node2").Return(node2, nil)
263264
store.On("GetWorkloads", mock.Anything, []string{"c3"}).Return([]*types.Workload{c3}, nil)

‎resources/cpumem/cpumem.go

+7-2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ type cpuMemRequest struct {
1111
CPUQuotaRequest float64
1212
CPUQuotaLimit float64
1313
CPUBind bool
14+
CPU types.CPUMap
1415

1516
memoryRequest int64
1617
memoryLimit int64
@@ -24,6 +25,7 @@ func MakeRequest(opts types.ResourceOptions) (resourcetypes.ResourceRequest, err
2425
CPUBind: opts.CPUBind,
2526
memoryRequest: opts.MemoryRequest,
2627
memoryLimit: opts.MemoryLimit,
28+
CPU: opts.CPU,
2729
}
2830
return cmr, cmr.Validate()
2931
}
@@ -74,9 +76,12 @@ func (cm cpuMemRequest) MakeScheduler() resourcetypes.SchedulerV2 {
7476
}
7577

7678
var CPUPlans map[string][]types.CPUMap
77-
if !cm.CPUBind || cm.CPUQuotaRequest == 0 {
79+
switch {
80+
case !cm.CPUBind || cm.CPUQuotaRequest == 0:
7881
scheduleInfos, total, err = schedulerV1.SelectMemoryNodes(scheduleInfos, cm.CPUQuotaRequest, cm.memoryRequest)
79-
} else {
82+
case cm.CPU != nil:
83+
scheduleInfos[0], CPUPlans, total, err = schedulerV1.ReselectCPUNodes(scheduleInfos[0], cm.CPU, cm.CPUQuotaRequest, cm.memoryRequest)
84+
default:
8085
scheduleInfos, CPUPlans, total, err = schedulerV1.SelectCPUNodes(scheduleInfos, cm.CPUQuotaRequest, cm.memoryRequest)
8186
}
8287
return ResourcePlans{

‎resources/cpumem/cpumem_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@ func (test *requestCPUNodeTest) getScheduleInfo() []resourcetypes.ScheduleInfo {
206206
func (test *requestCPUNodeTest) getScheduler() scheduler.Scheduler {
207207
mockScheduler := &schedulerMocks.Scheduler{}
208208
mockScheduler.On(
209-
"SelectCPUNodes", mock.Anything, mock.Anything, mock.Anything,
209+
"SelectCPUNodes", mock.Anything, mock.Anything, mock.Anything, mock.Anything,
210210
).Return(test.scheduleInfos, test.cpuMap, 1, nil)
211211
mockScheduler.On(
212212
"SelectMemoryNodess", mock.Anything, mock.Anything, mock.Anything,
@@ -280,7 +280,7 @@ func (test *requestMemNodeTest) getScheduleInfo() []resourcetypes.ScheduleInfo {
280280
func (test *requestMemNodeTest) getScheduler() scheduler.Scheduler {
281281
mockScheduler := &schedulerMocks.Scheduler{}
282282
mockScheduler.On(
283-
"SelectCPUNodes", mock.Anything, mock.Anything, mock.Anything,
283+
"SelectCPUNodes", mock.Anything, mock.Anything, mock.Anything, mock.Anything,
284284
).Return(test.scheduleInfos, nil, 1, errors.New("should not select memory node here"))
285285
mockScheduler.On(
286286
"SelectMemoryNodes", mock.Anything, mock.Anything, mock.Anything,

‎scheduler/complex/cpu_test.go

+175
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package complexscheduler
22

33
import (
4+
"reflect"
45
"testing"
56

67
"github.com/docker/go-units"
@@ -60,3 +61,177 @@ func resetscheduleInfos() []resourcetypes.ScheduleInfo {
6061
},
6162
}
6263
}
64+
65+
func TestCPUReallocPlan(t *testing.T) {
66+
// shrink: 1.7->1.0
67+
scheduleInfo := resourcetypes.ScheduleInfo{
68+
NodeMeta: types.NodeMeta{
69+
Name: "n1",
70+
CPU: types.CPUMap{
71+
"0": 0,
72+
"1": 30,
73+
"2": 0,
74+
},
75+
},
76+
}
77+
CPU := types.CPUMap{
78+
"0": 100,
79+
"1": 30,
80+
"2": 40,
81+
}
82+
si, remain, aff := cpuReallocPlan(scheduleInfo, 1, CPU, 100)
83+
assert.EqualValues(t, 0, remain)
84+
assert.True(t, reflect.DeepEqual(aff, types.CPUMap{"0": 100}))
85+
assert.True(t, reflect.DeepEqual(si.CPU, types.CPUMap{"0": 0, "1": 60, "2": 40}))
86+
87+
// shrink: 1.7->1.2
88+
scheduleInfo = resourcetypes.ScheduleInfo{
89+
NodeMeta: types.NodeMeta{
90+
Name: "n1",
91+
CPU: types.CPUMap{
92+
"0": 0,
93+
"1": 30,
94+
"2": 0,
95+
},
96+
},
97+
}
98+
CPU = types.CPUMap{
99+
"0": 100,
100+
"1": 30,
101+
"2": 40,
102+
}
103+
si, remain, aff = cpuReallocPlan(scheduleInfo, 1.2, CPU, 100)
104+
assert.EqualValues(t, 0, remain)
105+
assert.True(t, reflect.DeepEqual(aff, types.CPUMap{"0": 100, "2": 20}))
106+
assert.True(t, reflect.DeepEqual(si.CPU, types.CPUMap{"0": 0, "1": 60, "2": 20}))
107+
108+
// expand: 1.7->2, find complement
109+
scheduleInfo = resourcetypes.ScheduleInfo{
110+
NodeMeta: types.NodeMeta{
111+
Name: "n1",
112+
CPU: types.CPUMap{
113+
"0": 0,
114+
"1": 80,
115+
"2": 0,
116+
"3": 0,
117+
},
118+
},
119+
}
120+
CPU = types.CPUMap{
121+
"0": 100,
122+
"1": 20,
123+
"2": 40,
124+
"3": 10,
125+
}
126+
si, remain, aff = cpuReallocPlan(scheduleInfo, 2, CPU, 100)
127+
assert.EqualValues(t, 0, remain)
128+
assert.True(t, reflect.DeepEqual(aff, types.CPUMap{"0": 100, "1": 100}))
129+
assert.True(t, reflect.DeepEqual(si.CPU, types.CPUMap{"0": 0, "1": 0, "2": 40, "3": 10}))
130+
131+
// expand: 1.7->2, lose complement
132+
scheduleInfo = resourcetypes.ScheduleInfo{
133+
NodeMeta: types.NodeMeta{
134+
Name: "n1",
135+
CPU: types.CPUMap{
136+
"0": 0,
137+
"1": 69,
138+
"2": 10,
139+
},
140+
},
141+
}
142+
CPU = types.CPUMap{
143+
"0": 100,
144+
"1": 30,
145+
"2": 40,
146+
}
147+
si, remain, aff = cpuReallocPlan(scheduleInfo, 2, CPU, 100)
148+
assert.EqualValues(t, 1, remain)
149+
assert.True(t, reflect.DeepEqual(aff, types.CPUMap{"0": 100}))
150+
assert.True(t, reflect.DeepEqual(si.CPU, types.CPUMap{"0": 0, "1": 99, "2": 50}))
151+
}
152+
153+
func TestCPUReallocWithPriorPlan(t *testing.T) {
154+
po, err := New(types.Config{Scheduler: types.SchedConfig{
155+
MaxShare: 0,
156+
ShareBase: 100,
157+
}})
158+
assert.Nil(t, err)
159+
160+
// direct return after realloc plan
161+
scheduleInfo := resourcetypes.ScheduleInfo{
162+
NodeMeta: types.NodeMeta{
163+
Name: "n1",
164+
CPU: types.CPUMap{
165+
"0": 0,
166+
"1": 70,
167+
"2": 0,
168+
},
169+
},
170+
}
171+
CPU := types.CPUMap{
172+
"0": 100,
173+
"1": 30,
174+
"2": 40,
175+
}
176+
si, cpuPlans, total, err := po.ReselectCPUNodes(scheduleInfo, CPU, 2, 0)
177+
assert.Nil(t, err)
178+
assert.EqualValues(t, 1, total)
179+
assert.True(t, reflect.DeepEqual(cpuPlans, map[string][]types.CPUMap{"n1": {{"0": 100, "1": 100}}}))
180+
assert.EqualValues(t, 1, si.Capacity)
181+
182+
// realloc plan + cpu prior plan
183+
scheduleInfo = resourcetypes.ScheduleInfo{
184+
NodeMeta: types.NodeMeta{
185+
Name: "n1",
186+
CPU: types.CPUMap{
187+
"0": 100,
188+
"1": 60,
189+
"2": 0,
190+
"3": 100,
191+
"4": 100,
192+
},
193+
},
194+
}
195+
CPU = types.CPUMap{
196+
"0": 100,
197+
"1": 30,
198+
"2": 40,
199+
}
200+
si, cpuPlans, total, err = po.ReselectCPUNodes(scheduleInfo, CPU, 2, 0)
201+
assert.Nil(t, err)
202+
assert.EqualValues(t, 3, total)
203+
asserted := 0
204+
for _, plan := range cpuPlans["n1"] {
205+
if _, ok := plan["3"]; ok {
206+
assert.True(t, reflect.DeepEqual(plan, types.CPUMap{"0": 100, "3": 100}))
207+
asserted++
208+
} else if _, ok := plan["4"]; ok {
209+
assert.True(t, reflect.DeepEqual(plan, types.CPUMap{"0": 100, "4": 100}))
210+
asserted++
211+
} else {
212+
assert.True(t, reflect.DeepEqual(plan, types.CPUMap{"0": 200}))
213+
asserted++
214+
}
215+
}
216+
assert.EqualValues(t, 3, asserted)
217+
assert.EqualValues(t, 3, si.Capacity)
218+
219+
// realloc plan + cpu prior error
220+
scheduleInfo = resourcetypes.ScheduleInfo{
221+
NodeMeta: types.NodeMeta{
222+
Name: "n1",
223+
CPU: types.CPUMap{
224+
"0": 0,
225+
"1": 60,
226+
"2": 0,
227+
},
228+
},
229+
}
230+
CPU = types.CPUMap{
231+
"0": 100,
232+
"1": 30,
233+
"2": 40,
234+
}
235+
_, _, _, err = po.ReselectCPUNodes(scheduleInfo, CPU, 2, 0)
236+
assert.EqualError(t, err, "not enough resource")
237+
}

0 commit comments

Comments
 (0)