27
27
#include <linux/delay.h>
28
28
#include <linux/slab.h>
29
29
#include <linux/module.h>
30
+ #include <linux/balloon_compaction.h>
30
31
31
32
/*
32
33
* Balloon device works in 4K page units. So each page is pointed to by
33
34
* multiple balloon pages. All memory counters in this driver are in balloon
34
35
* page units.
35
36
*/
36
- #define VIRTIO_BALLOON_PAGES_PER_PAGE (PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT)
37
+ #define VIRTIO_BALLOON_PAGES_PER_PAGE (unsigned)(PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT)
38
+ #define VIRTIO_BALLOON_ARRAY_PFNS_MAX 256
37
39
38
40
struct virtio_balloon
39
41
{
@@ -52,15 +54,19 @@ struct virtio_balloon
52
54
/* Number of balloon pages we've told the Host we're not using. */
53
55
unsigned int num_pages ;
54
56
/*
55
- * The pages we've told the Host we're not using.
57
+ * The pages we've told the Host we're not using are enqueued
58
+ * at vb_dev_info->pages list.
56
59
* Each page on this list adds VIRTIO_BALLOON_PAGES_PER_PAGE
57
60
* to num_pages above.
58
61
*/
59
- struct list_head pages ;
62
+ struct balloon_dev_info * vb_dev_info ;
63
+
64
+ /* Synchronize access/update to this struct virtio_balloon elements */
65
+ struct mutex balloon_lock ;
60
66
61
67
/* The array of pfns we tell the Host about. */
62
68
unsigned int num_pfns ;
63
- u32 pfns [256 ];
69
+ u32 pfns [VIRTIO_BALLOON_ARRAY_PFNS_MAX ];
64
70
65
71
/* Memory statistics */
66
72
int need_stats_update ;
@@ -122,33 +128,34 @@ static void set_page_pfns(u32 pfns[], struct page *page)
122
128
123
129
static void fill_balloon (struct virtio_balloon * vb , size_t num )
124
130
{
131
+ struct balloon_dev_info * vb_dev_info = vb -> vb_dev_info ;
132
+
125
133
/* We can only do one array worth at a time. */
126
134
num = min (num , ARRAY_SIZE (vb -> pfns ));
127
135
136
+ mutex_lock (& vb -> balloon_lock );
128
137
for (vb -> num_pfns = 0 ; vb -> num_pfns < num ;
129
138
vb -> num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE ) {
130
- struct page * page = alloc_page ( GFP_HIGHUSER | __GFP_NORETRY |
131
- __GFP_NOMEMALLOC | __GFP_NOWARN );
139
+ struct page * page = balloon_page_enqueue ( vb_dev_info );
140
+
132
141
if (!page ) {
133
142
if (printk_ratelimit ())
134
143
dev_printk (KERN_INFO , & vb -> vdev -> dev ,
135
- "Out of puff! Can't get %zu pages\n" ,
136
- num );
144
+ "Out of puff! Can't get %u pages\n" ,
145
+ VIRTIO_BALLOON_PAGES_PER_PAGE );
137
146
/* Sleep for at least 1/5 of a second before retry. */
138
147
msleep (200 );
139
148
break ;
140
149
}
141
150
set_page_pfns (vb -> pfns + vb -> num_pfns , page );
142
151
vb -> num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE ;
143
152
totalram_pages -- ;
144
- list_add (& page -> lru , & vb -> pages );
145
153
}
146
154
147
- /* Didn't get any? Oh well. */
148
- if (vb -> num_pfns == 0 )
149
- return ;
150
-
151
- tell_host (vb , vb -> inflate_vq );
155
+ /* Did we get any? */
156
+ if (vb -> num_pfns != 0 )
157
+ tell_host (vb , vb -> inflate_vq );
158
+ mutex_unlock (& vb -> balloon_lock );
152
159
}
153
160
154
161
static void release_pages_by_pfn (const u32 pfns [], unsigned int num )
@@ -157,22 +164,25 @@ static void release_pages_by_pfn(const u32 pfns[], unsigned int num)
157
164
158
165
/* Find pfns pointing at start of each page, get pages and free them. */
159
166
for (i = 0 ; i < num ; i += VIRTIO_BALLOON_PAGES_PER_PAGE ) {
160
- __free_page (balloon_pfn_to_page (pfns [i ]));
167
+ balloon_page_free (balloon_pfn_to_page (pfns [i ]));
161
168
totalram_pages ++ ;
162
169
}
163
170
}
164
171
165
172
static void leak_balloon (struct virtio_balloon * vb , size_t num )
166
173
{
167
174
struct page * page ;
175
+ struct balloon_dev_info * vb_dev_info = vb -> vb_dev_info ;
168
176
169
177
/* We can only do one array worth at a time. */
170
178
num = min (num , ARRAY_SIZE (vb -> pfns ));
171
179
180
+ mutex_lock (& vb -> balloon_lock );
172
181
for (vb -> num_pfns = 0 ; vb -> num_pfns < num ;
173
182
vb -> num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE ) {
174
- page = list_first_entry (& vb -> pages , struct page , lru );
175
- list_del (& page -> lru );
183
+ page = balloon_page_dequeue (vb_dev_info );
184
+ if (!page )
185
+ break ;
176
186
set_page_pfns (vb -> pfns + vb -> num_pfns , page );
177
187
vb -> num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE ;
178
188
}
@@ -183,6 +193,7 @@ static void leak_balloon(struct virtio_balloon *vb, size_t num)
183
193
* is true, we *have* to do it in this order
184
194
*/
185
195
tell_host (vb , vb -> deflate_vq );
196
+ mutex_unlock (& vb -> balloon_lock );
186
197
release_pages_by_pfn (vb -> pfns , vb -> num_pfns );
187
198
}
188
199
@@ -339,9 +350,84 @@ static int init_vqs(struct virtio_balloon *vb)
339
350
return 0 ;
340
351
}
341
352
353
+ static const struct address_space_operations virtio_balloon_aops ;
354
+ #ifdef CONFIG_BALLOON_COMPACTION
355
+ /*
356
+ * virtballoon_migratepage - perform the balloon page migration on behalf of
357
+ * a compation thread. (called under page lock)
358
+ * @mapping: the page->mapping which will be assigned to the new migrated page.
359
+ * @newpage: page that will replace the isolated page after migration finishes.
360
+ * @page : the isolated (old) page that is about to be migrated to newpage.
361
+ * @mode : compaction mode -- not used for balloon page migration.
362
+ *
363
+ * After a ballooned page gets isolated by compaction procedures, this is the
364
+ * function that performs the page migration on behalf of a compaction thread
365
+ * The page migration for virtio balloon is done in a simple swap fashion which
366
+ * follows these two macro steps:
367
+ * 1) insert newpage into vb->pages list and update the host about it;
368
+ * 2) update the host about the old page removed from vb->pages list;
369
+ *
370
+ * This function preforms the balloon page migration task.
371
+ * Called through balloon_mapping->a_ops->migratepage
372
+ */
373
+ int virtballoon_migratepage (struct address_space * mapping ,
374
+ struct page * newpage , struct page * page , enum migrate_mode mode )
375
+ {
376
+ struct balloon_dev_info * vb_dev_info = balloon_page_device (page );
377
+ struct virtio_balloon * vb ;
378
+ unsigned long flags ;
379
+
380
+ BUG_ON (!vb_dev_info );
381
+
382
+ vb = vb_dev_info -> balloon_device ;
383
+
384
+ /*
385
+ * In order to avoid lock contention while migrating pages concurrently
386
+ * to leak_balloon() or fill_balloon() we just give up the balloon_lock
387
+ * this turn, as it is easier to retry the page migration later.
388
+ * This also prevents fill_balloon() getting stuck into a mutex
389
+ * recursion in the case it ends up triggering memory compaction
390
+ * while it is attempting to inflate the ballon.
391
+ */
392
+ if (!mutex_trylock (& vb -> balloon_lock ))
393
+ return - EAGAIN ;
394
+
395
+ /* balloon's page migration 1st step -- inflate "newpage" */
396
+ spin_lock_irqsave (& vb_dev_info -> pages_lock , flags );
397
+ balloon_page_insert (newpage , mapping , & vb_dev_info -> pages );
398
+ vb_dev_info -> isolated_pages -- ;
399
+ spin_unlock_irqrestore (& vb_dev_info -> pages_lock , flags );
400
+ vb -> num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE ;
401
+ set_page_pfns (vb -> pfns , newpage );
402
+ tell_host (vb , vb -> inflate_vq );
403
+
404
+ /*
405
+ * balloon's page migration 2nd step -- deflate "page"
406
+ *
407
+ * It's safe to delete page->lru here because this page is at
408
+ * an isolated migration list, and this step is expected to happen here
409
+ */
410
+ balloon_page_delete (page );
411
+ vb -> num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE ;
412
+ set_page_pfns (vb -> pfns , page );
413
+ tell_host (vb , vb -> deflate_vq );
414
+
415
+ mutex_unlock (& vb -> balloon_lock );
416
+
417
+ return MIGRATEPAGE_BALLOON_SUCCESS ;
418
+ }
419
+
420
+ /* define the balloon_mapping->a_ops callback to allow balloon page migration */
421
+ static const struct address_space_operations virtio_balloon_aops = {
422
+ .migratepage = virtballoon_migratepage ,
423
+ };
424
+ #endif /* CONFIG_BALLOON_COMPACTION */
425
+
342
426
static int virtballoon_probe (struct virtio_device * vdev )
343
427
{
344
428
struct virtio_balloon * vb ;
429
+ struct address_space * vb_mapping ;
430
+ struct balloon_dev_info * vb_devinfo ;
345
431
int err ;
346
432
347
433
vdev -> priv = vb = kmalloc (sizeof (* vb ), GFP_KERNEL );
@@ -350,16 +436,37 @@ static int virtballoon_probe(struct virtio_device *vdev)
350
436
goto out ;
351
437
}
352
438
353
- INIT_LIST_HEAD (& vb -> pages );
354
439
vb -> num_pages = 0 ;
440
+ mutex_init (& vb -> balloon_lock );
355
441
init_waitqueue_head (& vb -> config_change );
356
442
init_waitqueue_head (& vb -> acked );
357
443
vb -> vdev = vdev ;
358
444
vb -> need_stats_update = 0 ;
359
445
446
+ vb_devinfo = balloon_devinfo_alloc (vb );
447
+ if (IS_ERR (vb_devinfo )) {
448
+ err = PTR_ERR (vb_devinfo );
449
+ goto out_free_vb ;
450
+ }
451
+
452
+ vb_mapping = balloon_mapping_alloc (vb_devinfo ,
453
+ (balloon_compaction_check ()) ?
454
+ & virtio_balloon_aops : NULL );
455
+ if (IS_ERR (vb_mapping )) {
456
+ /*
457
+ * IS_ERR(vb_mapping) && PTR_ERR(vb_mapping) == -EOPNOTSUPP
458
+ * This means !CONFIG_BALLOON_COMPACTION, otherwise we get off.
459
+ */
460
+ err = PTR_ERR (vb_mapping );
461
+ if (err != - EOPNOTSUPP )
462
+ goto out_free_vb_devinfo ;
463
+ }
464
+
465
+ vb -> vb_dev_info = vb_devinfo ;
466
+
360
467
err = init_vqs (vb );
361
468
if (err )
362
- goto out_free_vb ;
469
+ goto out_free_vb_mapping ;
363
470
364
471
vb -> thread = kthread_run (balloon , vb , "vballoon" );
365
472
if (IS_ERR (vb -> thread )) {
@@ -371,6 +478,10 @@ static int virtballoon_probe(struct virtio_device *vdev)
371
478
372
479
out_del_vqs :
373
480
vdev -> config -> del_vqs (vdev );
481
+ out_free_vb_mapping :
482
+ balloon_mapping_free (vb_mapping );
483
+ out_free_vb_devinfo :
484
+ balloon_devinfo_free (vb_devinfo );
374
485
out_free_vb :
375
486
kfree (vb );
376
487
out :
@@ -396,6 +507,8 @@ static void __devexit virtballoon_remove(struct virtio_device *vdev)
396
507
397
508
kthread_stop (vb -> thread );
398
509
remove_common (vb );
510
+ balloon_mapping_free (vb -> vb_dev_info -> mapping );
511
+ balloon_devinfo_free (vb -> vb_dev_info );
399
512
kfree (vb );
400
513
}
401
514
0 commit comments