Skip to content

Commit e225042

Browse files
aquinitorvalds
authored andcommitted
virtio_balloon: introduce migration primitives to balloon pages
Memory fragmentation introduced by ballooning might reduce significantly the number of 2MB contiguous memory blocks that can be used within a guest, thus imposing performance penalties associated with the reduced number of transparent huge pages that could be used by the guest workload. Besides making balloon pages movable at allocation time and introducing the necessary primitives to perform balloon page migration/compaction, this patch also introduces the following locking scheme, in order to enhance the syncronization methods for accessing elements of struct virtio_balloon, thus providing protection against concurrent access introduced by parallel memory migration threads. - balloon_lock (mutex) : synchronizes the access demand to elements of struct virtio_balloon and its queue operations; [yongjun_wei@trendmicro.com.cn: fix missing unlock on error in fill_balloon()] [akpm@linux-foundation.org: avoid having multiple return points in fill_balloon()] [akpm@linux-foundation.org: fix printk warning]Signed-off-by: Rafael Aquini <aquini@redhat.com> Acked-by: Michael S. Tsirkin <mst@redhat.com> Cc: Rusty Russell <rusty@rustcorp.com.au> Cc: Rik van Riel <riel@redhat.com> Cc: Mel Gorman <mel@csn.ul.ie> Cc: Andi Kleen <andi@firstfloor.org> Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> Cc: Minchan Kim <minchan@kernel.org> Signed-off-by: Wei Yongjun <yongjun_wei@trendmicro.com.cn> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
1 parent bf6bddf commit e225042

File tree

1 file changed

+132
-19
lines changed

1 file changed

+132
-19
lines changed

drivers/virtio/virtio_balloon.c

+132-19
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,15 @@
2727
#include <linux/delay.h>
2828
#include <linux/slab.h>
2929
#include <linux/module.h>
30+
#include <linux/balloon_compaction.h>
3031

3132
/*
3233
* Balloon device works in 4K page units. So each page is pointed to by
3334
* multiple balloon pages. All memory counters in this driver are in balloon
3435
* page units.
3536
*/
36-
#define VIRTIO_BALLOON_PAGES_PER_PAGE (PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT)
37+
#define VIRTIO_BALLOON_PAGES_PER_PAGE (unsigned)(PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT)
38+
#define VIRTIO_BALLOON_ARRAY_PFNS_MAX 256
3739

3840
struct virtio_balloon
3941
{
@@ -52,15 +54,19 @@ struct virtio_balloon
5254
/* Number of balloon pages we've told the Host we're not using. */
5355
unsigned int num_pages;
5456
/*
55-
* The pages we've told the Host we're not using.
57+
* The pages we've told the Host we're not using are enqueued
58+
* at vb_dev_info->pages list.
5659
* Each page on this list adds VIRTIO_BALLOON_PAGES_PER_PAGE
5760
* to num_pages above.
5861
*/
59-
struct list_head pages;
62+
struct balloon_dev_info *vb_dev_info;
63+
64+
/* Synchronize access/update to this struct virtio_balloon elements */
65+
struct mutex balloon_lock;
6066

6167
/* The array of pfns we tell the Host about. */
6268
unsigned int num_pfns;
63-
u32 pfns[256];
69+
u32 pfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX];
6470

6571
/* Memory statistics */
6672
int need_stats_update;
@@ -122,33 +128,34 @@ static void set_page_pfns(u32 pfns[], struct page *page)
122128

123129
static void fill_balloon(struct virtio_balloon *vb, size_t num)
124130
{
131+
struct balloon_dev_info *vb_dev_info = vb->vb_dev_info;
132+
125133
/* We can only do one array worth at a time. */
126134
num = min(num, ARRAY_SIZE(vb->pfns));
127135

136+
mutex_lock(&vb->balloon_lock);
128137
for (vb->num_pfns = 0; vb->num_pfns < num;
129138
vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
130-
struct page *page = alloc_page(GFP_HIGHUSER | __GFP_NORETRY |
131-
__GFP_NOMEMALLOC | __GFP_NOWARN);
139+
struct page *page = balloon_page_enqueue(vb_dev_info);
140+
132141
if (!page) {
133142
if (printk_ratelimit())
134143
dev_printk(KERN_INFO, &vb->vdev->dev,
135-
"Out of puff! Can't get %zu pages\n",
136-
num);
144+
"Out of puff! Can't get %u pages\n",
145+
VIRTIO_BALLOON_PAGES_PER_PAGE);
137146
/* Sleep for at least 1/5 of a second before retry. */
138147
msleep(200);
139148
break;
140149
}
141150
set_page_pfns(vb->pfns + vb->num_pfns, page);
142151
vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;
143152
totalram_pages--;
144-
list_add(&page->lru, &vb->pages);
145153
}
146154

147-
/* Didn't get any? Oh well. */
148-
if (vb->num_pfns == 0)
149-
return;
150-
151-
tell_host(vb, vb->inflate_vq);
155+
/* Did we get any? */
156+
if (vb->num_pfns != 0)
157+
tell_host(vb, vb->inflate_vq);
158+
mutex_unlock(&vb->balloon_lock);
152159
}
153160

154161
static void release_pages_by_pfn(const u32 pfns[], unsigned int num)
@@ -157,22 +164,25 @@ static void release_pages_by_pfn(const u32 pfns[], unsigned int num)
157164

158165
/* Find pfns pointing at start of each page, get pages and free them. */
159166
for (i = 0; i < num; i += VIRTIO_BALLOON_PAGES_PER_PAGE) {
160-
__free_page(balloon_pfn_to_page(pfns[i]));
167+
balloon_page_free(balloon_pfn_to_page(pfns[i]));
161168
totalram_pages++;
162169
}
163170
}
164171

165172
static void leak_balloon(struct virtio_balloon *vb, size_t num)
166173
{
167174
struct page *page;
175+
struct balloon_dev_info *vb_dev_info = vb->vb_dev_info;
168176

169177
/* We can only do one array worth at a time. */
170178
num = min(num, ARRAY_SIZE(vb->pfns));
171179

180+
mutex_lock(&vb->balloon_lock);
172181
for (vb->num_pfns = 0; vb->num_pfns < num;
173182
vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
174-
page = list_first_entry(&vb->pages, struct page, lru);
175-
list_del(&page->lru);
183+
page = balloon_page_dequeue(vb_dev_info);
184+
if (!page)
185+
break;
176186
set_page_pfns(vb->pfns + vb->num_pfns, page);
177187
vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE;
178188
}
@@ -183,6 +193,7 @@ static void leak_balloon(struct virtio_balloon *vb, size_t num)
183193
* is true, we *have* to do it in this order
184194
*/
185195
tell_host(vb, vb->deflate_vq);
196+
mutex_unlock(&vb->balloon_lock);
186197
release_pages_by_pfn(vb->pfns, vb->num_pfns);
187198
}
188199

@@ -339,9 +350,84 @@ static int init_vqs(struct virtio_balloon *vb)
339350
return 0;
340351
}
341352

353+
static const struct address_space_operations virtio_balloon_aops;
354+
#ifdef CONFIG_BALLOON_COMPACTION
355+
/*
356+
* virtballoon_migratepage - perform the balloon page migration on behalf of
357+
* a compation thread. (called under page lock)
358+
* @mapping: the page->mapping which will be assigned to the new migrated page.
359+
* @newpage: page that will replace the isolated page after migration finishes.
360+
* @page : the isolated (old) page that is about to be migrated to newpage.
361+
* @mode : compaction mode -- not used for balloon page migration.
362+
*
363+
* After a ballooned page gets isolated by compaction procedures, this is the
364+
* function that performs the page migration on behalf of a compaction thread
365+
* The page migration for virtio balloon is done in a simple swap fashion which
366+
* follows these two macro steps:
367+
* 1) insert newpage into vb->pages list and update the host about it;
368+
* 2) update the host about the old page removed from vb->pages list;
369+
*
370+
* This function preforms the balloon page migration task.
371+
* Called through balloon_mapping->a_ops->migratepage
372+
*/
373+
int virtballoon_migratepage(struct address_space *mapping,
374+
struct page *newpage, struct page *page, enum migrate_mode mode)
375+
{
376+
struct balloon_dev_info *vb_dev_info = balloon_page_device(page);
377+
struct virtio_balloon *vb;
378+
unsigned long flags;
379+
380+
BUG_ON(!vb_dev_info);
381+
382+
vb = vb_dev_info->balloon_device;
383+
384+
/*
385+
* In order to avoid lock contention while migrating pages concurrently
386+
* to leak_balloon() or fill_balloon() we just give up the balloon_lock
387+
* this turn, as it is easier to retry the page migration later.
388+
* This also prevents fill_balloon() getting stuck into a mutex
389+
* recursion in the case it ends up triggering memory compaction
390+
* while it is attempting to inflate the ballon.
391+
*/
392+
if (!mutex_trylock(&vb->balloon_lock))
393+
return -EAGAIN;
394+
395+
/* balloon's page migration 1st step -- inflate "newpage" */
396+
spin_lock_irqsave(&vb_dev_info->pages_lock, flags);
397+
balloon_page_insert(newpage, mapping, &vb_dev_info->pages);
398+
vb_dev_info->isolated_pages--;
399+
spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags);
400+
vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
401+
set_page_pfns(vb->pfns, newpage);
402+
tell_host(vb, vb->inflate_vq);
403+
404+
/*
405+
* balloon's page migration 2nd step -- deflate "page"
406+
*
407+
* It's safe to delete page->lru here because this page is at
408+
* an isolated migration list, and this step is expected to happen here
409+
*/
410+
balloon_page_delete(page);
411+
vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
412+
set_page_pfns(vb->pfns, page);
413+
tell_host(vb, vb->deflate_vq);
414+
415+
mutex_unlock(&vb->balloon_lock);
416+
417+
return MIGRATEPAGE_BALLOON_SUCCESS;
418+
}
419+
420+
/* define the balloon_mapping->a_ops callback to allow balloon page migration */
421+
static const struct address_space_operations virtio_balloon_aops = {
422+
.migratepage = virtballoon_migratepage,
423+
};
424+
#endif /* CONFIG_BALLOON_COMPACTION */
425+
342426
static int virtballoon_probe(struct virtio_device *vdev)
343427
{
344428
struct virtio_balloon *vb;
429+
struct address_space *vb_mapping;
430+
struct balloon_dev_info *vb_devinfo;
345431
int err;
346432

347433
vdev->priv = vb = kmalloc(sizeof(*vb), GFP_KERNEL);
@@ -350,16 +436,37 @@ static int virtballoon_probe(struct virtio_device *vdev)
350436
goto out;
351437
}
352438

353-
INIT_LIST_HEAD(&vb->pages);
354439
vb->num_pages = 0;
440+
mutex_init(&vb->balloon_lock);
355441
init_waitqueue_head(&vb->config_change);
356442
init_waitqueue_head(&vb->acked);
357443
vb->vdev = vdev;
358444
vb->need_stats_update = 0;
359445

446+
vb_devinfo = balloon_devinfo_alloc(vb);
447+
if (IS_ERR(vb_devinfo)) {
448+
err = PTR_ERR(vb_devinfo);
449+
goto out_free_vb;
450+
}
451+
452+
vb_mapping = balloon_mapping_alloc(vb_devinfo,
453+
(balloon_compaction_check()) ?
454+
&virtio_balloon_aops : NULL);
455+
if (IS_ERR(vb_mapping)) {
456+
/*
457+
* IS_ERR(vb_mapping) && PTR_ERR(vb_mapping) == -EOPNOTSUPP
458+
* This means !CONFIG_BALLOON_COMPACTION, otherwise we get off.
459+
*/
460+
err = PTR_ERR(vb_mapping);
461+
if (err != -EOPNOTSUPP)
462+
goto out_free_vb_devinfo;
463+
}
464+
465+
vb->vb_dev_info = vb_devinfo;
466+
360467
err = init_vqs(vb);
361468
if (err)
362-
goto out_free_vb;
469+
goto out_free_vb_mapping;
363470

364471
vb->thread = kthread_run(balloon, vb, "vballoon");
365472
if (IS_ERR(vb->thread)) {
@@ -371,6 +478,10 @@ static int virtballoon_probe(struct virtio_device *vdev)
371478

372479
out_del_vqs:
373480
vdev->config->del_vqs(vdev);
481+
out_free_vb_mapping:
482+
balloon_mapping_free(vb_mapping);
483+
out_free_vb_devinfo:
484+
balloon_devinfo_free(vb_devinfo);
374485
out_free_vb:
375486
kfree(vb);
376487
out:
@@ -396,6 +507,8 @@ static void __devexit virtballoon_remove(struct virtio_device *vdev)
396507

397508
kthread_stop(vb->thread);
398509
remove_common(vb);
510+
balloon_mapping_free(vb->vb_dev_info->mapping);
511+
balloon_devinfo_free(vb->vb_dev_info);
399512
kfree(vb);
400513
}
401514

0 commit comments

Comments
 (0)