Commit 54c6a6ba authored by David Hildenbrand's avatar David Hildenbrand Committed by Michael S. Tsirkin
Browse files

virito-mem: subblock states are specific to Sub Block Mode (SBM) 



Let's rename and move accordingly. While at it, rename sb_bitmap to
"sb_states".

Reviewed-by: default avatarWei Yang <richard.weiyang@linux.alibaba.com>
Reviewed-by: default avatarPankaj Gupta <pankaj.gupta.linux@gmail.com>
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: Jason Wang <jasowang@redhat.com>
Cc: Pankaj Gupta <pankaj.gupta.linux@gmail.com>
Signed-off-by: default avatarDavid Hildenbrand <david@redhat.com>
Link: https://lore.kernel.org/r/20201112133815.13332-20-david@redhat.com


Signed-off-by: default avatarMichael S. Tsirkin <mst@redhat.com>
parent 99f0b55e

drivers/virtio/virtio_mem.c

+69 −63
Original line number Diff line number Diff line
@@ -137,17 +137,23 @@ struct virtio_mem { 
		 * memory in one 4 KiB page.

		 */

		uint8_t *mb_states;

	} sbm;



		/*

	 * $nb_sb_per_mb bit per memory block. Handled similar to sbm.mb_states.

		 * Bitmap: one bit per subblock. Allocated similar to

		 * sbm.mb_states.

		 *

		 * A set bit means the corresponding subblock is plugged,

		 * otherwise it's unblocked.

		 *

	 * With 4MB subblocks, we manage 128GB of memory in one page.

		 * With 4 MiB subblocks, we manage 128 GiB of memory in one

		 * 4 KiB page.

		 */

	unsigned long *sb_bitmap;

		unsigned long *sb_states;

	} sbm;



	/*

	 * Mutex that protects the sbm.mb_count, sbm.mb_states, and sb_bitmap.

	 * Mutex that protects the sbm.mb_count, sbm.mb_states, and

	 * sbm.sb_states.

	 *

	 * When this lock is held the pointers can't change, ONLINE and

	 * OFFLINE blocks can't change the state and no subblocks will get
@@ -323,7 +329,7 @@ static int virtio_mem_sbm_mb_states_prepare_next_mb(struct virtio_mem *vm) 
 * Calculate the bit number in the subblock bitmap for the given subblock

 * inside the given memory block.

 */

static int virtio_mem_sb_bitmap_bit_nr(struct virtio_mem *vm,

static int virtio_mem_sbm_sb_state_bit_nr(struct virtio_mem *vm,

					  unsigned long mb_id, int sb_id)

{

	return (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
@@ -334,13 +340,13 @@ static int virtio_mem_sb_bitmap_bit_nr(struct virtio_mem *vm, 
 *

 * Will not modify the state of the memory block.

 */

static void virtio_mem_mb_set_sb_plugged(struct virtio_mem *vm,

static void virtio_mem_sbm_set_sb_plugged(struct virtio_mem *vm,

					  unsigned long mb_id, int sb_id,

					  int count)

{

	const int bit = virtio_mem_sb_bitmap_bit_nr(vm, mb_id, sb_id);

	const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id);



	__bitmap_set(vm->sb_bitmap, bit, count);

	__bitmap_set(vm->sbm.sb_states, bit, count);

}



/*
@@ -348,86 +354,87 @@ static void virtio_mem_mb_set_sb_plugged(struct virtio_mem *vm, 
 *

 * Will not modify the state of the memory block.

 */

static void virtio_mem_mb_set_sb_unplugged(struct virtio_mem *vm,

static void virtio_mem_sbm_set_sb_unplugged(struct virtio_mem *vm,

					    unsigned long mb_id, int sb_id,

					    int count)

{

	const int bit = virtio_mem_sb_bitmap_bit_nr(vm, mb_id, sb_id);

	const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id);



	__bitmap_clear(vm->sb_bitmap, bit, count);

	__bitmap_clear(vm->sbm.sb_states, bit, count);

}



/*

 * Test if all selected subblocks are plugged.

 */

static bool virtio_mem_mb_test_sb_plugged(struct virtio_mem *vm,

static bool virtio_mem_sbm_test_sb_plugged(struct virtio_mem *vm,

					   unsigned long mb_id, int sb_id,

					   int count)

{

	const int bit = virtio_mem_sb_bitmap_bit_nr(vm, mb_id, sb_id);

	const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id);



	if (count == 1)

		return test_bit(bit, vm->sb_bitmap);

		return test_bit(bit, vm->sbm.sb_states);



	/* TODO: Helper similar to bitmap_set() */

	return find_next_zero_bit(vm->sb_bitmap, bit + count, bit) >=

	return find_next_zero_bit(vm->sbm.sb_states, bit + count, bit) >=

	       bit + count;

}



/*

 * Test if all selected subblocks are unplugged.

 */

static bool virtio_mem_mb_test_sb_unplugged(struct virtio_mem *vm,

static bool virtio_mem_sbm_test_sb_unplugged(struct virtio_mem *vm,

					     unsigned long mb_id, int sb_id,

					     int count)

{

	const int bit = virtio_mem_sb_bitmap_bit_nr(vm, mb_id, sb_id);

	const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id);



	/* TODO: Helper similar to bitmap_set() */

	return find_next_bit(vm->sb_bitmap, bit + count, bit) >= bit + count;

	return find_next_bit(vm->sbm.sb_states, bit + count, bit) >=

	       bit + count;

}



/*

 * Find the first unplugged subblock. Returns vm->nb_sb_per_mb in case there is

 * none.

 */

static int virtio_mem_mb_first_unplugged_sb(struct virtio_mem *vm,

static int virtio_mem_sbm_first_unplugged_sb(struct virtio_mem *vm,

					    unsigned long mb_id)

{

	const int bit = virtio_mem_sb_bitmap_bit_nr(vm, mb_id, 0);

	const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, 0);



	return find_next_zero_bit(vm->sb_bitmap, bit + vm->nb_sb_per_mb, bit) -

	       bit;

	return find_next_zero_bit(vm->sbm.sb_states,

				  bit + vm->nb_sb_per_mb, bit) - bit;

}



/*

 * Prepare the subblock bitmap for the next memory block.

 */

static int virtio_mem_sb_bitmap_prepare_next_mb(struct virtio_mem *vm)

static int virtio_mem_sbm_sb_states_prepare_next_mb(struct virtio_mem *vm)

{

	const unsigned long old_nb_mb = vm->next_mb_id - vm->first_mb_id;

	const unsigned long old_nb_bits = old_nb_mb * vm->nb_sb_per_mb;

	const unsigned long new_nb_bits = (old_nb_mb + 1) * vm->nb_sb_per_mb;

	int old_pages = PFN_UP(BITS_TO_LONGS(old_nb_bits) * sizeof(long));

	int new_pages = PFN_UP(BITS_TO_LONGS(new_nb_bits) * sizeof(long));

	unsigned long *new_sb_bitmap, *old_sb_bitmap;

	unsigned long *new_bitmap, *old_bitmap;



	if (vm->sb_bitmap && old_pages == new_pages)

	if (vm->sbm.sb_states && old_pages == new_pages)

		return 0;



	new_sb_bitmap = vzalloc(new_pages * PAGE_SIZE);

	if (!new_sb_bitmap)

	new_bitmap = vzalloc(new_pages * PAGE_SIZE);

	if (!new_bitmap)

		return -ENOMEM;



	mutex_lock(&vm->hotplug_mutex);

	if (new_sb_bitmap)

		memcpy(new_sb_bitmap, vm->sb_bitmap, old_pages * PAGE_SIZE);

	if (new_bitmap)

		memcpy(new_bitmap, vm->sbm.sb_states, old_pages * PAGE_SIZE);



	old_sb_bitmap = vm->sb_bitmap;

	vm->sb_bitmap = new_sb_bitmap;

	old_bitmap = vm->sbm.sb_states;

	vm->sbm.sb_states = new_bitmap;

	mutex_unlock(&vm->hotplug_mutex);



	vfree(old_sb_bitmap);

	vfree(old_bitmap);

	return 0;

}
@@ -638,7 +645,7 @@ static void virtio_mem_notify_going_offline(struct virtio_mem *vm, 
	int sb_id;



	for (sb_id = 0; sb_id < vm->nb_sb_per_mb; sb_id++) {

		if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))

		if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1))

			continue;

		pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +

			       sb_id * vm->subblock_size);
@@ -654,7 +661,7 @@ static void virtio_mem_notify_cancel_offline(struct virtio_mem *vm, 
	int sb_id;



	for (sb_id = 0; sb_id < vm->nb_sb_per_mb; sb_id++) {

		if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))

		if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1))

			continue;

		pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +

			       sb_id * vm->subblock_size);
@@ -944,7 +951,7 @@ static void virtio_mem_online_page_cb(struct page *page, unsigned int order) 
		 * If plugged, online the pages, otherwise, set them fake

		 * offline (PageOffline).

		 */

		if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))

		if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1))

			generic_online_page(page, order);

		else

			virtio_mem_set_fake_offline(PFN_DOWN(addr), 1 << order,
@@ -1102,7 +1109,7 @@ static int virtio_mem_mb_plug_sb(struct virtio_mem *vm, unsigned long mb_id, 


	rc = virtio_mem_send_plug_request(vm, addr, size);

	if (!rc)

		virtio_mem_mb_set_sb_plugged(vm, mb_id, sb_id, count);

		virtio_mem_sbm_set_sb_plugged(vm, mb_id, sb_id, count);

	return rc;

}
@@ -1120,7 +1127,7 @@ static int virtio_mem_mb_unplug_sb(struct virtio_mem *vm, unsigned long mb_id, 


	rc = virtio_mem_send_unplug_request(vm, addr, size);

	if (!rc)

		virtio_mem_mb_set_sb_unplugged(vm, mb_id, sb_id, count);

		virtio_mem_sbm_set_sb_unplugged(vm, mb_id, sb_id, count);

	return rc;

}
@@ -1143,14 +1150,14 @@ static int virtio_mem_mb_unplug_any_sb(struct virtio_mem *vm, 
	while (*nb_sb) {

		/* Find the next candidate subblock */

		while (sb_id >= 0 &&

		       virtio_mem_mb_test_sb_unplugged(vm, mb_id, sb_id, 1))

		       virtio_mem_sbm_test_sb_unplugged(vm, mb_id, sb_id, 1))

			sb_id--;

		if (sb_id < 0)

			break;

		/* Try to unplug multiple subblocks at a time */

		count = 1;

		while (count < *nb_sb && sb_id > 0 &&

		       virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id - 1, 1)) {

		       virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id - 1, 1)) {

			count++;

			sb_id--;

		}
@@ -1196,7 +1203,7 @@ static int virtio_mem_prepare_next_mb(struct virtio_mem *vm, 
		return rc;



	/* Resize the subblock bitmap if required. */

	rc = virtio_mem_sb_bitmap_prepare_next_mb(vm);

	rc = virtio_mem_sbm_sb_states_prepare_next_mb(vm);

	if (rc)

		return rc;
@@ -1281,14 +1288,13 @@ static int virtio_mem_mb_plug_any_sb(struct virtio_mem *vm, unsigned long mb_id, 
		return -EINVAL;



	while (*nb_sb) {

		sb_id = virtio_mem_mb_first_unplugged_sb(vm, mb_id);

		sb_id = virtio_mem_sbm_first_unplugged_sb(vm, mb_id);

		if (sb_id >= vm->nb_sb_per_mb)

			break;

		count = 1;

		while (count < *nb_sb &&

		       sb_id + count < vm->nb_sb_per_mb &&

		       !virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id + count,

						      1))

		       !virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id + count, 1))

			count++;



		rc = virtio_mem_mb_plug_sb(vm, mb_id, sb_id, count);
@@ -1305,7 +1311,7 @@ static int virtio_mem_mb_plug_any_sb(struct virtio_mem *vm, unsigned long mb_id, 
		virtio_mem_fake_online(pfn, nr_pages);

	}



	if (virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {

	if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {

		if (online)

			virtio_mem_sbm_set_mb_state(vm, mb_id,

						    VIRTIO_MEM_SBM_MB_ONLINE);
@@ -1405,13 +1411,13 @@ static int virtio_mem_mb_unplug_any_sb_offline(struct virtio_mem *vm, 
	rc = virtio_mem_mb_unplug_any_sb(vm, mb_id, nb_sb);



	/* some subblocks might have been unplugged even on failure */

	if (!virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb))

	if (!virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb))

		virtio_mem_sbm_set_mb_state(vm, mb_id,

					    VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL);

	if (rc)

		return rc;



	if (virtio_mem_mb_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {

	if (virtio_mem_sbm_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {

		/*

		 * Remove the block from Linux - this should never fail.

		 * Hinder the block from getting onlined by marking it
@@ -1480,7 +1486,7 @@ static int virtio_mem_mb_unplug_any_sb_online(struct virtio_mem *vm, 


	/* If possible, try to unplug the complete block in one shot. */

	if (*nb_sb >= vm->nb_sb_per_mb &&

	    virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {

	    virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {

		rc = virtio_mem_mb_unplug_sb_online(vm, mb_id, 0,

						    vm->nb_sb_per_mb);

		if (!rc) {
@@ -1494,7 +1500,7 @@ static int virtio_mem_mb_unplug_any_sb_online(struct virtio_mem *vm, 
	for (sb_id = vm->nb_sb_per_mb - 1; sb_id >= 0 && *nb_sb; sb_id--) {

		/* Find the next candidate subblock */

		while (sb_id >= 0 &&

		       !virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))

		       !virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1))

			sb_id--;

		if (sb_id < 0)

			break;
@@ -1513,7 +1519,7 @@ static int virtio_mem_mb_unplug_any_sb_online(struct virtio_mem *vm, 
	 * remove it. This will usually not fail, as no memory is in use

	 * anymore - however some other notifiers might NACK the request.

	 */

	if (virtio_mem_mb_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {

	if (virtio_mem_sbm_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {

		mutex_unlock(&vm->hotplug_mutex);

		rc = virtio_mem_mb_offline_and_remove(vm, mb_id);

		mutex_lock(&vm->hotplug_mutex);
@@ -2035,7 +2041,7 @@ static void virtio_mem_remove(struct virtio_device *vdev) 


	/* remove all tracking data - no locking needed */

	vfree(vm->sbm.mb_states);

	vfree(vm->sb_bitmap);

	vfree(vm->sbm.sb_states);



	/* reset the device and cleanup the queues */

	vdev->config->reset(vdev);