Merge back system-wide sleep material for v6.6.

	struct mem_zone_bm_rtree *zone;

	struct rtree_node *node;

	unsigned long node_pfn;

	unsigned long cur_pfn;

	int node_bit;

};

	bm->cur.node = list_entry(bm->cur.zone->leaves.next,

				  struct rtree_node, list);

	bm->cur.node_pfn = 0;

	bm->cur.cur_pfn = BM_END_OF_MAP;

	bm->cur.node_bit = 0;

}

	bm->cur.zone = zone;

	bm->cur.node = node;

	bm->cur.node_pfn = (pfn - zone->start_pfn) & ~BM_BLOCK_MASK;

	bm->cur.cur_pfn = pfn;

	/* Set return values */

	*addr = node->data;

	clear_bit(bit, bm->cur.node->data);

}

static unsigned long memory_bm_get_current(struct memory_bitmap *bm)

{

	return bm->cur.cur_pfn;

}

static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn)

{

	void *addr;

		if (bit < bits) {

			pfn = bm->cur.zone->start_pfn + bm->cur.node_pfn + bit;

			bm->cur.node_bit = bit + 1;

			bm->cur.cur_pfn = pfn;

			return pfn;

		}

	} while (rtree_next_node(bm));

	bm->cur.cur_pfn = BM_END_OF_MAP;

	return BM_END_OF_MAP;

}

/*

 * This is needed, because copy_page and memcpy are not usable for copying

 * task structs.

 * task structs. Returns true if the page was filled with only zeros,

 * otherwise false.

 */

static inline void do_copy_page(long *dst, long *src)

static inline bool do_copy_page(long *dst, long *src)

{

	long z = 0;

	int n;

	for (n = PAGE_SIZE / sizeof(long); n; n--)

	for (n = PAGE_SIZE / sizeof(long); n; n--) {

		z |= *src;

		*dst++ = *src++;

	}

	return !z;

}

/**

 * safe_copy_page - Copy a page in a safe way.

 * Check if the page we are going to copy is marked as present in the kernel

 * page tables. This always is the case if CONFIG_DEBUG_PAGEALLOC or

 * CONFIG_ARCH_HAS_SET_DIRECT_MAP is not set. In that case kernel_page_present()

 * always returns 'true'.

 * always returns 'true'. Returns true if the page was entirely composed of

 * zeros, otherwise it will return false.

 */

static void safe_copy_page(void *dst, struct page *s_page)

static bool safe_copy_page(void *dst, struct page *s_page)

{

	bool zeros_only;

	if (kernel_page_present(s_page)) {

		do_copy_page(dst, page_address(s_page));

		zeros_only = do_copy_page(dst, page_address(s_page));

	} else {

		hibernate_map_page(s_page);

		do_copy_page(dst, page_address(s_page));

		zeros_only = do_copy_page(dst, page_address(s_page));

		hibernate_unmap_page(s_page);

	}

	return zeros_only;

}

#ifdef CONFIG_HIGHMEM

		saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn);

}

static void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)

static bool copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)

{

	struct page *s_page, *d_page;

	void *src, *dst;

	bool zeros_only;

	s_page = pfn_to_page(src_pfn);

	d_page = pfn_to_page(dst_pfn);

	if (PageHighMem(s_page)) {

		src = kmap_atomic(s_page);

		dst = kmap_atomic(d_page);

		do_copy_page(dst, src);

		zeros_only = do_copy_page(dst, src);

		kunmap_atomic(dst);

		kunmap_atomic(src);

	} else {

			 * The page pointed to by src may contain some kernel

			 * data modified by kmap_atomic()

			 */

			safe_copy_page(buffer, s_page);

			zeros_only = safe_copy_page(buffer, s_page);

			dst = kmap_atomic(d_page);

			copy_page(dst, buffer);

			kunmap_atomic(dst);

		} else {

			safe_copy_page(page_address(d_page), s_page);

			zeros_only = safe_copy_page(page_address(d_page), s_page);

		}

	}

	return zeros_only;

}

#else

#define page_is_saveable(zone, pfn)	saveable_page(zone, pfn)

static inline void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)

static inline int copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)

{

	safe_copy_page(page_address(pfn_to_page(dst_pfn)),

	return safe_copy_page(page_address(pfn_to_page(dst_pfn)),

				pfn_to_page(src_pfn));

}

#endif /* CONFIG_HIGHMEM */

static void copy_data_pages(struct memory_bitmap *copy_bm,

			    struct memory_bitmap *orig_bm)

/*

 * Copy data pages will copy all pages into pages pulled from the copy_bm.

 * If a page was entirely filled with zeros it will be marked in the zero_bm.

 *

 * Returns the number of pages copied.

 */

static unsigned long copy_data_pages(struct memory_bitmap *copy_bm,

			    struct memory_bitmap *orig_bm,

			    struct memory_bitmap *zero_bm)

{

	unsigned long copied_pages = 0;

	struct zone *zone;

	unsigned long pfn;

	unsigned long pfn, copy_pfn;

	for_each_populated_zone(zone) {

		unsigned long max_zone_pfn;

	}

	memory_bm_position_reset(orig_bm);

	memory_bm_position_reset(copy_bm);

	copy_pfn = memory_bm_next_pfn(copy_bm);

	for(;;) {

		pfn = memory_bm_next_pfn(orig_bm);

		if (unlikely(pfn == BM_END_OF_MAP))

			break;

		copy_data_page(memory_bm_next_pfn(copy_bm), pfn);

		if (copy_data_page(copy_pfn, pfn)) {

			memory_bm_set_bit(zero_bm, pfn);

			/* Use this copy_pfn for a page that is not full of zeros */

			continue;

		}

		copied_pages++;

		copy_pfn = memory_bm_next_pfn(copy_bm);

	}

	return copied_pages;

}

/* Total number of image pages */

static unsigned int nr_copy_pages;

/* Number of pages needed for saving the original pfns of the image pages */

static unsigned int nr_meta_pages;

/* Number of zero pages */

static unsigned int nr_zero_pages;

/*

 * Numbers of normal and highmem page frames allocated for hibernation image

 * before suspending devices.

 */

static struct memory_bitmap copy_bm;

/* Memory bitmap which tracks which saveable pages were zero filled. */

static struct memory_bitmap zero_bm;

/**

 * swsusp_free - Free pages allocated for hibernation image.

 *

out:

	nr_copy_pages = 0;

	nr_meta_pages = 0;

	nr_zero_pages = 0;

	restore_pblist = NULL;

	buffer = NULL;

	alloc_normal = 0;

		goto err_out;

	}

	error = memory_bm_create(&zero_bm, GFP_IMAGE, PG_ANY);

	if (error) {

		pr_err("Cannot allocate zero bitmap\n");

		goto err_out;

	}

	alloc_normal = 0;

	alloc_highmem = 0;

	nr_zero_pages = 0;

	/* Count the number of saveable data pages. */

	save_highmem = count_highmem_pages();

	 * Kill them.

	 */

	drain_local_pages(NULL);

	copy_data_pages(&copy_bm, &orig_bm);

	nr_copy_pages = copy_data_pages(&copy_bm, &orig_bm, &zero_bm);

	/*

	 * End of critical section. From now on, we can write to memory,

	 * but we should not touch disk. This specially means we must _not_

	 * touch swap space! Except we must write out our image of course.

	 */

	nr_pages += nr_highmem;

	nr_copy_pages = nr_pages;

	/* We don't actually copy the zero pages */

	nr_zero_pages = nr_pages - nr_copy_pages;

	nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE);

	pr_info("Image created (%d pages copied)\n", nr_pages);

	pr_info("Image created (%d pages copied, %d zero pages)\n", nr_copy_pages, nr_zero_pages);

	return 0;

}

	return init_header_complete(info);

}

#define ENCODED_PFN_ZERO_FLAG ((unsigned long)1 << (BITS_PER_LONG - 1))

#define ENCODED_PFN_MASK (~ENCODED_PFN_ZERO_FLAG)

/**

 * pack_pfns - Prepare PFNs for saving.

 * @bm: Memory bitmap.

 * @buf: Memory buffer to store the PFNs in.

 * @zero_bm: Memory bitmap containing PFNs of zero pages.

 *

 * PFNs corresponding to set bits in @bm are stored in the area of memory

 * pointed to by @buf (1 page at a time).

 * pointed to by @buf (1 page at a time). Pages which were filled with only

 * zeros will have the highest bit set in the packed format to distinguish

 * them from PFNs which will be contained in the image file.

 */

static inline void pack_pfns(unsigned long *buf, struct memory_bitmap *bm)

static inline void pack_pfns(unsigned long *buf, struct memory_bitmap *bm,

		struct memory_bitmap *zero_bm)

{

	int j;

		buf[j] = memory_bm_next_pfn(bm);

		if (unlikely(buf[j] == BM_END_OF_MAP))

			break;

		if (memory_bm_test_bit(zero_bm, buf[j]))

			buf[j] |= ENCODED_PFN_ZERO_FLAG;

	}

}

		memory_bm_position_reset(&copy_bm);

	} else if (handle->cur <= nr_meta_pages) {

		clear_page(buffer);

		pack_pfns(buffer, &orig_bm);

		pack_pfns(buffer, &orig_bm, &zero_bm);

	} else {

		struct page *page;

 * unpack_orig_pfns - Set bits corresponding to given PFNs in a memory bitmap.

 * @bm: Memory bitmap.

 * @buf: Area of memory containing the PFNs.

 * @zero_bm: Memory bitmap with the zero PFNs marked.

 *

 * For each element of the array pointed to by @buf (1 page at a time), set the

 * corresponding bit in @bm.

 * corresponding bit in @bm. If the page was originally populated with only

 * zeros then a corresponding bit will also be set in @zero_bm.

 */

static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm)

static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm,

		struct memory_bitmap *zero_bm)

{

	unsigned long decoded_pfn;

        bool zero;

	int j;

	for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {

		if (unlikely(buf[j] == BM_END_OF_MAP))

			break;

		if (pfn_valid(buf[j]) && memory_bm_pfn_present(bm, buf[j])) {

			memory_bm_set_bit(bm, buf[j]);

		zero = !!(buf[j] & ENCODED_PFN_ZERO_FLAG);

		decoded_pfn = buf[j] & ENCODED_PFN_MASK;

		if (pfn_valid(decoded_pfn) && memory_bm_pfn_present(bm, decoded_pfn)) {

			memory_bm_set_bit(bm, decoded_pfn);

			if (zero) {

				memory_bm_set_bit(zero_bm, decoded_pfn);

				nr_zero_pages++;

			}

		} else {

			if (!pfn_valid(buf[j]))

			if (!pfn_valid(decoded_pfn))

				pr_err(FW_BUG "Memory map mismatch at 0x%llx after hibernation\n",

				       (unsigned long long)PFN_PHYS(buf[j]));

				       (unsigned long long)PFN_PHYS(decoded_pfn));

			return -EFAULT;

		}

	}

 * prepare_image - Make room for loading hibernation image.

 * @new_bm: Uninitialized memory bitmap structure.

 * @bm: Memory bitmap with unsafe pages marked.

 * @zero_bm: Memory bitmap containing the zero pages.

 *

 * Use @bm to mark the pages that will be overwritten in the process of

 * restoring the system memory state from the suspend image ("unsafe" pages)

 * pages will be used for just yet.  Instead, we mark them all as allocated and

 * create a lists of "safe" pages to be used later.  On systems with high

 * memory a list of "safe" highmem pages is created too.

 *

 * Because it was not known which pages were unsafe when @zero_bm was created,

 * make a copy of it and recreate it within safe pages.

 */

static int prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm)

static int prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm,

		struct memory_bitmap *zero_bm)

{

	unsigned int nr_pages, nr_highmem;

	struct memory_bitmap tmp;

	struct linked_page *lp;

	int error;

	duplicate_memory_bitmap(new_bm, bm);

	memory_bm_free(bm, PG_UNSAFE_KEEP);

	/* Make a copy of zero_bm so it can be created in safe pages */

	error = memory_bm_create(&tmp, GFP_ATOMIC, PG_ANY);

	if (error)

		goto Free;

	duplicate_memory_bitmap(&tmp, zero_bm);

	memory_bm_free(zero_bm, PG_UNSAFE_KEEP);

	/* Recreate zero_bm in safe pages */

	error = memory_bm_create(zero_bm, GFP_ATOMIC, PG_SAFE);

	if (error)

		goto Free;

	duplicate_memory_bitmap(zero_bm, &tmp);

	memory_bm_free(&tmp, PG_UNSAFE_KEEP);

	/* At this point zero_bm is in safe pages and it can be used for restoring. */

	if (nr_highmem > 0) {

		error = prepare_highmem_image(bm, &nr_highmem);

		if (error)

	 *

	 * nr_copy_pages cannot be less than allocated_unsafe_pages too.

	 */

	nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;

	nr_pages = (nr_zero_pages + nr_copy_pages) - nr_highmem - allocated_unsafe_pages;

	nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE);

	while (nr_pages > 0) {

		lp = get_image_page(GFP_ATOMIC, PG_SAFE);

		nr_pages--;

	}

	/* Preallocate memory for the image */

	nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;

	nr_pages = (nr_zero_pages + nr_copy_pages) - nr_highmem - allocated_unsafe_pages;

	while (nr_pages > 0) {

		lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC);

		if (!lp) {

	static struct chain_allocator ca;

	int error = 0;

next:

	/* Check if we have already loaded the entire image */

	if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages)

	if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages + nr_zero_pages)

		return 0;

	handle->sync_read = 1;

		if (error)

			return error;

		error = memory_bm_create(&zero_bm, GFP_ATOMIC, PG_ANY);

		if (error)

			return error;

		nr_zero_pages = 0;

		hibernate_restore_protection_begin();

	} else if (handle->cur <= nr_meta_pages + 1) {

		error = unpack_orig_pfns(buffer, &copy_bm);

		error = unpack_orig_pfns(buffer, &copy_bm, &zero_bm);

		if (error)

			return error;

		if (handle->cur == nr_meta_pages + 1) {

			error = prepare_image(&orig_bm, &copy_bm);

			error = prepare_image(&orig_bm, &copy_bm, &zero_bm);

			if (error)

				return error;

			chain_init(&ca, GFP_ATOMIC, PG_SAFE);

			memory_bm_position_reset(&orig_bm);

			memory_bm_position_reset(&zero_bm);

			restore_pblist = NULL;

			handle->buffer = get_buffer(&orig_bm, &ca);

			handle->sync_read = 0;

			handle->sync_read = 0;

	}

	handle->cur++;

	/* Zero pages were not included in the image, memset it and move on. */

	if (handle->cur > nr_meta_pages + 1 &&

	    memory_bm_test_bit(&zero_bm, memory_bm_get_current(&orig_bm))) {

		memset(handle->buffer, 0, PAGE_SIZE);

		goto next;

	}

	return PAGE_SIZE;

}

	copy_last_highmem_page();

	hibernate_restore_protect_page(handle->buffer);

	/* Do that only if we have loaded the image entirely */

	if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages) {

	if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages + nr_zero_pages) {

		memory_bm_recycle(&orig_bm);

		free_highmem_data();

	}

int snapshot_image_loaded(struct snapshot_handle *handle)

{

	return !(!nr_copy_pages || !last_highmem_page_copied() ||

			handle->cur <= nr_meta_pages + nr_copy_pages);

			handle->cur <= nr_meta_pages + nr_copy_pages + nr_zero_pages);

}

#ifdef CONFIG_HIGHMEM