f2fs: compress: add compress_inode to cache compressed blocks (6ce19aff) · Commits · EulixOS / Software / Kernel

Documentation/filesystems/f2fs.rst

+3 −0

Original line number	Diff line number	Diff line
		@@ -289,6 +289,9 @@ compress_mode=%s Control file compression mode. This supports "fs" and "user"
		choosing the target file and the timing. The user can do manual
		compression/decompression on the compression enabled files using
		ioctls.
		compress_cache Support to use address space of a filesystem managed inode to
		cache compressed block, in order to improve cache hit ratio of
		random read.
		inlinecrypt When possible, encrypt/decrypt the contents of encrypted
		files using the blk-crypto framework rather than
		filesystem-layer encryption. This allows the use of

fs/f2fs/compress.c

+166 −2

Original line number	Diff line number	Diff line
		@@ -12,9 +12,11 @@
		#include <linux/lzo.h>
		#include <linux/lz4.h>
		#include <linux/zstd.h>
		#include <linux/pagevec.h>

		#include "f2fs.h"
		#include "node.h"
		#include "segment.h"
		#include <trace/events/f2fs.h>

		static struct kmem_cache *cic_entry_slab;
		@@ -736,7 +738,7 @@ static int f2fs_compress_pages(struct compress_ctx *cc)
		return ret;
		}

		static void f2fs_decompress_cluster(struct decompress_io_ctx *dic)
		void f2fs_decompress_cluster(struct decompress_io_ctx *dic)
		{
		struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode);
		struct f2fs_inode_info *fi = F2FS_I(dic->inode);
		@@ -835,7 +837,8 @@ static void f2fs_decompress_cluster(struct decompress_io_ctx *dic)
		* page being waited on in the cluster, and if so, it decompresses the cluster
		* (or in the case of a failure, cleans up without actually decompressing).
		*/
		void f2fs_end_read_compressed_page(struct page *page, bool failed)
		void f2fs_end_read_compressed_page(struct page *page, bool failed,
		block_t blkaddr)
		{
		struct decompress_io_ctx *dic =
		(struct decompress_io_ctx *)page_private(page);
		@@ -845,6 +848,9 @@ void f2fs_end_read_compressed_page(struct page *page, bool failed)

		if (failed)
		WRITE_ONCE(dic->failed, true);
		else if (blkaddr)
		f2fs_cache_compressed_page(sbi, page,
		dic->inode->i_ino, blkaddr);

		if (atomic_dec_and_test(&dic->remaining_pages))
		f2fs_decompress_cluster(dic);
		@@ -1660,6 +1666,164 @@ void f2fs_put_page_dic(struct page *page)
		f2fs_put_dic(dic);
		}

		const struct address_space_operations f2fs_compress_aops = {
		.releasepage = f2fs_release_page,
		.invalidatepage = f2fs_invalidate_page,
		};

		struct address_space COMPRESS_MAPPING(struct f2fs_sb_info sbi)
		{
		return sbi->compress_inode->i_mapping;
		}

		void f2fs_invalidate_compress_page(struct f2fs_sb_info *sbi, block_t blkaddr)
		{
		if (!sbi->compress_inode)
		return;
		invalidate_mapping_pages(COMPRESS_MAPPING(sbi), blkaddr, blkaddr);
		}

		void f2fs_cache_compressed_page(struct f2fs_sb_info sbi, struct page page,
		nid_t ino, block_t blkaddr)
		{
		struct page *cpage;
		int ret;

		if (!test_opt(sbi, COMPRESS_CACHE))
		return;

		if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE_READ))
		return;

		if (!f2fs_available_free_memory(sbi, COMPRESS_PAGE))
		return;

		cpage = find_get_page(COMPRESS_MAPPING(sbi), blkaddr);
		if (cpage) {
		f2fs_put_page(cpage, 0);
		return;
		}

		cpage = alloc_page(__GFP_NOWARN \| __GFP_IO);
		if (!cpage)
		return;

		ret = add_to_page_cache_lru(cpage, COMPRESS_MAPPING(sbi),
		blkaddr, GFP_NOFS);
		if (ret) {
		f2fs_put_page(cpage, 0);
		return;
		}

		set_page_private_data(cpage, ino);

		if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE_READ))
		goto out;

		memcpy(page_address(cpage), page_address(page), PAGE_SIZE);
		SetPageUptodate(cpage);
		out:
		f2fs_put_page(cpage, 1);
		}

		bool f2fs_load_compressed_page(struct f2fs_sb_info sbi, struct page page,
		block_t blkaddr)
		{
		struct page *cpage;
		bool hitted = false;

		if (!test_opt(sbi, COMPRESS_CACHE))
		return false;

		cpage = f2fs_pagecache_get_page(COMPRESS_MAPPING(sbi),
		blkaddr, FGP_LOCK \| FGP_NOWAIT, GFP_NOFS);
		if (cpage) {
		if (PageUptodate(cpage)) {
		atomic_inc(&sbi->compress_page_hit);
		memcpy(page_address(page),
		page_address(cpage), PAGE_SIZE);
		hitted = true;
		}
		f2fs_put_page(cpage, 1);
		}

		return hitted;
		}

		void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino)
		{
		struct address_space *mapping = sbi->compress_inode->i_mapping;
		struct pagevec pvec;
		pgoff_t index = 0;
		pgoff_t end = MAX_BLKADDR(sbi);

		if (!mapping->nrpages)
		return;

		pagevec_init(&pvec);

		do {
		unsigned int nr_pages;
		int i;

		nr_pages = pagevec_lookup_range(&pvec, mapping,
		&index, end - 1);
		if (!nr_pages)
		break;

		for (i = 0; i < nr_pages; i++) {
		struct page *page = pvec.pages[i];

		if (page->index > end)
		break;

		lock_page(page);
		if (page->mapping != mapping) {
		unlock_page(page);
		continue;
		}

		if (ino != get_page_private_data(page)) {
		unlock_page(page);
		continue;
		}

		generic_error_remove_page(mapping, page);
		unlock_page(page);
		}
		pagevec_release(&pvec);
		cond_resched();
		} while (index < end);
		}

		int f2fs_init_compress_inode(struct f2fs_sb_info *sbi)
		{
		struct inode *inode;

		if (!test_opt(sbi, COMPRESS_CACHE))
		return 0;

		inode = f2fs_iget(sbi->sb, F2FS_COMPRESS_INO(sbi));
		if (IS_ERR(inode))
		return PTR_ERR(inode);
		sbi->compress_inode = inode;

		sbi->compress_percent = COMPRESS_PERCENT;
		sbi->compress_watermark = COMPRESS_WATERMARK;

		atomic_set(&sbi->compress_page_hit, 0);

		return 0;
		}

		void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi)
		{
		if (!sbi->compress_inode)
		return;
		iput(sbi->compress_inode);
		sbi->compress_inode = NULL;
		}

		int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi)
		{
		dev_t dev = sbi->sb->s_bdev->bd_dev;

fs/f2fs/data.c

+35 −6

Original line number	Diff line number	Diff line
		@@ -132,7 +132,7 @@ static void f2fs_finish_read_bio(struct bio *bio)

		if (f2fs_is_compressed_page(page)) {
		if (bio->bi_status)
		f2fs_end_read_compressed_page(page, true);
		f2fs_end_read_compressed_page(page, true, 0);
		f2fs_put_page_dic(page);
		continue;
		}
		@@ -228,15 +228,19 @@ static void f2fs_handle_step_decompress(struct bio_post_read_ctx *ctx)
		struct bio_vec *bv;
		struct bvec_iter_all iter_all;
		bool all_compressed = true;
		block_t blkaddr = SECTOR_TO_BLOCK(ctx->bio->bi_iter.bi_sector);

		bio_for_each_segment_all(bv, ctx->bio, iter_all) {
		struct page *page = bv->bv_page;

		/* PG_error was set if decryption failed. */
		if (f2fs_is_compressed_page(page))
		f2fs_end_read_compressed_page(page, PageError(page));
		f2fs_end_read_compressed_page(page, PageError(page),
		blkaddr);
		else
		all_compressed = false;

		blkaddr++;
		}

		/*
		@@ -1352,9 +1356,11 @@ static int __allocate_data_block(struct dnode_of_data *dn, int seg_type)
		old_blkaddr = dn->data_blkaddr;
		f2fs_allocate_data_block(sbi, NULL, old_blkaddr, &dn->data_blkaddr,
		&sum, seg_type, NULL);
		if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
		if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
		invalidate_mapping_pages(META_MAPPING(sbi),
		old_blkaddr, old_blkaddr);
		f2fs_invalidate_compress_page(sbi, old_blkaddr);
		}
		f2fs_update_data_blkaddr(dn, dn->data_blkaddr);

		/*
		@@ -2174,7 +2180,7 @@ int f2fs_read_multi_pages(struct compress_ctx cc, struct bio *bio_ret,
		goto out_put_dnode;
		}

		for (i = 0; i < dic->nr_cpages; i++) {
		for (i = 0; i < cc->nr_cpages; i++) {
		struct page *page = dic->cpages[i];
		block_t blkaddr;
		struct bio_post_read_ctx *ctx;
		@@ -2182,6 +2188,14 @@ int f2fs_read_multi_pages(struct compress_ctx cc, struct bio *bio_ret,
		blkaddr = data_blkaddr(dn.inode, dn.node_page,
		dn.ofs_in_node + i + 1);

		f2fs_wait_on_block_writeback(inode, blkaddr);

		if (f2fs_load_compressed_page(sbi, page, blkaddr)) {
		if (atomic_dec_and_test(&dic->remaining_pages))
		f2fs_decompress_cluster(dic);
		continue;
		}

		if (bio && (!page_is_mergeable(sbi, bio,
		*last_block_in_bio, blkaddr) \|\|
		!f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) {
		@@ -2203,8 +2217,6 @@ int f2fs_read_multi_pages(struct compress_ctx cc, struct bio *bio_ret,
		}
		}

		f2fs_wait_on_block_writeback(inode, blkaddr);

		if (bio_add_page(bio, page, blocksize, 0) < blocksize)
		goto submit_and_realloc;

		@@ -3618,6 +3630,13 @@ void f2fs_invalidate_page(struct page *page, unsigned int offset,

		clear_page_private_gcing(page);

		if (test_opt(sbi, COMPRESS_CACHE)) {
		if (f2fs_compressed_file(inode))
		f2fs_invalidate_compress_pages(sbi, inode->i_ino);
		if (inode->i_ino == F2FS_COMPRESS_INO(sbi))
		clear_page_private_data(page);
		}

		if (page_private_atomic(page))
		return f2fs_drop_inmem_page(inode, page);

		@@ -3635,6 +3654,16 @@ int f2fs_release_page(struct page *page, gfp_t wait)
		if (page_private_atomic(page))
		return 0;

		if (test_opt(F2FS_P_SB(page), COMPRESS_CACHE)) {
		struct f2fs_sb_info *sbi = F2FS_P_SB(page);
		struct inode *inode = page->mapping->host;

		if (f2fs_compressed_file(inode))
		f2fs_invalidate_compress_pages(sbi, inode->i_ino);
		if (inode->i_ino == F2FS_COMPRESS_INO(sbi))
		clear_page_private_data(page);
		}

		clear_page_private_gcing(page);

		detach_page_private(page);

fs/f2fs/debug.c

+13 −0

Original line number	Diff line number	Diff line
		@@ -152,6 +152,12 @@ static void update_general_status(struct f2fs_sb_info *sbi)
		si->node_pages = NODE_MAPPING(sbi)->nrpages;
		if (sbi->meta_inode)
		si->meta_pages = META_MAPPING(sbi)->nrpages;
		#ifdef CONFIG_F2FS_FS_COMPRESSION
		if (sbi->compress_inode) {
		si->compress_pages = COMPRESS_MAPPING(sbi)->nrpages;
		si->compress_page_hit = atomic_read(&sbi->compress_page_hit);
		}
		#endif
		si->nats = NM_I(sbi)->nat_cnt[TOTAL_NAT];
		si->dirty_nats = NM_I(sbi)->nat_cnt[DIRTY_NAT];
		si->sits = MAIN_SEGS(sbi);
		@@ -309,6 +315,12 @@ static void update_mem_info(struct f2fs_sb_info *sbi)

		si->page_mem += (unsigned long long)npages << PAGE_SHIFT;
		}
		#ifdef CONFIG_F2FS_FS_COMPRESSION
		if (sbi->compress_inode) {
		unsigned npages = COMPRESS_MAPPING(sbi)->nrpages;
		si->page_mem += (unsigned long long)npages << PAGE_SHIFT;
		}
		#endif
		}

		static int stat_show(struct seq_file s, void v)
		@@ -476,6 +488,7 @@ static int stat_show(struct seq_file s, void v)
		"volatile IO: %4d (Max. %4d)\n",
		si->inmem_pages, si->aw_cnt, si->max_aw_cnt,
		si->vw_cnt, si->max_vw_cnt);
		seq_printf(s, " - compress: %4d, hit:%8d\n", si->compress_pages, si->compress_page_hit);
		seq_printf(s, " - nodes: %4d in %4d\n",
		si->ndirty_node, si->node_pages);
		seq_printf(s, " - dents: %4d in dirs:%4d (%4d)\n",

fs/f2fs/f2fs.h

+68 −3

Original line number	Diff line number	Diff line
		@@ -98,6 +98,7 @@ extern const char *f2fs_fault_name[FAULT_MAX];
		#define F2FS_MOUNT_ATGC 0x08000000
		#define F2FS_MOUNT_MERGE_CHECKPOINT 0x10000000
		#define F2FS_MOUNT_GC_MERGE 0x20000000
		#define F2FS_MOUNT_COMPRESS_CACHE 0x40000000

		#define F2FS_OPTION(sbi) ((sbi)->mount_opt)
		#define clear_opt(sbi, option) (F2FS_OPTION(sbi).opt &= ~F2FS_MOUNT_##option)
		@@ -1374,6 +1375,37 @@ PAGE_PRIVATE_CLEAR_FUNC(gcing, ONGOING_MIGRATION);
		PAGE_PRIVATE_CLEAR_FUNC(atomic, ATOMIC_WRITE);
		PAGE_PRIVATE_CLEAR_FUNC(dummy, DUMMY_WRITE);

		static inline unsigned long get_page_private_data(struct page *page)
		{
		unsigned long data = page_private(page);

		if (!test_bit(PAGE_PRIVATE_NOT_POINTER, &data))
		return 0;
		return data >> PAGE_PRIVATE_MAX;
		}

		static inline void set_page_private_data(struct page *page, unsigned long data)
		{
		if (!PagePrivate(page)) {
		get_page(page);
		SetPagePrivate(page);
		}
		set_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page));
		page_private(page) \|= data << PAGE_PRIVATE_MAX;
		}

		static inline void clear_page_private_data(struct page *page)
		{
		page_private(page) &= (1 << PAGE_PRIVATE_MAX) - 1;
		if (page_private(page) == 1 << PAGE_PRIVATE_NOT_POINTER) {
		set_page_private(page, 0);
		if (PagePrivate(page)) {
		ClearPagePrivate(page);
		put_page(page);
		}
		}
		}

		/* For compression */
		enum compress_algorithm_type {
		COMPRESS_LZO,
		@@ -1388,6 +1420,9 @@ enum compress_flag {
		COMPRESS_MAX_FLAG,
		};

		#define COMPRESS_WATERMARK 20
		#define COMPRESS_PERCENT 20

		#define COMPRESS_DATA_RESERVED_SIZE 4
		struct compress_data {
		__le32 clen; /* compressed data size */
		@@ -1700,6 +1735,12 @@ struct f2fs_sb_info {
		u64 compr_written_block;
		u64 compr_saved_block;
		u32 compr_new_inode;

		/* For compressed block cache */
		struct inode compress_inode; / cache compressed blocks */
		unsigned int compress_percent; /* cache page percentage */
		unsigned int compress_watermark; /* cache page watermark */
		atomic_t compress_page_hit; /* cache hit count */
		#endif
		};

		@@ -3671,7 +3712,8 @@ struct f2fs_stat_info {
		unsigned int bimodal, avg_vblocks;
		int util_free, util_valid, util_invalid;
		int rsvd_segs, overp_segs;
		int dirty_count, node_pages, meta_pages;
		int dirty_count, node_pages, meta_pages, compress_pages;
		int compress_page_hit;
		int prefree_count, call_count, cp_count, bg_cp_count;
		int tot_segs, node_segs, data_segs, free_segs, free_secs;
		int bg_node_segs, bg_data_segs;
		@@ -4007,7 +4049,9 @@ void f2fs_compress_write_end_io(struct bio bio, struct page page);
		bool f2fs_is_compress_backend_ready(struct inode *inode);
		int f2fs_init_compress_mempool(void);
		void f2fs_destroy_compress_mempool(void);
		void f2fs_end_read_compressed_page(struct page *page, bool failed);
		void f2fs_decompress_cluster(struct decompress_io_ctx *dic);
		void f2fs_end_read_compressed_page(struct page *page, bool failed,
		block_t blkaddr);
		bool f2fs_cluster_is_empty(struct compress_ctx *cc);
		bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index);
		void f2fs_compress_ctx_add_page(struct compress_ctx cc, struct page page);
		@@ -4025,10 +4069,19 @@ void f2fs_put_page_dic(struct page *page);
		int f2fs_init_compress_ctx(struct compress_ctx *cc);
		void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse);
		void f2fs_init_compress_info(struct f2fs_sb_info *sbi);
		int f2fs_init_compress_inode(struct f2fs_sb_info *sbi);
		void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi);
		int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi);
		void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi);
		int __init f2fs_init_compress_cache(void);
		void f2fs_destroy_compress_cache(void);
		struct address_space COMPRESS_MAPPING(struct f2fs_sb_info sbi);
		void f2fs_invalidate_compress_page(struct f2fs_sb_info *sbi, block_t blkaddr);
		void f2fs_cache_compressed_page(struct f2fs_sb_info sbi, struct page page,
		nid_t ino, block_t blkaddr);
		bool f2fs_load_compressed_page(struct f2fs_sb_info sbi, struct page page,
		block_t blkaddr);
		void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino);
		#define inc_compr_inode_stat(inode) \
		do { \
		struct f2fs_sb_info *sbi = F2FS_I_SB(inode); \
		@@ -4057,7 +4110,9 @@ static inline struct page f2fs_compress_control_page(struct page page)
		}
		static inline int f2fs_init_compress_mempool(void) { return 0; }
		static inline void f2fs_destroy_compress_mempool(void) { }
		static inline void f2fs_end_read_compressed_page(struct page *page, bool failed)
		static inline void f2fs_decompress_cluster(struct decompress_io_ctx *dic) { }
		static inline void f2fs_end_read_compressed_page(struct page *page,
		bool failed, block_t blkaddr)
		{
		WARN_ON_ONCE(1);
		}
		@@ -4065,10 +4120,20 @@ static inline void f2fs_put_page_dic(struct page *page)
		{
		WARN_ON_ONCE(1);
		}
		static inline int f2fs_init_compress_inode(struct f2fs_sb_info *sbi) { return 0; }
		static inline void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi) { }
		static inline int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) { return 0; }
		static inline void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) { }
		static inline int __init f2fs_init_compress_cache(void) { return 0; }
		static inline void f2fs_destroy_compress_cache(void) { }
		static inline void f2fs_invalidate_compress_page(struct f2fs_sb_info *sbi,
		block_t blkaddr) { }
		static inline void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi,
		struct page *page, nid_t ino, block_t blkaddr) { }
		static inline bool f2fs_load_compressed_page(struct f2fs_sb_info *sbi,
		struct page *page, block_t blkaddr) { return false; }
		static inline void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi,
		nid_t ino) { }
		#define inc_compr_inode_stat(inode) do { } while (0)
		#endif