Commit 5c2a6425 authored by Gao Xiang's avatar Gao Xiang
Browse files

erofs: introduce partial-referenced pclusters 

Due to deduplication for compressed data, pclusters can be partially
referenced with their prefixes.

Together with the user-space implementation, it enables EROFS
variable-length global compressed data deduplication with rolling
hash.

Link: https://lore.kernel.org/r/20220923014915.4362-1-hsiangkao@linux.alibaba.com


Reviewed-by: default avatarYue Hu <huyue2@coolpad.com>
Signed-off-by: default avatarGao Xiang <hsiangkao@linux.alibaba.com>
parent b15b2e30

fs/erofs/decompressor_lzma.c

+3 −0
Original line number Diff line number Diff line
@@ -217,6 +217,9 @@ int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq, 
			strm->buf.out_size = min_t(u32, outlen,

						   PAGE_SIZE - pageofs);

			outlen -= strm->buf.out_size;

			if (!rq->out[no] && rq->fillgaps)	/* deduped */

				rq->out[no] = erofs_allocpage(pagepool,

						GFP_KERNEL | __GFP_NOFAIL);

			if (rq->out[no])

				strm->buf.out = kmap(rq->out[no]) + pageofs;

			pageofs = 0;

fs/erofs/erofs_fs.h

+6 −1
Original line number Diff line number Diff line
@@ -26,6 +26,7 @@ 
#define EROFS_FEATURE_INCOMPAT_COMPR_HEAD2	0x00000008

#define EROFS_FEATURE_INCOMPAT_ZTAILPACKING	0x00000010

#define EROFS_FEATURE_INCOMPAT_FRAGMENTS	0x00000020

#define EROFS_FEATURE_INCOMPAT_DEDUPE		0x00000020

#define EROFS_ALL_FEATURE_INCOMPAT		\

	(EROFS_FEATURE_INCOMPAT_ZERO_PADDING | \

	 EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \
@@ -34,7 +35,8 @@ 
	 EROFS_FEATURE_INCOMPAT_DEVICE_TABLE | \

	 EROFS_FEATURE_INCOMPAT_COMPR_HEAD2 | \

	 EROFS_FEATURE_INCOMPAT_ZTAILPACKING | \

	 EROFS_FEATURE_INCOMPAT_FRAGMENTS)

	 EROFS_FEATURE_INCOMPAT_FRAGMENTS | \

	 EROFS_FEATURE_INCOMPAT_DEDUPE)



#define EROFS_SB_EXTSLOT_SIZE	16
@@ -371,6 +373,9 @@ enum { 
#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS        2

#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT         0



/* (noncompact only, HEAD) This pcluster refers to partial decompressed data */

#define Z_EROFS_VLE_DI_PARTIAL_REF		(1 << 15)



/*

 * D0_CBLKCNT will be marked _only_ at the 1st non-head lcluster to store the

 * compressed block count of a compressed extent (in logical clusters, aka.

fs/erofs/internal.h

+4 −0
Original line number Diff line number Diff line
@@ -291,6 +291,7 @@ EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE) 
EROFS_FEATURE_FUNCS(compr_head2, incompat, INCOMPAT_COMPR_HEAD2)

EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING)

EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS)

EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE)

EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)



/* atomic flag definitions */
@@ -392,6 +393,7 @@ enum { 
	BH_Encoded = BH_PrivateStart,

	BH_FullMapped,

	BH_Fragment,

	BH_Partialref,

};



/* Has a disk mapping */
@@ -404,6 +406,8 @@ enum { 
#define EROFS_MAP_FULL_MAPPED	(1 << BH_FullMapped)

/* Located in the special packed inode */

#define EROFS_MAP_FRAGMENT	(1 << BH_Fragment)

/* The extent refers to partial decompressed data */

#define EROFS_MAP_PARTIAL_REF	(1 << BH_Partialref)



struct erofs_map_blocks {

	struct erofs_buf buf;

fs/erofs/super.c

+2 −0
Original line number Diff line number Diff line
@@ -424,6 +424,8 @@ static int erofs_read_superblock(struct super_block *sb) 
		erofs_info(sb, "EXPERIMENTAL fscache-based on-demand read feature in use. Use at your own risk!");

	if (erofs_sb_has_fragments(sbi))

		erofs_info(sb, "EXPERIMENTAL compressed fragments feature in use. Use at your own risk!");

	if (erofs_sb_has_dedupe(sbi))

		erofs_info(sb, "EXPERIMENTAL global deduplication feature in use. Use at your own risk!");

out:

	erofs_put_metabuf(&buf);

	return ret;

fs/erofs/sysfs.c

+2 −0
Original line number Diff line number Diff line
@@ -77,6 +77,7 @@ EROFS_ATTR_FEATURE(compr_head2); 
EROFS_ATTR_FEATURE(sb_chksum);

EROFS_ATTR_FEATURE(ztailpacking);

EROFS_ATTR_FEATURE(fragments);

EROFS_ATTR_FEATURE(dedupe);



static struct attribute *erofs_feat_attrs[] = {

	ATTR_LIST(zero_padding),
@@ -88,6 +89,7 @@ static struct attribute *erofs_feat_attrs[] = { 
	ATTR_LIST(sb_chksum),

	ATTR_LIST(ztailpacking),

	ATTR_LIST(fragments),

	ATTR_LIST(dedupe),

	NULL,

};

ATTRIBUTE_GROUPS(erofs_feat);