Commit 22cfe4b4 authored by Eric Biggers's avatar Eric Biggers
Browse files

ext4: add fs-verity read support 



Make ext4_mpage_readpages() verify data as it is read from fs-verity
files, using the helper functions from fs/verity/.

To support both encryption and verity simultaneously, this required
refactoring the decryption workflow into a generic "post-read
processing" workflow which can do decryption, verification, or both.

The case where the ext4 block size is not equal to the PAGE_SIZE is not
supported yet, since in that case ext4_mpage_readpages() sometimes falls
back to block_read_full_page(), which does not support fs-verity yet.

Co-developed-by: default avatarTheodore Ts'o <tytso@mit.edu>
Signed-off-by: default avatarTheodore Ts'o <tytso@mit.edu>
Signed-off-by: default avatarEric Biggers <ebiggers@google.com>
parent c93d8f88

fs/ext4/ext4.h

+2 −0
Original line number Diff line number Diff line
@@ -3191,6 +3191,8 @@ static inline void ext4_set_de_type(struct super_block *sb, 
extern int ext4_mpage_readpages(struct address_space *mapping,

				struct list_head *pages, struct page *page,

				unsigned nr_pages, bool is_readahead);

extern int __init ext4_init_post_read_processing(void);

extern void ext4_exit_post_read_processing(void);



/* symlink.c */

extern const struct inode_operations ext4_encrypted_symlink_inode_operations;

fs/ext4/inode.c

+2 −0
Original line number Diff line number Diff line
@@ -3912,6 +3912,8 @@ static ssize_t ext4_direct_IO(struct kiocb *iocb, struct iov_iter *iter) 
	if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode))

		return 0;

#endif

	if (fsverity_active(inode))

		return 0;



	/*

	 * If we are doing data journalling we don't support O_DIRECT

fs/ext4/readpage.c

+176 −35
Original line number Diff line number Diff line
@@ -47,13 +47,103 @@ 


#include "ext4.h"



static inline bool ext4_bio_encrypted(struct bio *bio)

#define NUM_PREALLOC_POST_READ_CTXS	128



static struct kmem_cache *bio_post_read_ctx_cache;

static mempool_t *bio_post_read_ctx_pool;



/* postprocessing steps for read bios */

enum bio_post_read_step {

	STEP_INITIAL = 0,

	STEP_DECRYPT,

	STEP_VERITY,

};



struct bio_post_read_ctx {

	struct bio *bio;

	struct work_struct work;

	unsigned int cur_step;

	unsigned int enabled_steps;

};



static void __read_end_io(struct bio *bio)

{

	struct page *page;

	struct bio_vec *bv;

	struct bvec_iter_all iter_all;



	bio_for_each_segment_all(bv, bio, iter_all) {

		page = bv->bv_page;



		/* PG_error was set if any post_read step failed */

		if (bio->bi_status || PageError(page)) {

			ClearPageUptodate(page);

			/* will re-read again later */

			ClearPageError(page);

		} else {

			SetPageUptodate(page);

		}

		unlock_page(page);

	}

	if (bio->bi_private)

		mempool_free(bio->bi_private, bio_post_read_ctx_pool);

	bio_put(bio);

}



static void bio_post_read_processing(struct bio_post_read_ctx *ctx);



static void decrypt_work(struct work_struct *work)

{

	struct bio_post_read_ctx *ctx =

		container_of(work, struct bio_post_read_ctx, work);



	fscrypt_decrypt_bio(ctx->bio);



	bio_post_read_processing(ctx);

}



static void verity_work(struct work_struct *work)

{

	struct bio_post_read_ctx *ctx =

		container_of(work, struct bio_post_read_ctx, work);



	fsverity_verify_bio(ctx->bio);



	bio_post_read_processing(ctx);

}



static void bio_post_read_processing(struct bio_post_read_ctx *ctx)

{

	/*

	 * We use different work queues for decryption and for verity because

	 * verity may require reading metadata pages that need decryption, and

	 * we shouldn't recurse to the same workqueue.

	 */

	switch (++ctx->cur_step) {

	case STEP_DECRYPT:

		if (ctx->enabled_steps & (1 << STEP_DECRYPT)) {

			INIT_WORK(&ctx->work, decrypt_work);

			fscrypt_enqueue_decrypt_work(&ctx->work);

			return;

		}

		ctx->cur_step++;

		/* fall-through */

	case STEP_VERITY:

		if (ctx->enabled_steps & (1 << STEP_VERITY)) {

			INIT_WORK(&ctx->work, verity_work);

			fsverity_enqueue_verify_work(&ctx->work);

			return;

		}

		ctx->cur_step++;

		/* fall-through */

	default:

		__read_end_io(ctx->bio);

	}

}



static bool bio_post_read_required(struct bio *bio)

{

#ifdef CONFIG_FS_ENCRYPTION

	return unlikely(bio->bi_private != NULL);

#else

	return false;

#endif

	return bio->bi_private && !bio->bi_status;

}



/*
@@ -70,30 +160,53 @@ static inline bool ext4_bio_encrypted(struct bio *bio) 
 */

static void mpage_end_io(struct bio *bio)

{

	struct bio_vec *bv;

	struct bvec_iter_all iter_all;

	if (bio_post_read_required(bio)) {

		struct bio_post_read_ctx *ctx = bio->bi_private;



	if (ext4_bio_encrypted(bio)) {

		if (bio->bi_status) {

			fscrypt_release_ctx(bio->bi_private);

		} else {

			fscrypt_enqueue_decrypt_bio(bio->bi_private, bio);

		ctx->cur_step = STEP_INITIAL;

		bio_post_read_processing(ctx);

		return;

	}

	__read_end_io(bio);

}

	bio_for_each_segment_all(bv, bio, iter_all) {

		struct page *page = bv->bv_page;



		if (!bio->bi_status) {

			SetPageUptodate(page);

		} else {

			ClearPageUptodate(page);

			SetPageError(page);

static inline bool ext4_need_verity(const struct inode *inode, pgoff_t idx)

{

	return fsverity_active(inode) &&

	       idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);

}

		unlock_page(page);



static struct bio_post_read_ctx *get_bio_post_read_ctx(struct inode *inode,

						       struct bio *bio,

						       pgoff_t first_idx)

{

	unsigned int post_read_steps = 0;

	struct bio_post_read_ctx *ctx = NULL;



	if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode))

		post_read_steps |= 1 << STEP_DECRYPT;



	if (ext4_need_verity(inode, first_idx))

		post_read_steps |= 1 << STEP_VERITY;



	if (post_read_steps) {

		ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);

		if (!ctx)

			return ERR_PTR(-ENOMEM);

		ctx->bio = bio;

		ctx->enabled_steps = post_read_steps;

		bio->bi_private = ctx;

	}

	return ctx;

}



	bio_put(bio);

static inline loff_t ext4_readpage_limit(struct inode *inode)

{

	if (IS_ENABLED(CONFIG_FS_VERITY) &&

	    (IS_VERITY(inode) || ext4_verity_in_progress(inode)))

		return inode->i_sb->s_maxbytes;



	return i_size_read(inode);

}



int ext4_mpage_readpages(struct address_space *mapping,
@@ -141,7 +254,8 @@ int ext4_mpage_readpages(struct address_space *mapping, 


		block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);

		last_block = block_in_file + nr_pages * blocks_per_page;

		last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits;

		last_block_in_file = (ext4_readpage_limit(inode) +

				      blocksize - 1) >> blkbits;

		if (last_block > last_block_in_file)

			last_block = last_block_in_file;

		page_block = 0;
@@ -218,6 +332,9 @@ int ext4_mpage_readpages(struct address_space *mapping, 
			zero_user_segment(page, first_hole << blkbits,

					  PAGE_SIZE);

			if (first_hole == 0) {

				if (ext4_need_verity(inode, page->index) &&

				    !fsverity_verify_page(page))

					goto set_error_page;

				SetPageUptodate(page);

				unlock_page(page);

				goto next_page;
@@ -241,18 +358,16 @@ int ext4_mpage_readpages(struct address_space *mapping, 
			bio = NULL;

		}

		if (bio == NULL) {

			struct fscrypt_ctx *ctx = NULL;

			struct bio_post_read_ctx *ctx;



			if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode)) {

				ctx = fscrypt_get_ctx(GFP_NOFS);

				if (IS_ERR(ctx))

					goto set_error_page;

			}

			bio = bio_alloc(GFP_KERNEL,

				min_t(int, nr_pages, BIO_MAX_PAGES));

			if (!bio) {

				if (ctx)

					fscrypt_release_ctx(ctx);

			if (!bio)

				goto set_error_page;

			ctx = get_bio_post_read_ctx(inode, bio, page->index);

			if (IS_ERR(ctx)) {

				bio_put(bio);

				bio = NULL;

				goto set_error_page;

			}

			bio_set_dev(bio, bdev);
@@ -293,3 +408,29 @@ int ext4_mpage_readpages(struct address_space *mapping, 
		submit_bio(bio);

	return 0;

}



int __init ext4_init_post_read_processing(void)

{

	bio_post_read_ctx_cache =

		kmem_cache_create("ext4_bio_post_read_ctx",

				  sizeof(struct bio_post_read_ctx), 0, 0, NULL);

	if (!bio_post_read_ctx_cache)

		goto fail;

	bio_post_read_ctx_pool =

		mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS,

					 bio_post_read_ctx_cache);

	if (!bio_post_read_ctx_pool)

		goto fail_free_cache;

	return 0;



fail_free_cache:

	kmem_cache_destroy(bio_post_read_ctx_cache);

fail:

	return -ENOMEM;

}



void ext4_exit_post_read_processing(void)

{

	mempool_destroy(bio_post_read_ctx_pool);

	kmem_cache_destroy(bio_post_read_ctx_cache);

}

fs/ext4/super.c

+8 −1
Original line number Diff line number Diff line
@@ -6103,6 +6103,10 @@ static int __init ext4_init_fs(void) 
		return err;



	err = ext4_init_pending();

	if (err)

		goto out7;



	err = ext4_init_post_read_processing();

	if (err)

		goto out6;
@@ -6144,8 +6148,10 @@ out3: 
out4:

	ext4_exit_pageio();

out5:

	ext4_exit_pending();

	ext4_exit_post_read_processing();

out6:

	ext4_exit_pending();

out7:

	ext4_exit_es();



	return err;
@@ -6162,6 +6168,7 @@ static void __exit ext4_exit_fs(void) 
	ext4_exit_sysfs();

	ext4_exit_system_zone();

	ext4_exit_pageio();

	ext4_exit_post_read_processing();

	ext4_exit_es();

	ext4_exit_pending();

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE