fscache: Maintain a catalogue of allocated cookies (ec0328e4) · Commits · 戴 / test

fs/fscache/cookie.c

+238 −56

Original line number	Diff line number	Diff line
		@@ -21,6 +21,9 @@ struct kmem_cache *fscache_cookie_jar;

		static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);

		#define fscache_cookie_hash_shift 15
		static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];

		static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
		loff_t object_size);
		static int fscache_alloc_object(struct fscache_cache *cache,
		@@ -28,6 +31,44 @@ static int fscache_alloc_object(struct fscache_cache *cache,
		static int fscache_attach_object(struct fscache_cookie *cookie,
		struct fscache_object *object);

		static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
		{
		struct hlist_node *object;
		const u8 *k;
		unsigned loop;

		pr_err("%c-cookie c=%p [p=%p fl=%lx nc=%u na=%u]\n",
		prefix, cookie, cookie->parent, cookie->flags,
		atomic_read(&cookie->n_children),
		atomic_read(&cookie->n_active));
		pr_err("%c-cookie d=%p n=%p\n",
		prefix, cookie->def, cookie->netfs_data);

		object = READ_ONCE(cookie->backing_objects.first);
		if (object)
		pr_err("%c-cookie o=%p\n",
		prefix, hlist_entry(object, struct fscache_object, cookie_link));

		pr_err("%c-key=[%u] '", prefix, cookie->key_len);
		k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
		cookie->inline_key : cookie->key;
		for (loop = 0; loop < cookie->key_len; loop++)
		pr_cont("%02x", k[loop]);
		pr_cont("'\n");
		}

		void fscache_free_cookie(struct fscache_cookie *cookie)
		{
		if (cookie) {
		BUG_ON(!hlist_empty(&cookie->backing_objects));
		if (cookie->aux_len > sizeof(cookie->inline_aux))
		kfree(cookie->aux);
		if (cookie->key_len > sizeof(cookie->inline_key))
		kfree(cookie->key);
		kmem_cache_free(fscache_cookie_jar, cookie);
		}
		}

		/*
		* initialise an cookie jar slab element prior to any use
		*/
		@@ -41,6 +82,170 @@ void fscache_cookie_init_once(void *_cookie)
		INIT_HLIST_HEAD(&cookie->backing_objects);
		}

		/*
		* Set the index key in a cookie. The cookie struct has space for a 12-byte
		* key plus length and hash, but if that's not big enough, it's instead a
		* pointer to a buffer containing 3 bytes of hash, 1 byte of length and then
		* the key data.
		*/
		static int fscache_set_key(struct fscache_cookie *cookie,
		const void *index_key, size_t index_key_len)
		{
		unsigned long long h;
		u32 *buf;
		int i;

		cookie->key_len = index_key_len;

		if (index_key_len > sizeof(cookie->inline_key)) {
		buf = kzalloc(index_key_len, GFP_KERNEL);
		if (!buf)
		return -ENOMEM;
		cookie->key = buf;
		} else {
		buf = (u32 *)cookie->inline_key;
		buf[0] = 0;
		buf[1] = 0;
		buf[2] = 0;
		}

		memcpy(buf, index_key, index_key_len);

		/* Calculate a hash and combine this with the length in the first word
		* or first half word
		*/
		h = (unsigned long)cookie->parent;
		h += index_key_len + cookie->type;
		for (i = 0; i < (index_key_len + sizeof(u32) - 1) / sizeof(u32); i++)
		h += buf[i];

		cookie->key_hash = h ^ (h >> 32);
		return 0;
		}

		static long fscache_compare_cookie(const struct fscache_cookie *a,
		const struct fscache_cookie *b)
		{
		const void ka, kb;

		if (a->key_hash != b->key_hash)
		return (long)a->key_hash - (long)b->key_hash;
		if (a->parent != b->parent)
		return (long)a->parent - (long)b->parent;
		if (a->key_len != b->key_len)
		return (long)a->key_len - (long)b->key_len;
		if (a->type != b->type)
		return (long)a->type - (long)b->type;

		if (a->key_len <= sizeof(a->inline_key)) {
		ka = &a->inline_key;
		kb = &b->inline_key;
		} else {
		ka = a->key;
		kb = b->key;
		}
		return memcmp(ka, kb, a->key_len);
		}

		/*
		* Allocate a cookie.
		*/
		struct fscache_cookie *fscache_alloc_cookie(
		struct fscache_cookie *parent,
		const struct fscache_cookie_def *def,
		const void *index_key, size_t index_key_len,
		const void *aux_data, size_t aux_data_len,
		void *netfs_data,
		loff_t object_size)
		{
		struct fscache_cookie *cookie;

		/* allocate and initialise a cookie */
		cookie = kmem_cache_alloc(fscache_cookie_jar, GFP_KERNEL);
		if (!cookie)
		return NULL;

		cookie->key_len = index_key_len;
		cookie->aux_len = aux_data_len;

		if (fscache_set_key(cookie, index_key, index_key_len) < 0)
		goto nomem;

		if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
		memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
		} else {
		cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
		if (!cookie->aux)
		goto nomem;
		}

		atomic_set(&cookie->usage, 1);
		atomic_set(&cookie->n_children, 0);

		/* We keep the active count elevated until relinquishment to prevent an
		* attempt to wake up every time the object operations queue quiesces.
		*/
		atomic_set(&cookie->n_active, 1);

		cookie->def = def;
		cookie->parent = parent;
		cookie->netfs_data = netfs_data;
		cookie->flags = (1 << FSCACHE_COOKIE_NO_DATA_YET);
		cookie->type = def->type;

		/* radix tree insertion won't use the preallocation pool unless it's
		* told it may not wait */
		INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
		return cookie;

		nomem:
		fscache_free_cookie(cookie);
		return NULL;
		}

		/*
		* Attempt to insert the new cookie into the hash. If there's a collision, we
		* return the old cookie if it's not in use and an error otherwise.
		*/
		struct fscache_cookie fscache_hash_cookie(struct fscache_cookie candidate)
		{
		struct fscache_cookie *cursor;
		struct hlist_bl_head *h;
		struct hlist_bl_node *p;
		unsigned int bucket;

		bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
		h = &fscache_cookie_hash[bucket];

		hlist_bl_lock(h);
		hlist_bl_for_each_entry(cursor, p, h, hash_link) {
		if (fscache_compare_cookie(candidate, cursor) == 0)
		goto collision;
		}

		__set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags);
		fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent);
		atomic_inc(&candidate->parent->n_children);
		hlist_bl_add_head(&candidate->hash_link, h);
		hlist_bl_unlock(h);
		return candidate;

		collision:
		if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) {
		trace_fscache_cookie(cursor, fscache_cookie_collision,
		atomic_read(&cursor->usage));
		pr_err("Duplicate cookie detected\n");
		fscache_print_cookie(cursor, 'O');
		fscache_print_cookie(candidate, 'N');
		hlist_bl_unlock(h);
		return NULL;
		}

		fscache_cookie_get(cursor, fscache_cookie_get_reacquire);
		hlist_bl_unlock(h);
		return cursor;
		}

		/*
		* request a cookie to represent an object (index, datafile, xattr, etc)
		* - parent specifies the parent object
		@@ -65,7 +270,7 @@ struct fscache_cookie *__fscache_acquire_cookie(
		loff_t object_size,
		bool enable)
		{
		struct fscache_cookie *cookie;
		struct fscache_cookie candidate, cookie;

		BUG_ON(!def);

		@@ -95,53 +300,24 @@ struct fscache_cookie *__fscache_acquire_cookie(
		BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX &&
		parent->type != FSCACHE_COOKIE_TYPE_INDEX);

		/* allocate and initialise a cookie */
		cookie = kmem_cache_alloc(fscache_cookie_jar, GFP_KERNEL);
		if (!cookie) {
		candidate = fscache_alloc_cookie(parent, def,
		index_key, index_key_len,
		aux_data, aux_data_len,
		netfs_data, object_size);
		if (!candidate) {
		fscache_stat(&fscache_n_acquires_oom);
		_leave(" [ENOMEM]");
		return NULL;
		}

		cookie->key_len = index_key_len;
		cookie->aux_len = aux_data_len;

		if (cookie->key_len <= sizeof(cookie->inline_key)) {
		memcpy(cookie->inline_key, index_key, cookie->key_len);
		} else {
		cookie->key = kmemdup(index_key, cookie->key_len, GFP_KERNEL);
		if (!cookie->key)
		goto nomem;
		}

		if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
		memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
		} else {
		cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
		if (!cookie->aux)
		goto nomem;
		cookie = fscache_hash_cookie(candidate);
		if (!cookie) {
		trace_fscache_cookie(candidate, fscache_cookie_discard, 1);
		goto out;
		}

		atomic_set(&cookie->usage, 1);
		atomic_set(&cookie->n_children, 0);

		/* We keep the active count elevated until relinquishment to prevent an
		* attempt to wake up every time the object operations queue quiesces.
		*/
		atomic_set(&cookie->n_active, 1);

		fscache_cookie_get(parent, fscache_cookie_get_acquire_parent);
		atomic_inc(&parent->n_children);

		cookie->def = def;
		cookie->parent = parent;
		cookie->netfs_data = netfs_data;
		cookie->flags = (1 << FSCACHE_COOKIE_NO_DATA_YET);
		cookie->type = def->type;

		/* radix tree insertion won't use the preallocation pool unless it's
		* told it may not wait */
		INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
		if (cookie == candidate)
		candidate = NULL;

		switch (cookie->type) {
		case FSCACHE_COOKIE_TYPE_INDEX:
		@@ -178,16 +354,10 @@ struct fscache_cookie *__fscache_acquire_cookie(
		}

		fscache_stat(&fscache_n_acquires_ok);
		_leave(" = %p", cookie);
		return cookie;

		nomem:
		if (cookie->aux_len > sizeof(cookie->inline_aux))
		kfree(cookie->aux);
		if (cookie->key_len > sizeof(cookie->inline_key))
		kfree(cookie->key);
		kmem_cache_free(fscache_cookie_jar, cookie);
		return NULL;
		out:
		fscache_free_cookie(candidate);
		return cookie;
		}
		EXPORT_SYMBOL(__fscache_acquire_cookie);

		@@ -678,6 +848,22 @@ void __fscache_relinquish_cookie(struct fscache_cookie *cookie,
		}
		EXPORT_SYMBOL(__fscache_relinquish_cookie);

		/*
		* Remove a cookie from the hash table.
		*/
		static void fscache_unhash_cookie(struct fscache_cookie *cookie)
		{
		struct hlist_bl_head *h;
		unsigned int bucket;

		bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
		h = &fscache_cookie_hash[bucket];

		hlist_bl_lock(h);
		hlist_bl_del(&cookie->hash_link);
		hlist_bl_unlock(h);
		}

		/*
		* Drop a reference to a cookie.
		*/
		@@ -698,12 +884,8 @@ void fscache_cookie_put(struct fscache_cookie *cookie,
		BUG_ON(usage < 0);

		parent = cookie->parent;
		BUG_ON(!hlist_empty(&cookie->backing_objects));
		if (cookie->aux_len > sizeof(cookie->inline_aux))
		kfree(cookie->aux);
		if (cookie->key_len > sizeof(cookie->inline_key))
		kfree(cookie->key);
		kmem_cache_free(fscache_cookie_jar, cookie);
		fscache_unhash_cookie(cookie);
		fscache_free_cookie(cookie);

		cookie = parent;
		where = fscache_cookie_put_parent;

fs/fscache/internal.h

+7 −0

Original line number	Diff line number	Diff line
		@@ -49,7 +49,14 @@ extern struct fscache_cache *fscache_select_cache_for_object(
		*/
		extern struct kmem_cache *fscache_cookie_jar;

		extern void fscache_free_cookie(struct fscache_cookie *);
		extern void fscache_cookie_init_once(void *);
		extern struct fscache_cookie fscache_alloc_cookie(struct fscache_cookie ,
		const struct fscache_cookie_def *,
		const void *, size_t,
		const void *, size_t,
		void *, loff_t);
		extern struct fscache_cookie fscache_hash_cookie(struct fscache_cookie );
		extern void fscache_cookie_put(struct fscache_cookie *,
		enum fscache_cookie_trace);

fs/fscache/netfs.c

+20 −61

Original line number	Diff line number	Diff line
		@@ -14,85 +14,51 @@
		#include <linux/slab.h>
		#include "internal.h"

		static LIST_HEAD(fscache_netfs_list);

		/*
		* register a network filesystem for caching
		*/
		int __fscache_register_netfs(struct fscache_netfs *netfs)
		{
		struct fscache_netfs *ptr;
		struct fscache_cookie *cookie;
		int ret;
		struct fscache_cookie candidate, cookie;

		_enter("{%s}", netfs->name);

		INIT_LIST_HEAD(&netfs->link);

		/* allocate a cookie for the primary index */
		cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL);

		if (!cookie) {
		candidate = fscache_alloc_cookie(&fscache_fsdef_index,
		&fscache_fsdef_netfs_def,
		netfs->name, strlen(netfs->name),
		&netfs->version, sizeof(netfs->version),
		netfs, 0);
		if (!candidate) {
		_leave(" = -ENOMEM");
		return -ENOMEM;
		}

		cookie->key_len = strlen(netfs->name);
		if (cookie->key_len <= sizeof(cookie->inline_key)) {
		memcpy(cookie->inline_key, netfs->name, strlen(netfs->name));
		} else {
		ret = -ENOMEM;
		cookie->key = kmemdup(netfs->name, cookie->key_len, GFP_KERNEL);
		if (!cookie->key)
		goto nomem;
		}

		cookie->aux_len = sizeof(netfs->version);
		memcpy(cookie->inline_aux, &netfs->version, cookie->aux_len);

		/* initialise the primary index cookie */
		atomic_set(&cookie->usage, 1);
		atomic_set(&cookie->n_children, 0);
		atomic_set(&cookie->n_active, 1);

		cookie->def = &fscache_fsdef_netfs_def;
		cookie->parent = &fscache_fsdef_index;
		cookie->netfs_data = netfs;
		cookie->flags = 1 << FSCACHE_COOKIE_ENABLED;
		cookie->type = FSCACHE_COOKIE_TYPE_INDEX;

		spin_lock_init(&cookie->lock);
		spin_lock_init(&cookie->stores_lock);
		INIT_HLIST_HEAD(&cookie->backing_objects);
		candidate->flags = 1 << FSCACHE_COOKIE_ENABLED;

		/* check the netfs type is not already present */
		down_write(&fscache_addremove_sem);

		ret = -EEXIST;
		list_for_each_entry(ptr, &fscache_netfs_list, link) {
		if (strcmp(ptr->name, netfs->name) == 0)
		cookie = fscache_hash_cookie(candidate);
		if (!cookie)
		goto already_registered;
		if (cookie != candidate) {
		trace_fscache_cookie(candidate, fscache_cookie_discard, 1);
		fscache_free_cookie(candidate);
		}

		fscache_cookie_get(cookie->parent, fscache_cookie_get_register_netfs);
		atomic_inc(&cookie->parent->n_children);

		netfs->primary_index = cookie;
		list_add(&netfs->link, &fscache_netfs_list);
		ret = 0;

		pr_notice("Netfs '%s' registered for caching\n", netfs->name);
		trace_fscache_netfs(netfs);
		_leave(" = 0");
		return 0;

		already_registered:
		up_write(&fscache_addremove_sem);

		nomem:
		if (ret < 0)
		kmem_cache_free(fscache_cookie_jar, cookie);

		_leave(" = %d", ret);
		return ret;
		fscache_cookie_put(candidate, fscache_cookie_put_dup_netfs);
		_leave(" = -EEXIST");
		return -EEXIST;
		}
		EXPORT_SYMBOL(__fscache_register_netfs);

		@@ -104,15 +70,8 @@ void __fscache_unregister_netfs(struct fscache_netfs *netfs)
		{
		_enter("{%s.%u}", netfs->name, netfs->version);

		down_write(&fscache_addremove_sem);

		list_del(&netfs->link);
		fscache_relinquish_cookie(netfs->primary_index, NULL, false);

		up_write(&fscache_addremove_sem);

		pr_notice("Netfs '%s' unregistered from caching\n",
		netfs->name);
		pr_notice("Netfs '%s' unregistered from caching\n", netfs->name);

		_leave("");
		}

include/linux/fscache.h

+6 −2

Original line number	Diff line number	Diff line
		@@ -22,6 +22,7 @@
		#include <linux/list.h>
		#include <linux/pagemap.h>
		#include <linux/pagevec.h>
		#include <linux/list_bl.h>

		#if defined(CONFIG_FSCACHE) \|\| defined(CONFIG_FSCACHE_MODULE)
		#define fscache_available() (1)
		@@ -124,7 +125,6 @@ struct fscache_netfs {
		uint32_t version; /* indexing version */
		const char name; / filesystem name */
		struct fscache_cookie *primary_index;
		struct list_head link; /* internal link */
		};

		/*
		@@ -143,6 +143,7 @@ struct fscache_cookie {
		struct hlist_head backing_objects; /* object(s) backing this file/index */
		const struct fscache_cookie_def def; / definition */
		struct fscache_cookie parent; / parent of this entry */
		struct hlist_bl_node hash_link; /* Link in hash table */
		void netfs_data; / back pointer to netfs */
		struct radix_tree_root stores; /* pages to be stored on this cookie */
		#define FSCACHE_COOKIE_PENDING_TAG 0 /* pages tag: pending write to cache */
		@@ -156,11 +157,14 @@ struct fscache_cookie {
		#define FSCACHE_COOKIE_RELINQUISHED 4 /* T if cookie has been relinquished */
		#define FSCACHE_COOKIE_ENABLED 5 /* T if cookie is enabled */
		#define FSCACHE_COOKIE_ENABLEMENT_LOCK 6 /* T if cookie is being en/disabled */
		#define FSCACHE_COOKIE_AUX_UPDATED 7 /* T if the auxiliary data was updated */
		#define FSCACHE_COOKIE_AUX_UPDATED 8 /* T if the auxiliary data was updated */
		#define FSCACHE_COOKIE_ACQUIRED 9 /* T if cookie is in use */
		#define FSCACHE_COOKIE_RELINQUISHING 10 /* T if cookie is being relinquished */

		u8 type; /* Type of object */
		u8 key_len; /* Length of index key */
		u8 aux_len; /* Length of auxiliary data */
		u32 key_hash; /* Hash of parent, type, key, len */
		union {
		void key; / Index key */
		u8 inline_key[16]; /* - If the key is short enough */

include/trace/events/fscache.h

+8 −0

Original line number	Diff line number	Diff line
		@@ -24,10 +24,14 @@
		#define __FSCACHE_DECLARE_TRACE_ENUMS_ONCE_ONLY

		enum fscache_cookie_trace {
		fscache_cookie_collision,
		fscache_cookie_discard,
		fscache_cookie_get_acquire_parent,
		fscache_cookie_get_attach_object,
		fscache_cookie_get_reacquire,
		fscache_cookie_get_register_netfs,
		fscache_cookie_put_acquire_nobufs,
		fscache_cookie_put_dup_netfs,
		fscache_cookie_put_relinquish,
		fscache_cookie_put_object,
		fscache_cookie_put_parent,
		@@ -86,10 +90,14 @@ enum fscache_page_op_trace {
		* Declare tracing information enums and their string mappings for display.
		*/
		#define fscache_cookie_traces \
		EM(fscache_cookie_collision, "COLLISION") \
		EM(fscache_cookie_discard, "DISCARD") \
		EM(fscache_cookie_get_acquire_parent, "GET prn") \
		EM(fscache_cookie_get_attach_object, "GET obj") \
		EM(fscache_cookie_get_reacquire, "GET raq") \
		EM(fscache_cookie_get_register_netfs, "GET net") \
		EM(fscache_cookie_put_acquire_nobufs, "PUT nbf") \
		EM(fscache_cookie_put_dup_netfs, "PUT dnt") \
		EM(fscache_cookie_put_relinquish, "PUT rlq") \
		EM(fscache_cookie_put_object, "PUT obj") \
		E_(fscache_cookie_put_parent, "PUT prn")

Admin message