fs: implement faster dentry memcmp (9d55c369) · Commits · 戴 / test

The standard memcmp function on a Westmere system shows up hot in profiles in the `git diff` workload (both parallel and single threaded), and it is likely due to the costs associated with trapping into microcode, and little opportunity to improve memory access (dentry name is not likely to take up more than a cacheline). So replace it with an open-coded byte comparison. This increases code size by 8 bytes in the critical __d_lookup_rcu function, but the speedup is huge, averaging 10 runs of each: git diff st user sys elapsed CPU before 1.15 2.57 3.82 97.1 after 1.14 2.35 3.61 96.8 git diff mt user sys elapsed CPU before 1.27 3.85 1.46 349 after 1.26 3.54 1.43 333 Elapsed time for single threaded git diff at 95.0% confidence: -0.21 +/- 0.01 -5.45% +/- 0.24% It's -0.66% +/- 0.06% elapsed time on my Opteron, so rep cmp costs on the fam10h seem to be relatively smaller, but there is still a win. Signed-off-by:

Nick Piggin <npiggin@kernel.dk>

fs/dcache.c

+3 −9

Original line number	Diff line number	Diff line
		@@ -1454,9 +1454,7 @@ static struct dentry __d_instantiate_unique(struct dentry entry,
		continue;
		if (alias->d_parent != entry->d_parent)
		continue;
		if (qstr->len != len)
		continue;
		if (memcmp(qstr->name, name, len))
		if (dentry_cmp(qstr->name, qstr->len, name, len))
		continue;
		__dget(alias);
		return alias;
		@@ -1810,9 +1808,7 @@ seqretry:
		tlen, tname, name))
		continue;
		} else {
		if (tlen != len)
		continue;
		if (memcmp(tname, str, tlen))
		if (dentry_cmp(tname, tlen, str, len))
		continue;
		}
		/*
		@@ -1925,9 +1921,7 @@ struct dentry __d_lookup(struct dentry parent, struct qstr *name)
		tlen, tname, name))
		goto next;
		} else {
		if (tlen != len)
		goto next;
		if (memcmp(tname, str, tlen))
		if (dentry_cmp(tname, tlen, str, len))
		goto next;
		}

include/linux/dcache.h

+21 −0

Original line number	Diff line number	Diff line
		@@ -47,6 +47,27 @@ struct dentry_stat_t {
		};
		extern struct dentry_stat_t dentry_stat;

		/*
		* Compare 2 name strings, return 0 if they match, otherwise non-zero.
		* The strings are both count bytes long, and count is non-zero.
		*/
		static inline int dentry_cmp(const unsigned char *cs, size_t scount,
		const unsigned char *ct, size_t tcount)
		{
		int ret;
		if (scount != tcount)
		return 1;
		do {
		ret = (cs != ct);
		if (ret)
		break;
		cs++;
		ct++;
		tcount--;
		} while (tcount);
		return ret;
		}

		/* Name hashing routines. Initial hash value */
		/* Hash courtesy of the R5 hash in reiserfs modulo sign bits */
		#define init_name_hash() 0

Admin message