From 363b7fd76c91dc611a56d992e9550bb1ba070e1a Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 29 Mar 2023 10:29:34 -0400 Subject: [PATCH] dm: improve hash_locks sizing and hash function Both bufio and bio-prison-v1 use the identical model for splitting their respective locks and rbtrees. Improve dm_num_hash_locks() to distribute across more rbtrees to improve overall performance -- but the maximum number of locks/rbtrees is still 64. Also factor out a common hash function named dm_hash_locks_index(), the magic numbers used were determined to be best using this program: https://gist.github.com/jthornber/e05c47daa7b500c56dc339269c5467fc Signed-off-by: Joe Thornber Signed-off-by: Mike Snitzer --- drivers/md/dm-bio-prison-v1.c | 5 +++-- drivers/md/dm-bufio.c | 2 +- drivers/md/dm.h | 14 +++++++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/drivers/md/dm-bio-prison-v1.c b/drivers/md/dm-bio-prison-v1.c index 6b726b8b300d..92afdca760ae 100644 --- a/drivers/md/dm-bio-prison-v1.c +++ b/drivers/md/dm-bio-prison-v1.c @@ -117,9 +117,10 @@ static int cmp_keys(struct dm_cell_key *lhs, return 0; } -static unsigned lock_nr(struct dm_cell_key *key, unsigned int num_locks) +static inline unsigned int lock_nr(struct dm_cell_key *key, unsigned int num_locks) { - return (key->block_begin >> BIO_PRISON_MAX_RANGE_SHIFT) & (num_locks - 1); + return dm_hash_locks_index((key->block_begin >> BIO_PRISON_MAX_RANGE_SHIFT), + num_locks); } bool dm_cell_key_has_valid_range(struct dm_cell_key *key) diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index c1126ad45bdb..8a448185662c 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -398,7 +398,7 @@ struct dm_buffer_cache { static inline unsigned int cache_index(sector_t block, unsigned int num_locks) { - return block & (num_locks - 1); + return dm_hash_locks_index(block, num_locks); } static inline void cache_read_lock(struct dm_buffer_cache *bc, sector_t block) diff --git a/drivers/md/dm.h b/drivers/md/dm.h index a1a5defddb07..a856e0aee73b 100644 --- a/drivers/md/dm.h +++ b/drivers/md/dm.h @@ -233,9 +233,21 @@ unsigned int dm_get_reserved_bio_based_ios(void); static inline unsigned int dm_num_hash_locks(void) { - unsigned int num_locks = roundup_pow_of_two(num_online_cpus()); + unsigned int num_locks = roundup_pow_of_two(num_online_cpus()) << 1; return min_t(unsigned int, num_locks, DM_HASH_LOCKS_MAX); } +#define DM_HASH_LOCKS_MULT 4294967291ULL +#define DM_HASH_LOCKS_SHIFT 6 + +static inline unsigned int dm_hash_locks_index(sector_t block, + unsigned int num_locks) +{ + sector_t h1 = (block * DM_HASH_LOCKS_MULT) >> DM_HASH_LOCKS_SHIFT; + sector_t h2 = h1 >> DM_HASH_LOCKS_SHIFT; + + return (h1 ^ h2) & (num_locks - 1); +} + #endif