From f93aaaf0a638cbaab8ce73c7c4a07f8444d06d66 Mon Sep 17 00:00:00 2001 From: Edward Houston Date: Tue, 31 Mar 2026 14:05:58 +0200 Subject: [PATCH 1/4] feat: shared LRU block cache across all three DBs Single shared cache for txstore, history, and cache DBs instead of per-DB caches. --db-block-cache-mb now controls the total cache size. Eliminates wasted allocation on the tiny cache_db and lets txstore and history claim cache proportional to actual usage via LRU eviction. --- src/new_index/db.rs | 16 ++++++++++++++-- src/new_index/schema.rs | 15 ++++++++++++--- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/new_index/db.rs b/src/new_index/db.rs index 43fad6786..5c7acddb7 100644 --- a/src/new_index/db.rs +++ b/src/new_index/db.rs @@ -91,6 +91,10 @@ pub enum DBFlush { impl DB { pub fn open(path: &Path, config: &Config, verify_compat: bool) -> DB { + Self::open_with_cache(path, config, verify_compat, None) + } + + pub fn open_with_cache(path: &Path, config: &Config, verify_compat: bool, shared_cache: Option<&rocksdb::Cache>) -> DB { debug!("opening DB at {:?}", path); let mut db_opts = rocksdb::Options::default(); db_opts.create_if_missing(true); @@ -148,8 +152,16 @@ impl DB { // Configure block cache and table options let mut block_opts = rocksdb::BlockBasedOptions::default(); - let cache_size_bytes = config.db_block_cache_mb * 1024 * 1024; - block_opts.set_block_cache(&rocksdb::Cache::new_lru_cache(cache_size_bytes)); + let owned_cache; + let cache = match shared_cache { + Some(c) => c, + None => { + let cache_size_bytes = config.db_block_cache_mb * 1024 * 1024; + owned_cache = rocksdb::Cache::new_lru_cache(cache_size_bytes); + &owned_cache + } + }; + block_opts.set_block_cache(cache); // When --cache-index-filter-blocks is passed, store index and filter blocks // inside the block cache so their memory is bounded by --db-block-cache-mb. // Without this (the default), RocksDB keeps them on the heap where they may diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index cdb3dea22..788dac3b5 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -61,15 +61,24 @@ impl Store { pub fn open(config: &Config, metrics: &Metrics, verify_compat: bool) -> Self { let path = config.db_path.join("newindex"); - let txstore_db = DB::open(&path.join("txstore"), config, verify_compat); + // Create a single shared LRU cache for all three DBs. The total size is + // --db-block-cache-mb (not multiplied by 3). RocksDB's LRU cache is + // thread-safe, so all DBs share one eviction pool. This lets the + // txstore (which holds the bulk of the data) claim as much cache as it + // needs without being artificially capped at 1/3 of the total. + let cache_size_bytes = config.db_block_cache_mb * 1024 * 1024; + let shared_cache = rocksdb::Cache::new_lru_cache(cache_size_bytes); + info!("shared LRU block cache: db_block_cache_mb='{}'", config.db_block_cache_mb); + + let txstore_db = DB::open_with_cache(&path.join("txstore"), config, verify_compat, Some(&shared_cache)); let added_blockhashes = load_blockhashes(&txstore_db, &BlockRow::done_filter()); debug!("{} blocks were added", added_blockhashes.len()); - let history_db = DB::open(&path.join("history"), config, verify_compat); + let history_db = DB::open_with_cache(&path.join("history"), config, verify_compat, Some(&shared_cache)); let indexed_blockhashes = load_blockhashes(&history_db, &BlockRow::done_filter()); debug!("{} blocks were indexed", indexed_blockhashes.len()); - let cache_db = DB::open(&path.join("cache"), config, verify_compat); + let cache_db = DB::open_with_cache(&path.join("cache"), config, verify_compat, Some(&shared_cache)); let db_metrics = Arc::new(RocksDbMetrics::new(&metrics)); txstore_db.start_stats_exporter(Arc::clone(&db_metrics), "txstore_db"); From cbea5479bc6b098b1307db061d0fd90897660052 Mon Sep 17 00:00:00 2001 From: Edward Houston Date: Fri, 10 Apr 2026 10:04:43 +0200 Subject: [PATCH 2/4] Adding type annotation on owned_cache per review comment --- src/new_index/db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/new_index/db.rs b/src/new_index/db.rs index 5c7acddb7..94b69375c 100644 --- a/src/new_index/db.rs +++ b/src/new_index/db.rs @@ -152,7 +152,7 @@ impl DB { // Configure block cache and table options let mut block_opts = rocksdb::BlockBasedOptions::default(); - let owned_cache; + let owned_cache: rocksdb::Cache; let cache = match shared_cache { Some(c) => c, None => { From 0e28effe852946677ef978ea8554564d60f78367 Mon Sep 17 00:00:00 2001 From: Edward Houston Date: Mon, 13 Apr 2026 12:08:17 +0200 Subject: [PATCH 3/4] =?UTF-8?q?Increase=20default=20block=20cache=20to=202?= =?UTF-8?q?4=20MB=20to=20match=20prior=20per-DB=20total=20(3=C3=978=20MB)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config.rs b/src/config.rs index 9eeb9ac02..a6f454c43 100644 --- a/src/config.rs +++ b/src/config.rs @@ -242,9 +242,9 @@ impl Config { ).arg( Arg::with_name("db_block_cache_mb") .long("db-block-cache-mb") - .help("RocksDB block cache size in MB per database. Bounds index/filter block memory; use 4096+ for initial sync to avoid table-reader heap growth.") + .help("RocksDB block cache size in MB (shared across all databases). Bounds index/filter block memory; use 4096+ for initial sync to avoid table-reader heap growth.") .takes_value(true) - .default_value("8") + .default_value("24") ).arg( Arg::with_name("db_parallelism") .long("db-parallelism") From e8f5f5c22a0b602d39e58093009a4b9bb2850993 Mon Sep 17 00:00:00 2001 From: Edward Houston Date: Wed, 15 Apr 2026 17:09:45 +0200 Subject: [PATCH 4/4] Simplify DB::open to use the shared block cache Address review feedback: remove the Option<&Cache> parameter and the owned_cache lifetime workaround. --- src/new_index/db.rs | 17 ++--------------- src/new_index/schema.rs | 8 ++++---- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/src/new_index/db.rs b/src/new_index/db.rs index 94b69375c..fd62d21c8 100644 --- a/src/new_index/db.rs +++ b/src/new_index/db.rs @@ -90,11 +90,7 @@ pub enum DBFlush { } impl DB { - pub fn open(path: &Path, config: &Config, verify_compat: bool) -> DB { - Self::open_with_cache(path, config, verify_compat, None) - } - - pub fn open_with_cache(path: &Path, config: &Config, verify_compat: bool, shared_cache: Option<&rocksdb::Cache>) -> DB { + pub fn open(path: &Path, config: &Config, verify_compat: bool, shared_cache: &rocksdb::Cache) -> DB { debug!("opening DB at {:?}", path); let mut db_opts = rocksdb::Options::default(); db_opts.create_if_missing(true); @@ -152,16 +148,7 @@ impl DB { // Configure block cache and table options let mut block_opts = rocksdb::BlockBasedOptions::default(); - let owned_cache: rocksdb::Cache; - let cache = match shared_cache { - Some(c) => c, - None => { - let cache_size_bytes = config.db_block_cache_mb * 1024 * 1024; - owned_cache = rocksdb::Cache::new_lru_cache(cache_size_bytes); - &owned_cache - } - }; - block_opts.set_block_cache(cache); + block_opts.set_block_cache(shared_cache); // When --cache-index-filter-blocks is passed, store index and filter blocks // inside the block cache so their memory is bounded by --db-block-cache-mb. // Without this (the default), RocksDB keeps them on the heap where they may diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index 788dac3b5..97deeca22 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -68,17 +68,17 @@ impl Store { // needs without being artificially capped at 1/3 of the total. let cache_size_bytes = config.db_block_cache_mb * 1024 * 1024; let shared_cache = rocksdb::Cache::new_lru_cache(cache_size_bytes); - info!("shared LRU block cache: db_block_cache_mb='{}'", config.db_block_cache_mb); + debug!("shared LRU block cache: db_block_cache_mb='{}'", config.db_block_cache_mb); - let txstore_db = DB::open_with_cache(&path.join("txstore"), config, verify_compat, Some(&shared_cache)); + let txstore_db = DB::open(&path.join("txstore"), config, verify_compat, &shared_cache); let added_blockhashes = load_blockhashes(&txstore_db, &BlockRow::done_filter()); debug!("{} blocks were added", added_blockhashes.len()); - let history_db = DB::open_with_cache(&path.join("history"), config, verify_compat, Some(&shared_cache)); + let history_db = DB::open(&path.join("history"), config, verify_compat, &shared_cache); let indexed_blockhashes = load_blockhashes(&history_db, &BlockRow::done_filter()); debug!("{} blocks were indexed", indexed_blockhashes.len()); - let cache_db = DB::open_with_cache(&path.join("cache"), config, verify_compat, Some(&shared_cache)); + let cache_db = DB::open(&path.join("cache"), config, verify_compat, &shared_cache); let db_metrics = Arc::new(RocksDbMetrics::new(&metrics)); txstore_db.start_stats_exporter(Arc::clone(&db_metrics), "txstore_db");