diff --git a/Cargo.toml b/Cargo.toml index f898fcd..6621912 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rmqtt-storage" -version = "0.5.0" +version = "0.7.3" authors = ["rmqtt "] edition = "2021" license = "MIT OR Apache-2.0" @@ -8,23 +8,29 @@ repository = "https://github.com/rmqtt/rmqtt-storage" homepage = "https://github.com/rmqtt/rmqtt-storage" description = "rmqtt-storage - Is a simple wrapper around some key-value storages" keywords = ["storage", "async"] -categories = ["Database interfaces"] +categories = ["database"] exclude = ["examples", ".gitignore", ".cargo/config"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[package.metadata.docs.rs] +all-features = true + [features] default = [] ttl = [] map_len = [] len = [] +sled = ["dep:sled"] +redis = ["dep:redis"] +redis-cluster = ["redis", "redis/cluster", "redis/cluster-async"] [dependencies] -sled = "0.34" -tokio = { version = "1", features = ["sync", "rt"] } -redis = { version = "0.24", features = [ "tokio-comp", "connection-manager" ] } +sled = { version = "0.34", optional = true } +redis = { version = "0.32", features = [ "tokio-comp", "safe_iterators", "connection-manager"], optional = true } +tokio = { version = "1", features = ["sync", "rt"] } futures = "0.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" @@ -33,13 +39,13 @@ async-trait = "0.1" bincode = "1.3" log = "0.4" chrono = { version = "0.4", default-features = false, features = ["clock"] } -dashmap = "5.5" +dashmap = "6.1" ahash = "0.8" convert = { package = "box-convert", version = "0.1", features = ["bytesize"] } [dev-dependencies] tokio = { version = "1", features = ["sync", "time", "macros", "rt", "rt-multi-thread"] } - +cfg-if = "1" diff --git a/README.md b/README.md index 244dac9..7f2e218 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Add this to your `Cargo.toml`: ```toml [dependencies] -rmqtt-storage = "0.4" +rmqtt-storage = "0.7" ``` ## Features @@ -25,3 +25,4 @@ rmqtt-storage = "0.4" - Supports key expiration. - Provides an implementation for 'sled'. - Provides an implementation for 'redis'. +- Provides an implementation for 'redis cluster'. Note: the 'len' feature is not supported yet. diff --git a/src/lib.rs b/src/lib.rs index 905e89c..28e6832 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,68 +1,114 @@ -#[macro_use] -extern crate serde; +//! Provides a unified storage abstraction with multiple backend implementations (sled, redis, redis-cluster). +//! +//! This module defines generic storage interfaces (`StorageDB`, `Map`, `List`) and implements them +//! for different storage backends. It includes configuration handling, initialization functions, +//! and common storage operations with support for expiration and batch operations. -use anyhow::Error; -use serde::de; +#![deny(unsafe_code)] -use storage_redis::RedisStorageDB; +#[allow(unused_imports)] +use serde::{de, Deserialize, Serialize}; +// Conditionally include storage modules based on enabled features +#[cfg(any(feature = "redis", feature = "redis-cluster", feature = "sled"))] mod storage; +#[cfg(feature = "redis")] mod storage_redis; +#[cfg(feature = "redis-cluster")] +mod storage_redis_cluster; +#[cfg(feature = "sled")] mod storage_sled; +// Re-export public storage interfaces and implementations +#[cfg(any(feature = "redis", feature = "redis-cluster", feature = "sled"))] pub use storage::{DefaultStorageDB, List, Map, StorageDB, StorageList, StorageMap}; -pub use storage_redis::RedisConfig; +#[cfg(feature = "redis")] +pub use storage_redis::{RedisConfig, RedisStorageDB}; +#[cfg(feature = "redis-cluster")] +pub use storage_redis_cluster::{ + RedisConfig as RedisClusterConfig, RedisStorageDB as RedisClusterStorageDB, +}; +#[cfg(feature = "sled")] pub use storage_sled::{SledConfig, SledStorageDB}; +/// Custom result type for storage operations pub type Result = anyhow::Result; +/// Initializes the database based on provided configuration +/// +/// # Arguments +/// * `cfg` - Storage configuration specifying backend type and parameters +/// +/// # Returns +/// Instance of `DefaultStorageDB` configured with the selected backend +#[cfg(any(feature = "redis", feature = "redis-cluster", feature = "sled"))] pub async fn init_db(cfg: &Config) -> Result { match cfg.typ { + #[cfg(feature = "sled")] StorageType::Sled => { let db = SledStorageDB::new(cfg.sled.clone()).await?; Ok(DefaultStorageDB::Sled(db)) } + #[cfg(feature = "redis")] StorageType::Redis => { let db = RedisStorageDB::new(cfg.redis.clone()).await?; Ok(DefaultStorageDB::Redis(db)) } + #[cfg(feature = "redis-cluster")] + StorageType::RedisCluster => { + let db = RedisClusterStorageDB::new(cfg.redis_cluster.clone()).await?; + Ok(DefaultStorageDB::RedisCluster(db)) + } } } + +/// Configuration structure for storage system +/// +/// Contains backend-specific configurations and is conditionally compiled +/// based on enabled storage features. #[derive(Debug, Clone, Deserialize, Serialize)] +#[cfg(any(feature = "redis", feature = "redis-cluster", feature = "sled"))] pub struct Config { - #[serde(default = "Config::storage_type_default")] + /// Storage backend type (Sled, Redis, or RedisCluster) + // #[serde(default = "Config::storage_type_default")] #[serde(alias = "type")] pub typ: StorageType, + + /// Configuration for Sled backend (feature-gated) #[serde(default)] + #[cfg(feature = "sled")] pub sled: SledConfig, + + /// Configuration for Redis backend (feature-gated) #[serde(default)] + #[cfg(feature = "redis")] pub redis: RedisConfig, -} -impl Default for Config { - fn default() -> Self { - Config { - typ: Config::storage_type_default(), - sled: SledConfig::default(), - redis: RedisConfig::default(), - } - } -} - -impl Config { - fn storage_type_default() -> StorageType { - StorageType::Sled - } + /// Configuration for Redis Cluster backend (feature-gated) + #[serde(default, rename = "redis-cluster")] + #[cfg(feature = "redis-cluster")] + pub redis_cluster: RedisClusterConfig, } +/// Enum representing available storage backend types +/// +/// Variants are conditionally included based on enabled features #[derive(Debug, Clone, Serialize)] +#[cfg(any(feature = "redis", feature = "redis-cluster", feature = "sled"))] pub enum StorageType { - //sled: high-performance embedded database with BTreeMap-like API for stateful systems. + /// Embedded database with BTreeMap-like API + #[cfg(feature = "sled")] Sled, - //redis: + /// Single-node Redis storage + #[cfg(feature = "redis")] Redis, + /// Redis Cluster distributed storage + #[cfg(feature = "redis-cluster")] + RedisCluster, } +/// Deserialization implementation for StorageType +#[cfg(any(feature = "redis", feature = "redis-cluster", feature = "sled"))] impl<'de> de::Deserialize<'de> for StorageType { #[inline] fn deserialize(deserializer: D) -> core::result::Result @@ -73,17 +119,29 @@ impl<'de> de::Deserialize<'de> for StorageType { .to_ascii_lowercase() .as_str() { + #[cfg(feature = "sled")] "sled" => StorageType::Sled, + #[cfg(feature = "redis")] "redis" => StorageType::Redis, - _ => StorageType::Sled, + #[cfg(feature = "redis-cluster")] + "redis-cluster" => StorageType::RedisCluster, + _ => { + return Err(de::Error::custom( + "invalid storage type, expected one of: 'sled', 'redis', 'redis-cluster'", + )) + } }; Ok(t) } } +/// Timestamp type in milliseconds #[allow(dead_code)] pub(crate) type TimestampMillis = i64; +/// Gets current timestamp in milliseconds +/// +/// Uses system time if available, falls back to chrono if system time is before UNIX_EPOCH #[allow(dead_code)] #[inline] pub(crate) fn timestamp_millis() -> TimestampMillis { @@ -98,6 +156,7 @@ pub(crate) fn timestamp_millis() -> TimestampMillis { } #[cfg(test)] +#[cfg(any(feature = "redis", feature = "redis-cluster", feature = "sled"))] mod tests { use super::storage::*; use super::*; @@ -107,16 +166,40 @@ mod tests { fn get_cfg(name: &str) -> Config { let cfg = Config { - typ: StorageType::Sled, + typ: { + cfg_if::cfg_if! { + if #[cfg(feature = "sled")] { + StorageType::Sled + } else if #[cfg(feature = "redis-cluster")] { + StorageType::RedisCluster + } else if #[cfg(feature = "redis")] { + StorageType::Redis + } else { + compile_error!("No storage backend feature enabled!"); + } + } + }, + #[cfg(feature = "sled")] sled: SledConfig { path: format!("./.catch/{}", name), cleanup_f: |_db| {}, ..Default::default() }, + #[cfg(feature = "redis")] redis: RedisConfig { url: "redis://127.0.0.1:6379/".into(), prefix: name.to_owned(), }, + #[cfg(feature = "redis-cluster")] + redis_cluster: RedisClusterConfig { + urls: [ + "redis://127.0.0.1:6380/".into(), + "redis://127.0.0.1:6381/".into(), + "redis://127.0.0.1:6382/".into(), + ] + .into(), + prefix: name.to_owned(), + }, }; cfg } @@ -183,6 +266,15 @@ mod tests { url: "redis://127.0.0.1:6379/".into(), prefix: "sled_cleanup".to_owned(), }, + redis_cluster: RedisClusterConfig { + urls: [ + "redis://127.0.0.1:6380/".into(), + "redis://127.0.0.1:6381/".into(), + "redis://127.0.0.1:6382/".into(), + ] + .into(), + prefix: "sled_cleanup".to_owned(), + }, }; let db = SledStorageDB::new(cfg.sled.clone()).await.unwrap(); @@ -485,6 +577,31 @@ mod tests { assert_eq!(list_002.is_empty().await.unwrap(), true); } + #[tokio::main] + #[test] + async fn test_db_contains_key2() { + let cfg = get_cfg("db_contains_key2"); + let db = init_db(&cfg).await.unwrap(); + let max = 10; + for i in 0..max { + db.insert(format!("key_{}", i), &1).await.unwrap(); + } + + for i in 0..max { + let c_res = db.contains_key(format!("key_{}", i)).await.unwrap(); + assert!(c_res); + } + + for i in 0..max { + db.remove(format!("key_{}", i)).await.unwrap(); + } + + for i in 0..max { + let c_res = db.contains_key(format!("key_{}", i)).await.unwrap(); + assert!(!c_res); + } + } + #[cfg(feature = "ttl")] #[tokio::main] #[test] @@ -896,6 +1013,45 @@ mod tests { assert_eq!(kv001.is_empty().await.unwrap(), true); } + #[tokio::main] + #[test] + async fn test_map_iter2() { + let cfg = get_cfg("map_iter2"); + let mut db = init_db(&cfg).await.unwrap(); + + let mut map_iter = db.map_iter().await.unwrap(); + while let Some(map) = map_iter.next().await { + let map = map.unwrap(); + map.clear().await.unwrap(); + } + + drop(map_iter); + + let max = 10; + + for i in 0..max { + let map1 = db.map(format!("map-{}", i), None).await.unwrap(); + map1.insert(format!("map-{}-data", i), &i).await.unwrap(); + } + + let mut map_iter = db.map_iter().await.unwrap(); + + // let aa = collect(map_iter).await; + let mut count = 0; + while let Some(map) = map_iter.next().await { + let mut map = map.unwrap(); + let mut iter = map.iter::().await.unwrap(); + while let Some(item) = iter.next().await { + let (key, val) = item.unwrap(); + println!("key: {:?}, val: {:?}", String::from_utf8_lossy(&key), val); + } + count += 1; + } + + println!("max: {:?}, count: {:?}", max, count); + assert_eq!(max, count); + } + // #[tokio::main] // #[test] // async fn test_map_retain() { @@ -1228,6 +1384,44 @@ mod tests { } } + #[tokio::main] + #[test] + async fn test_list_iter2() { + let cfg = get_cfg("list_iter2"); + let mut db = init_db(&cfg).await.unwrap(); + + let mut list_iter = db.list_iter().await.unwrap(); + while let Some(list) = list_iter.next().await { + let list = list.unwrap(); + list.clear().await.unwrap(); + } + + drop(list_iter); + + let max = 10; + + for i in 0..max { + let list1 = db.list(format!("list-{}", i), None).await.unwrap(); + list1.push(&i).await.unwrap(); + } + + let mut list_iter = db.list_iter().await.unwrap(); + + let mut count = 0; + while let Some(list) = list_iter.next().await { + let mut list = list.unwrap(); + let mut iter = list.iter::().await.unwrap(); + while let Some(item) = iter.next().await { + let val = item.unwrap(); + println!("val: {:?}", val); + } + count += 1; + } + + println!("max: {:?}, count: {:?}", max, count); + assert_eq!(max, count); + } + #[tokio::main] #[test] async fn test_map_iter() { @@ -1530,6 +1724,7 @@ mod tests { #[tokio::main] #[allow(dead_code)] + #[cfg(feature = "sled")] // #[test] async fn test_map_expire_list() { use super::{SledStorageDB, StorageDB}; @@ -1575,10 +1770,21 @@ mod tests { }, ..Default::default() }, + #[cfg(feature = "redis")] redis: RedisConfig { url: "redis://127.0.0.1:6379/".into(), prefix: "map_expire_list".to_owned(), }, + #[cfg(feature = "redis-cluster")] + redis_cluster: RedisClusterConfig { + urls: [ + "redis://127.0.0.1:6380/".into(), + "redis://127.0.0.1:6381/".into(), + "redis://127.0.0.1:6382/".into(), + ] + .into(), + prefix: "map_expire_list".to_owned(), + }, }; let mut db = SledStorageDB::new(cfg.sled.clone()).await.unwrap(); @@ -1620,7 +1826,7 @@ mod tests { for item in collect(iter).await { db.remove(item).await.unwrap(); } - + println!("test_db_size db_size: {:?}", db.db_size().await); db.insert("k1", &1).await.unwrap(); db.insert("k2", &2).await.unwrap(); db.insert("k3", &3).await.unwrap(); @@ -1655,6 +1861,7 @@ mod tests { let mut db = init_db(&cfg).await.unwrap(); let iter = db.scan("*").await.unwrap(); for item in collect(iter).await { + println!("removed item: {:?}", String::from_utf8_lossy(&item)); db.remove(item).await.unwrap(); } println!("test_scan db_size: {:?}", db.db_size().await); @@ -1692,7 +1899,11 @@ mod tests { let topic = format_topic("foo/abcd/#"); println!("topic: {}", topic); let iter = db.scan(topic.as_bytes()).await.unwrap(); - assert_eq!(collect(iter).await.len(), 6); + let items = collect(iter).await; + for item in items.iter() { + println!("item: {:?}", String::from_utf8_lossy(&item)); + } + assert_eq!(items.len(), 6); //"foo/abcd/\\**" let topic = format_topic("foo/abcd/*/#"); diff --git a/src/storage.rs b/src/storage.rs index 24ad529..d535a9c 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -1,31 +1,67 @@ +//! Abstract storage layer with support for multiple backends (sled, Redis, Redis Cluster) +//! +//! Defines core storage traits and unified interfaces for: +//! - Key-value storage (StorageDB) +//! - Map structures (Map) +//! - List structures (List) +//! +//! Provides backend-agnostic enums (DefaultStorageDB, StorageMap, StorageList) +//! that dispatch operations to concrete implementations based on enabled features. + use core::fmt; use async_trait::async_trait; use serde::de::DeserializeOwned; use serde::Serialize; +#[cfg(feature = "redis")] use crate::storage_redis::{RedisStorageDB, RedisStorageList, RedisStorageMap}; -use crate::storage_sled::SledStorageDB; -use crate::storage_sled::{SledStorageList, SledStorageMap}; +#[cfg(feature = "redis-cluster")] +use crate::storage_redis_cluster::{ + RedisStorageDB as RedisClusterStorageDB, RedisStorageList as RedisClusterStorageList, + RedisStorageMap as RedisClusterStorageMap, +}; +#[cfg(feature = "sled")] +use crate::storage_sled::{SledStorageDB, SledStorageList, SledStorageMap}; use crate::Result; #[allow(unused_imports)] use crate::TimestampMillis; +#[allow(unused)] +pub(crate) const SEPARATOR: &[u8] = b"@"; +#[allow(unused)] +pub(crate) const KEY_PREFIX: &[u8] = b"__rmqtt@"; +#[allow(unused)] +pub(crate) const KEY_PREFIX_LEN: &[u8] = b"__rmqtt_len@"; +#[allow(unused)] +pub(crate) const MAP_NAME_PREFIX: &[u8] = b"__rmqtt_map@"; +#[allow(unused)] +pub(crate) const LIST_NAME_PREFIX: &[u8] = b"__rmqtt_list@"; + +/// Type alias for storage keys pub type Key = Vec; +/// Result type for iteration items (key-value pair) pub type IterItem = Result<(Key, V)>; +/// Asynchronous iterator trait for storage operations #[async_trait] pub trait AsyncIterator { type Item; + + /// Fetches the next item from the iterator async fn next(&mut self) -> Option; } +/// Trait for splitting byte slices (used in sled backend) +#[cfg(feature = "sled")] pub trait SplitSubslice { + /// Splits slice at the first occurrence of given subslice fn split_subslice(&self, subslice: &[u8]) -> Option<(&[u8], &[u8])>; } +#[cfg(feature = "sled")] impl SplitSubslice for [u8] { fn split_subslice(&self, subslice: &[u8]) -> Option<(&[u8], &[u8])> { self.windows(subslice.len()) @@ -34,103 +70,130 @@ impl SplitSubslice for [u8] { } } +/// Core storage database operations #[async_trait] #[allow(clippy::len_without_is_empty)] pub trait StorageDB: Send + Sync { + /// Concrete Map type for this storage type MapType: Map; + + /// Concrete List type for this storage type ListType: List; + /// Creates or accesses a named map async fn map + Sync + Send>( &self, name: N, expire: Option, ) -> Result; + /// Removes an entire map async fn map_remove(&self, name: K) -> Result<()> where K: AsRef<[u8]> + Sync + Send; + /// Checks if a map exists async fn map_contains_key + Sync + Send>(&self, key: K) -> Result; + /// Creates or accesses a named list async fn list + Sync + Send>( &self, name: V, expire: Option, ) -> Result; + /// Removes an entire list async fn list_remove(&self, name: K) -> Result<()> where K: AsRef<[u8]> + Sync + Send; + /// Checks if a list exists async fn list_contains_key + Sync + Send>(&self, key: K) -> Result; + /// Inserts a key-value pair async fn insert(&self, key: K, val: &V) -> Result<()> where K: AsRef<[u8]> + Sync + Send, V: serde::ser::Serialize + Sync + Send; + /// Retrieves a value by key async fn get(&self, key: K) -> Result> where K: AsRef<[u8]> + Sync + Send, V: DeserializeOwned + Sync + Send; + /// Removes a key-value pair async fn remove(&self, key: K) -> Result<()> where K: AsRef<[u8]> + Sync + Send; + /// Batch insert of multiple key-value pairs async fn batch_insert(&self, key_vals: Vec<(Key, V)>) -> Result<()> where V: serde::ser::Serialize + Sync + Send; + /// Batch removal of keys async fn batch_remove(&self, keys: Vec) -> Result<()>; + /// Increments a counter value async fn counter_incr(&self, key: K, increment: isize) -> Result<()> where K: AsRef<[u8]> + Sync + Send; + /// Decrements a counter value async fn counter_decr(&self, key: K, increment: isize) -> Result<()> where K: AsRef<[u8]> + Sync + Send; + /// Gets current counter value async fn counter_get(&self, key: K) -> Result> where K: AsRef<[u8]> + Sync + Send; + /// Sets counter to specific value async fn counter_set(&self, key: K, val: isize) -> Result<()> where K: AsRef<[u8]> + Sync + Send; + /// Checks if key exists async fn contains_key + Sync + Send>(&self, key: K) -> Result; + /// Gets number of items in storage (requires "len" feature) #[cfg(feature = "len")] async fn len(&self) -> Result; + /// Gets total storage size in bytes async fn db_size(&self) -> Result; + /// Sets expiration timestamp for a key (requires "ttl" feature) #[cfg(feature = "ttl")] async fn expire_at(&self, key: K, at: TimestampMillis) -> Result where K: AsRef<[u8]> + Sync + Send; + /// Sets expiration duration for a key (requires "ttl" feature) #[cfg(feature = "ttl")] async fn expire(&self, key: K, dur: TimestampMillis) -> Result where K: AsRef<[u8]> + Sync + Send; + /// Gets remaining time-to-live for a key (requires "ttl" feature) #[cfg(feature = "ttl")] async fn ttl(&self, key: K) -> Result> where K: AsRef<[u8]> + Sync + Send; + /// Iterates over all maps in storage async fn map_iter<'a>( &'a mut self, ) -> Result> + Send + 'a>>; + /// Iterates over all lists in storage async fn list_iter<'a>( &'a mut self, ) -> Result> + Send + 'a>>; - //pattern - * or ? + /// Scans keys matching pattern (supports * and ? wildcards) async fn scan<'a, P>( &'a mut self, pattern: P, @@ -138,61 +201,78 @@ pub trait StorageDB: Send + Sync { where P: AsRef<[u8]> + Send + Sync; + /// Gets storage backend information async fn info(&self) -> Result; } +/// Map (dictionary) storage operations #[async_trait] pub trait Map: Sync + Send { + /// Gets the name of this map fn name(&self) -> &[u8]; + /// Inserts a key-value pair into the map async fn insert(&self, key: K, val: &V) -> Result<()> where K: AsRef<[u8]> + Sync + Send, V: serde::ser::Serialize + Sync + Send + ?Sized; + /// Retrieves a value from the map async fn get(&self, key: K) -> Result> where K: AsRef<[u8]> + Sync + Send, V: DeserializeOwned + Sync + Send; + /// Removes a key from the map async fn remove(&self, key: K) -> Result<()> where K: AsRef<[u8]> + Sync + Send; + /// Checks if key exists in the map async fn contains_key + Sync + Send>(&self, key: K) -> Result; + /// Gets number of items in map (requires "map_len" feature) #[cfg(feature = "map_len")] async fn len(&self) -> Result; + /// Checks if map is empty async fn is_empty(&self) -> Result; + /// Clears all entries in the map async fn clear(&self) -> Result<()>; + /// Removes a key and returns its value async fn remove_and_fetch(&self, key: K) -> Result> where K: AsRef<[u8]> + Sync + Send, V: DeserializeOwned + Sync + Send; + /// Removes all keys with given prefix async fn remove_with_prefix(&self, prefix: K) -> Result<()> where K: AsRef<[u8]> + Sync + Send; + /// Batch insert of key-value pairs async fn batch_insert(&self, key_vals: Vec<(Key, V)>) -> Result<()> where V: serde::ser::Serialize + Sync + Send; + /// Batch removal of keys async fn batch_remove(&self, keys: Vec) -> Result<()>; + /// Iterates over all key-value pairs async fn iter<'a, V>( &'a mut self, ) -> Result> + Send + 'a>> where V: DeserializeOwned + Sync + Send + 'a + 'static; + /// Iterates over all keys async fn key_iter<'a>( &'a mut self, ) -> Result> + Send + 'a>>; + /// Iterates over key-value pairs with given prefix async fn prefix_iter<'a, P, V>( &'a mut self, prefix: P, @@ -201,28 +281,36 @@ pub trait Map: Sync + Send { P: AsRef<[u8]> + Send + Sync, V: DeserializeOwned + Sync + Send + 'a + 'static; + /// Sets expiration timestamp for the entire map (requires "ttl" feature) #[cfg(feature = "ttl")] async fn expire_at(&self, at: TimestampMillis) -> Result; + /// Sets expiration duration for the entire map (requires "ttl" feature) #[cfg(feature = "ttl")] async fn expire(&self, dur: TimestampMillis) -> Result; + /// Gets remaining time-to-live for the map (requires "ttl" feature) #[cfg(feature = "ttl")] async fn ttl(&self) -> Result>; } +/// List storage operations #[async_trait] pub trait List: Sync + Send { + /// Gets the name of this list fn name(&self) -> &[u8]; + /// Appends a value to the end of the list async fn push(&self, val: &V) -> Result<()> where V: serde::ser::Serialize + Sync + Send; + /// Appends multiple values to the list async fn pushs(&self, vals: Vec) -> Result<()> where V: serde::ser::Serialize + Sync + Send; + /// Pushes with size limit and optional pop-front behavior async fn push_limit( &self, val: &V, @@ -233,47 +321,66 @@ pub trait List: Sync + Send { V: serde::ser::Serialize + Sync + Send, V: DeserializeOwned; + /// Removes and returns the first value in the list async fn pop(&self) -> Result> where V: DeserializeOwned + Sync + Send; + /// Retrieves all values in the list async fn all(&self) -> Result> where V: DeserializeOwned + Sync + Send; + /// Gets value by index async fn get_index(&self, idx: usize) -> Result> where V: DeserializeOwned + Sync + Send; + /// Gets number of items in the list async fn len(&self) -> Result; + /// Checks if list is empty async fn is_empty(&self) -> Result; + /// Clears all items from the list async fn clear(&self) -> Result<()>; + /// Iterates over all values async fn iter<'a, V>( &'a mut self, ) -> Result> + Send + 'a>> where V: DeserializeOwned + Sync + Send + 'a + 'static; + /// Sets expiration timestamp for the entire list (requires "ttl" feature) #[cfg(feature = "ttl")] async fn expire_at(&self, at: TimestampMillis) -> Result; + /// Sets expiration duration for the entire list (requires "ttl" feature) #[cfg(feature = "ttl")] async fn expire(&self, dur: TimestampMillis) -> Result; + /// Gets remaining time-to-live for the list (requires "ttl" feature) #[cfg(feature = "ttl")] async fn ttl(&self) -> Result>; } +/// Unified storage backend enum (dispatches to concrete implementations) #[derive(Clone)] pub enum DefaultStorageDB { + #[cfg(feature = "sled")] + /// Sled database backend Sled(SledStorageDB), + #[cfg(feature = "redis")] + /// Redis backend Redis(RedisStorageDB), + #[cfg(feature = "redis-cluster")] + /// Redis Cluster backend + RedisCluster(RedisClusterStorageDB), } impl DefaultStorageDB { + /// Accesses a named map #[inline] pub async fn map + Sync + Send>( &self, @@ -281,30 +388,47 @@ impl DefaultStorageDB { expire: Option, ) -> Result { Ok(match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => StorageMap::Sled(db.map(name, expire).await?), + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => StorageMap::Redis(db.map(name, expire).await?), + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => { + StorageMap::RedisCluster(db.map(name, expire).await?) + } }) } + /// Removes a named map #[inline] pub async fn map_remove(&self, name: K) -> Result<()> where K: AsRef<[u8]> + Sync + Send, { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.map_remove(name).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.map_remove(name).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.map_remove(name).await, } } + /// Checks if map exists #[inline] pub async fn map_contains_key + Sync + Send>(&self, key: K) -> Result { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.map_contains_key(key).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.map_contains_key(key).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.map_contains_key(key).await, } } + /// Accesses a named list #[inline] pub async fn list + Sync + Send>( &self, @@ -312,30 +436,47 @@ impl DefaultStorageDB { expire: Option, ) -> Result { Ok(match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => StorageList::Sled(db.list(name, expire).await?), + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => StorageList::Redis(db.list(name, expire).await?), + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => { + StorageList::RedisCluster(db.list(name, expire).await?) + } }) } + /// Removes a named list #[inline] pub async fn list_remove(&self, name: K) -> Result<()> where K: AsRef<[u8]> + Sync + Send, { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.list_remove(name).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.list_remove(name).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.list_remove(name).await, } } + /// Checks if list exists #[inline] pub async fn list_contains_key + Sync + Send>(&self, key: K) -> Result { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.list_contains_key(key).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.list_contains_key(key).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.list_contains_key(key).await, } } + /// Inserts a key-value pair #[inline] pub async fn insert(&self, key: K, val: &V) -> Result<()> where @@ -343,11 +484,16 @@ impl DefaultStorageDB { V: Serialize + Sync + Send, { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.insert(key, val).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.insert(key, val).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.insert(key, val).await, } } + /// Retrieves a value by key #[inline] pub async fn get(&self, key: K) -> Result> where @@ -355,110 +501,165 @@ impl DefaultStorageDB { V: DeserializeOwned + Sync + Send, { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.get(key).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.get(key).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.get(key).await, } } + /// Removes a key-value pair #[inline] pub async fn remove(&self, key: K) -> Result<()> where K: AsRef<[u8]> + Sync + Send, { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.remove(key).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.remove(key).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.remove(key).await, } } + /// Batch insert of key-value pairs #[inline] pub async fn batch_insert(&self, key_vals: Vec<(Key, V)>) -> Result<()> where V: serde::ser::Serialize + Sync + Send, { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.batch_insert(key_vals).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.batch_insert(key_vals).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.batch_insert(key_vals).await, } } + /// Batch removal of keys #[inline] pub async fn batch_remove(&self, keys: Vec) -> Result<()> { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.batch_remove(keys).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.batch_remove(keys).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.batch_remove(keys).await, } } + /// Increments a counter #[inline] pub async fn counter_incr(&self, key: K, increment: isize) -> Result<()> where K: AsRef<[u8]> + Sync + Send, { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.counter_incr(key, increment).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.counter_incr(key, increment).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.counter_incr(key, increment).await, } } + /// Decrements a counter #[inline] pub async fn counter_decr(&self, key: K, decrement: isize) -> Result<()> where K: AsRef<[u8]> + Sync + Send, { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.counter_decr(key, decrement).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.counter_decr(key, decrement).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.counter_decr(key, decrement).await, } } + /// Gets counter value #[inline] pub async fn counter_get(&self, key: K) -> Result> where K: AsRef<[u8]> + Sync + Send, { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.counter_get(key).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.counter_get(key).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.counter_get(key).await, } } + /// Sets counter value #[inline] pub async fn counter_set(&self, key: K, val: isize) -> Result<()> where K: AsRef<[u8]> + Sync + Send, { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.counter_set(key, val).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.counter_set(key, val).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.counter_set(key, val).await, } } + /// Gets number of items (requires "len" feature) #[inline] #[cfg(feature = "len")] pub async fn len(&self) -> Result { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.len().await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.len().await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.len().await, } } + /// Gets total storage size in bytes #[inline] pub async fn db_size(&self) -> Result { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.db_size().await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.db_size().await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.db_size().await, } } + /// Checks if key exists #[inline] pub async fn contains_key + Sync + Send>(&self, key: K) -> Result { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.contains_key(key).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.contains_key(key).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.contains_key(key).await, } } + /// Sets expiration timestamp (requires "ttl" feature) #[inline] #[cfg(feature = "ttl")] pub async fn expire_at(&self, key: K, at: TimestampMillis) -> Result @@ -466,11 +667,16 @@ impl DefaultStorageDB { K: AsRef<[u8]> + Sync + Send, { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.expire_at(key, at).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.expire_at(key, at).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.expire_at(key, at).await, } } + /// Sets expiration duration (requires "ttl" feature) #[inline] #[cfg(feature = "ttl")] pub async fn expire(&self, key: K, dur: TimestampMillis) -> Result @@ -478,11 +684,16 @@ impl DefaultStorageDB { K: AsRef<[u8]> + Sync + Send, { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.expire(key, dur).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.expire(key, dur).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.expire(key, dur).await, } } + /// Gets time-to-live (requires "ttl" feature) #[inline] #[cfg(feature = "ttl")] pub async fn ttl(&self, key: K) -> Result> @@ -490,31 +701,46 @@ impl DefaultStorageDB { K: AsRef<[u8]> + Sync + Send, { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.ttl(key).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.ttl(key).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.ttl(key).await, } } + /// Iterates over maps #[inline] pub async fn map_iter<'a>( &'a mut self, ) -> Result> + Send + 'a>> { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.map_iter().await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.map_iter().await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.map_iter().await, } } + /// Iterates over lists #[inline] pub async fn list_iter<'a>( &'a mut self, ) -> Result> + Send + 'a>> { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.list_iter().await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.list_iter().await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.list_iter().await, } } + /// Scans keys matching pattern #[inline] pub async fn scan<'a, P>( &'a mut self, @@ -524,32 +750,53 @@ impl DefaultStorageDB { P: AsRef<[u8]> + Send + Sync, { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.scan(pattern).await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.scan(pattern).await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.scan(pattern).await, } } + /// Gets storage information #[inline] pub async fn info(&self) -> Result { match self { + #[cfg(feature = "sled")] DefaultStorageDB::Sled(db) => db.info().await, + #[cfg(feature = "redis")] DefaultStorageDB::Redis(db) => db.info().await, + #[cfg(feature = "redis-cluster")] + DefaultStorageDB::RedisCluster(db) => db.info().await, } } } +/// Unified map implementation enum #[derive(Clone)] pub enum StorageMap { + #[cfg(feature = "sled")] + /// Sled map implementation Sled(SledStorageMap), + #[cfg(feature = "redis")] + /// Redis map implementation Redis(RedisStorageMap), + #[cfg(feature = "redis-cluster")] + /// Redis Cluster map implementation + RedisCluster(RedisClusterStorageMap), } #[async_trait] impl Map for StorageMap { fn name(&self) -> &[u8] { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.name(), + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.name(), + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.name(), } } @@ -559,8 +806,12 @@ impl Map for StorageMap { V: Serialize + Sync + Send + ?Sized, { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.insert(key, val).await, + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.insert(key, val).await, + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.insert(key, val).await, } } @@ -570,8 +821,12 @@ impl Map for StorageMap { V: DeserializeOwned + Sync + Send, { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.get(key).await, + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.get(key).await, + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.get(key).await, } } @@ -580,37 +835,57 @@ impl Map for StorageMap { K: AsRef<[u8]> + Sync + Send, { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.remove(key).await, + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.remove(key).await, + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.remove(key).await, } } async fn contains_key + Sync + Send>(&self, key: K) -> Result { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.contains_key(key).await, + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.contains_key(key).await, + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.contains_key(key).await, } } #[cfg(feature = "map_len")] async fn len(&self) -> Result { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.len().await, + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.len().await, + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.len().await, } } async fn is_empty(&self) -> Result { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.is_empty().await, + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.is_empty().await, + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.is_empty().await, } } async fn clear(&self) -> Result<()> { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.clear().await, + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.clear().await, + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.clear().await, } } @@ -620,8 +895,12 @@ impl Map for StorageMap { V: DeserializeOwned + Sync + Send, { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.remove_and_fetch(key).await, + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.remove_and_fetch(key).await, + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.remove_and_fetch(key).await, } } @@ -630,8 +909,12 @@ impl Map for StorageMap { K: AsRef<[u8]> + Sync + Send, { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.remove_with_prefix(prefix).await, + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.remove_with_prefix(prefix).await, + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.remove_with_prefix(prefix).await, } } @@ -640,15 +923,23 @@ impl Map for StorageMap { V: Serialize + Sync + Send, { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.batch_insert(key_vals).await, + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.batch_insert(key_vals).await, + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.batch_insert(key_vals).await, } } async fn batch_remove(&self, keys: Vec) -> Result<()> { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.batch_remove(keys).await, + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.batch_remove(keys).await, + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.batch_remove(keys).await, } } @@ -659,8 +950,12 @@ impl Map for StorageMap { V: DeserializeOwned + Sync + Send + 'a + 'static, { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.iter().await, + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.iter().await, + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.iter().await, } } @@ -668,8 +963,12 @@ impl Map for StorageMap { &'a mut self, ) -> Result> + Send + 'a>> { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.key_iter().await, + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.key_iter().await, + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.key_iter().await, } } @@ -682,47 +981,75 @@ impl Map for StorageMap { V: DeserializeOwned + Sync + Send + 'a + 'static, { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.prefix_iter(prefix).await, + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.prefix_iter(prefix).await, + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.prefix_iter(prefix).await, } } #[cfg(feature = "ttl")] async fn expire_at(&self, at: TimestampMillis) -> Result { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.expire_at(at).await, + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.expire_at(at).await, + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.expire_at(at).await, } } #[cfg(feature = "ttl")] async fn expire(&self, dur: TimestampMillis) -> Result { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.expire(dur).await, + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.expire(dur).await, + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.expire(dur).await, } } #[cfg(feature = "ttl")] async fn ttl(&self) -> Result> { match self { + #[cfg(feature = "sled")] StorageMap::Sled(m) => m.ttl().await, + #[cfg(feature = "redis")] StorageMap::Redis(m) => m.ttl().await, + #[cfg(feature = "redis-cluster")] + StorageMap::RedisCluster(m) => m.ttl().await, } } } +/// Unified list implementation enum #[derive(Clone)] pub enum StorageList { + #[cfg(feature = "sled")] + /// Sled list implementation Sled(SledStorageList), + #[cfg(feature = "redis")] + /// Redis list implementation Redis(RedisStorageList), + #[cfg(feature = "redis-cluster")] + /// Redis Cluster list implementation + RedisCluster(RedisClusterStorageList), } impl fmt::Debug for StorageList { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let name = match self { + #[cfg(feature = "sled")] StorageList::Sled(list) => list.name(), + #[cfg(feature = "redis")] StorageList::Redis(list) => list.name(), + #[cfg(feature = "redis-cluster")] + StorageList::RedisCluster(list) => list.name(), }; f.debug_tuple(&format!("StorageList({:?})", String::from_utf8_lossy(name))) @@ -734,8 +1061,12 @@ impl fmt::Debug for StorageList { impl List for StorageList { fn name(&self) -> &[u8] { match self { + #[cfg(feature = "sled")] StorageList::Sled(m) => m.name(), + #[cfg(feature = "redis")] StorageList::Redis(m) => m.name(), + #[cfg(feature = "redis-cluster")] + StorageList::RedisCluster(m) => m.name(), } } @@ -744,8 +1075,12 @@ impl List for StorageList { V: Serialize + Sync + Send, { match self { + #[cfg(feature = "sled")] StorageList::Sled(list) => list.push(val).await, + #[cfg(feature = "redis")] StorageList::Redis(list) => list.push(val).await, + #[cfg(feature = "redis-cluster")] + StorageList::RedisCluster(list) => list.push(val).await, } } @@ -754,8 +1089,12 @@ impl List for StorageList { V: serde::ser::Serialize + Sync + Send, { match self { + #[cfg(feature = "sled")] StorageList::Sled(list) => list.pushs(vals).await, + #[cfg(feature = "redis")] StorageList::Redis(list) => list.pushs(vals).await, + #[cfg(feature = "redis-cluster")] + StorageList::RedisCluster(list) => list.pushs(vals).await, } } @@ -770,8 +1109,14 @@ impl List for StorageList { V: DeserializeOwned, { match self { + #[cfg(feature = "sled")] StorageList::Sled(list) => list.push_limit(val, limit, pop_front_if_limited).await, + #[cfg(feature = "redis")] StorageList::Redis(list) => list.push_limit(val, limit, pop_front_if_limited).await, + #[cfg(feature = "redis-cluster")] + StorageList::RedisCluster(list) => { + list.push_limit(val, limit, pop_front_if_limited).await + } } } @@ -780,8 +1125,12 @@ impl List for StorageList { V: DeserializeOwned + Sync + Send, { match self { + #[cfg(feature = "sled")] StorageList::Sled(list) => list.pop().await, + #[cfg(feature = "redis")] StorageList::Redis(list) => list.pop().await, + #[cfg(feature = "redis-cluster")] + StorageList::RedisCluster(list) => list.pop().await, } } @@ -790,8 +1139,12 @@ impl List for StorageList { V: DeserializeOwned + Sync + Send, { match self { + #[cfg(feature = "sled")] StorageList::Sled(list) => list.all().await, + #[cfg(feature = "redis")] StorageList::Redis(list) => list.all().await, + #[cfg(feature = "redis-cluster")] + StorageList::RedisCluster(list) => list.all().await, } } @@ -800,29 +1153,45 @@ impl List for StorageList { V: DeserializeOwned + Sync + Send, { match self { + #[cfg(feature = "sled")] StorageList::Sled(list) => list.get_index(idx).await, + #[cfg(feature = "redis")] StorageList::Redis(list) => list.get_index(idx).await, + #[cfg(feature = "redis-cluster")] + StorageList::RedisCluster(list) => list.get_index(idx).await, } } async fn len(&self) -> Result { match self { + #[cfg(feature = "sled")] StorageList::Sled(list) => list.len().await, + #[cfg(feature = "redis")] StorageList::Redis(list) => list.len().await, + #[cfg(feature = "redis-cluster")] + StorageList::RedisCluster(list) => list.len().await, } } async fn is_empty(&self) -> Result { match self { + #[cfg(feature = "sled")] StorageList::Sled(list) => list.is_empty().await, + #[cfg(feature = "redis")] StorageList::Redis(list) => list.is_empty().await, + #[cfg(feature = "redis-cluster")] + StorageList::RedisCluster(list) => list.is_empty().await, } } async fn clear(&self) -> Result<()> { match self { + #[cfg(feature = "sled")] StorageList::Sled(list) => list.clear().await, + #[cfg(feature = "redis")] StorageList::Redis(list) => list.clear().await, + #[cfg(feature = "redis-cluster")] + StorageList::RedisCluster(list) => list.clear().await, } } @@ -833,32 +1202,48 @@ impl List for StorageList { V: DeserializeOwned + Sync + Send + 'a + 'static, { match self { + #[cfg(feature = "sled")] StorageList::Sled(list) => list.iter().await, + #[cfg(feature = "redis")] StorageList::Redis(list) => list.iter().await, + #[cfg(feature = "redis-cluster")] + StorageList::RedisCluster(list) => list.iter().await, } } #[cfg(feature = "ttl")] async fn expire_at(&self, at: TimestampMillis) -> Result { match self { + #[cfg(feature = "sled")] StorageList::Sled(l) => l.expire_at(at).await, + #[cfg(feature = "redis")] StorageList::Redis(l) => l.expire_at(at).await, + #[cfg(feature = "redis-cluster")] + StorageList::RedisCluster(l) => l.expire_at(at).await, } } #[cfg(feature = "ttl")] async fn expire(&self, dur: TimestampMillis) -> Result { match self { + #[cfg(feature = "sled")] StorageList::Sled(l) => l.expire(dur).await, + #[cfg(feature = "redis")] StorageList::Redis(l) => l.expire(dur).await, + #[cfg(feature = "redis-cluster")] + StorageList::RedisCluster(l) => l.expire(dur).await, } } #[cfg(feature = "ttl")] async fn ttl(&self) -> Result> { match self { + #[cfg(feature = "sled")] StorageList::Sled(l) => l.ttl().await, + #[cfg(feature = "redis")] StorageList::Redis(l) => l.ttl().await, + #[cfg(feature = "redis-cluster")] + StorageList::RedisCluster(l) => l.ttl().await, } } } diff --git a/src/storage_redis.rs b/src/storage_redis.rs index 1620e25..002a294 100644 --- a/src/storage_redis.rs +++ b/src/storage_redis.rs @@ -1,12 +1,25 @@ +//! Redis storage implementation for key-value, map, and list data structures +//! +//! This module provides a Redis-backed storage system with support for: +//! - Key-value storage with expiration +//! - Map (hash) data structures +//! - List data structures +//! - Counters with atomic operations +//! - Iteration and scanning capabilities +//! - Standalone Redis connection support + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + use anyhow::anyhow; use async_trait::async_trait; -use redis::aio::ConnectionManager; +use redis::aio::{ConnectionManager, ConnectionManagerConfig}; use redis::{pipe, AsyncCommands}; use serde::de::DeserializeOwned; +use serde::Deserialize; use serde::Serialize; use serde_json::Value; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; use crate::storage::{AsyncIterator, IterItem, Key, List, Map, StorageDB}; use crate::{Result, StorageList, StorageMap}; @@ -14,17 +27,17 @@ use crate::{Result, StorageList, StorageMap}; #[allow(unused_imports)] use crate::{timestamp_millis, TimestampMillis}; -const SEPARATOR: &[u8] = b"@"; -const KEY_PREFIX: &[u8] = b"__rmqtt@"; -const KEY_PREFIX_LEN: &[u8] = b"__rmqtt_len@"; -const MAP_NAME_PREFIX: &[u8] = b"__rmqtt_map@"; -const LIST_NAME_PREFIX: &[u8] = b"__rmqtt_list@"; +use crate::storage::{KEY_PREFIX, KEY_PREFIX_LEN, LIST_NAME_PREFIX, MAP_NAME_PREFIX, SEPARATOR}; +/// Type alias for Redis connection manager type RedisConnection = ConnectionManager; +/// Configuration for Redis storage #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RedisConfig { + /// Redis server URL pub url: String, + /// Key prefix for all storage operations pub prefix: String, } @@ -37,13 +50,17 @@ impl Default for RedisConfig { } } +/// Redis storage database implementation #[derive(Clone)] pub struct RedisStorageDB { + /// Prefix for all keys prefix: Key, + /// Asynchronous connection manager async_conn: RedisConnection, } impl RedisStorageDB { + /// Creates a new Redis storage instance #[inline] pub(crate) async fn new(cfg: RedisConfig) -> Result { let prefix = [cfg.prefix.as_bytes(), SEPARATOR].concat(); @@ -54,59 +71,68 @@ impl RedisStorageDB { return Err(anyhow!(e)); } }; - let async_conn = match client - .get_connection_manager_with_backoff( - 2, 100, - 2, - // Duration::from_secs(5), - // Duration::from_secs(8), - ) - .await - { + + // Configure connection manager with retry settings + let mgr_cfg = ConnectionManagerConfig::default() + .set_exponent_base(100) + .set_factor(2) + .set_number_of_retries(2) + .set_connection_timeout(Duration::from_secs(15)) + .set_response_timeout(Duration::from_secs(10)); + + // Create connection manager + let async_conn = match client.get_connection_manager_with_config(mgr_cfg).await { Ok(conn) => conn, Err(e) => { log::error!("get redis connection error, config is {:?}, {:?}", cfg, e); return Err(anyhow!(e)); } }; + + // Create database instance and start cleanup task let db = Self { prefix, async_conn }.cleanup(); Ok(db) } + /// Starts background cleanup task fn cleanup(self) -> Self { let db = self.clone(); - std::thread::spawn(move || loop { - std::thread::sleep(std::time::Duration::from_secs(30)); - let mut async_conn = db.async_conn(); - let db_zkey = db.make_len_sortedset_key(); - futures::executor::block_on(async move { + tokio::spawn(async move { + loop { + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + let mut async_conn = db.async_conn(); + let db_zkey = db.make_len_sortedset_key(); if let Err(e) = async_conn .zrembyscore::<'_, _, _, _, ()>(db_zkey.as_slice(), 0, timestamp_millis()) .await { log::error!("{:?}", e); } - }); + } }); self } + /// Gets a clone of the async connection #[inline] fn async_conn(&self) -> RedisConnection { self.async_conn.clone() } + /// Gets a mutable reference to the async connection #[inline] fn async_conn_mut(&mut self) -> &mut RedisConnection { &mut self.async_conn } + /// Creates key for length tracking sorted set #[inline] #[allow(dead_code)] fn make_len_sortedset_key(&self) -> Key { [KEY_PREFIX_LEN, self.prefix.as_slice()].concat() } + /// Creates full key with prefix #[inline] fn make_full_key(&self, key: K) -> Key where @@ -115,11 +141,13 @@ impl RedisStorageDB { [KEY_PREFIX, self.prefix.as_slice(), key.as_ref()].concat() } + /// Creates scan pattern with prefix #[inline] fn make_scan_pattern_match>(&self, pattern: P) -> Key { [KEY_PREFIX, self.prefix.as_slice(), pattern.as_ref()].concat() } + /// Creates full map name with prefix #[inline] fn make_map_full_name(&self, name: K) -> Key where @@ -128,6 +156,7 @@ impl RedisStorageDB { [MAP_NAME_PREFIX, self.prefix.as_slice(), name.as_ref()].concat() } + /// Creates full list name with prefix #[inline] fn make_list_full_name(&self, name: K) -> Key where @@ -136,26 +165,31 @@ impl RedisStorageDB { [LIST_NAME_PREFIX, self.prefix.as_slice(), name.as_ref()].concat() } + /// Creates map prefix pattern for scanning #[inline] fn make_map_prefix_match(&self) -> Key { [MAP_NAME_PREFIX, self.prefix.as_slice(), b"*"].concat() } + /// Creates list prefix pattern for scanning #[inline] fn make_list_prefix_match(&self) -> Key { [LIST_NAME_PREFIX, self.prefix.as_slice(), b"*"].concat() } + /// Extracts map key from full name #[inline] fn map_full_name_to_key<'a>(&self, full_name: &'a [u8]) -> &'a [u8] { full_name[MAP_NAME_PREFIX.len() + self.prefix.len()..].as_ref() } + /// Extracts list key from full name #[inline] fn list_full_name_to_key<'a>(&self, full_name: &'a [u8]) -> &'a [u8] { full_name[LIST_NAME_PREFIX.len() + self.prefix.len()..].as_ref() } + /// Gets full key name for a given key #[inline] async fn _get_full_name(&self, key: &[u8]) -> Result { let map_full_name = self.make_map_full_name(key); @@ -173,6 +207,7 @@ impl RedisStorageDB { Ok(full_name) } + /// Internal method to insert a key-value pair #[inline] async fn _insert( &self, @@ -194,10 +229,11 @@ impl RedisStorageDB { .atomic() .set(full_key.as_slice(), bincode::serialize(val)?) .pexpire(full_key.as_slice(), expire_interval) - .query_async::<_, ()>(&mut async_conn) + .query_async::<()>(&mut async_conn) .await?; } else { - self.async_conn() + let _: () = self + .async_conn() .set(full_key, bincode::serialize(val)?) .await?; } @@ -212,14 +248,14 @@ impl RedisStorageDB { .set(full_key.as_slice(), bincode::serialize(val)?) .pexpire(full_key.as_slice(), expire_interval) .zadd(db_zkey, key.as_ref(), timestamp_millis() + expire_interval) - .query_async::<_, ()>(&mut async_conn) + .query_async::<()>(&mut async_conn) .await?; } else { pipe() .atomic() .set(full_key.as_slice(), bincode::serialize(val)?) .zadd(db_zkey, key.as_ref(), i64::MAX) - .query_async::<_, ()>(&mut async_conn) + .query_async::<()>(&mut async_conn) .await?; } } @@ -227,12 +263,12 @@ impl RedisStorageDB { Ok(()) } + /// Internal method for batch insertion #[inline] async fn _batch_insert( &self, key_val_expires: Vec<(Key, Vec, Option)>, ) -> Result<()> { - // let full_key = self.make_full_key(k); #[cfg(not(feature = "len"))] { let keys_vals: Vec<(Key, &Vec)> = key_val_expires @@ -248,7 +284,7 @@ impl RedisStorageDB { rpipe = rpipe.expire(k, at); } } - rpipe.query_async::<_, ()>(&mut async_conn).await?; + rpipe.query_async::<()>(&mut async_conn).await?; } #[cfg(feature = "len")] @@ -279,11 +315,12 @@ impl RedisStorageDB { rpipe = rpipe.expire(k, at); } } - rpipe.query_async::<_, ((), ())>(&mut async_conn).await?; + rpipe.query_async::<((), ())>(&mut async_conn).await?; } Ok(()) } + /// Internal method for batch removal #[inline] async fn _batch_remove(&self, keys: Vec) -> Result<()> { let full_keys = keys @@ -292,7 +329,7 @@ impl RedisStorageDB { .collect::>(); #[cfg(not(feature = "len"))] { - self.async_conn().del(full_keys).await?; + let _: () = self.async_conn().del(full_keys).await?; } #[cfg(feature = "len")] { @@ -302,12 +339,13 @@ impl RedisStorageDB { .atomic() .del(full_keys.as_slice()) .zrem(db_zkey, keys) - .query_async::<_, ()>(&mut async_conn) + .query_async::<()>(&mut async_conn) .await?; } Ok(()) } + /// Internal method to increment a counter #[inline] async fn _counter_incr( &self, @@ -327,10 +365,10 @@ impl RedisStorageDB { .atomic() .incr(full_key.as_slice(), increment) .pexpire(full_key.as_slice(), expire_interval) - .query_async::<_, ()>(&mut async_conn) + .query_async::<()>(&mut async_conn) .await?; } else { - self.async_conn().incr(full_key, increment).await?; + let _: () = self.async_conn().incr(full_key, increment).await?; } } #[cfg(feature = "len")] @@ -343,20 +381,21 @@ impl RedisStorageDB { .incr(full_key.as_slice(), increment) .pexpire(full_key.as_slice(), expire_interval) .zadd(db_zkey, key.as_ref(), timestamp_millis() + expire_interval) - .query_async::<_, ()>(&mut async_conn) + .query_async::<()>(&mut async_conn) .await?; } else { pipe() .atomic() .incr(full_key.as_slice(), increment) .zadd(db_zkey, key.as_ref(), i64::MAX) - .query_async::<_, ()>(&mut async_conn) + .query_async::<()>(&mut async_conn) .await?; } } Ok(()) } + /// Internal method to decrement a counter #[inline] async fn _counter_decr( &self, @@ -377,10 +416,10 @@ impl RedisStorageDB { .atomic() .decr(full_key.as_slice(), decrement) .pexpire(full_key.as_slice(), expire_interval) - .query_async::<_, ()>(&mut async_conn) + .query_async::<()>(&mut async_conn) .await?; } else { - self.async_conn().decr(full_key, decrement).await?; + let _: () = self.async_conn().decr(full_key, decrement).await?; } } #[cfg(feature = "len")] @@ -393,20 +432,21 @@ impl RedisStorageDB { .decr(full_key.as_slice(), decrement) .pexpire(full_key.as_slice(), expire_interval) .zadd(db_zkey, key.as_ref(), timestamp_millis() + expire_interval) - .query_async::<_, ()>(&mut async_conn) + .query_async::<()>(&mut async_conn) .await?; } else { pipe() .atomic() .decr(full_key.as_slice(), decrement) .zadd(db_zkey, key.as_ref(), i64::MAX) - .query_async::<_, ()>(&mut async_conn) + .query_async::<()>(&mut async_conn) .await?; } } Ok(()) } + /// Internal method to set a counter value #[inline] async fn _counter_set( &self, @@ -426,10 +466,10 @@ impl RedisStorageDB { .atomic() .set(full_key.as_slice(), val) .pexpire(full_key.as_slice(), expire_interval) - .query_async::<_, ()>(&mut async_conn) + .query_async::<()>(&mut async_conn) .await?; } else { - self.async_conn().set(full_key, val).await?; + let _: () = self.async_conn().set(full_key, val).await?; } } #[cfg(feature = "len")] @@ -442,14 +482,14 @@ impl RedisStorageDB { .set(full_key.as_slice(), val) .pexpire(full_key.as_slice(), expire_interval) .zadd(db_zkey, key.as_ref(), timestamp_millis() + expire_interval) - .query_async::<_, ()>(&mut async_conn) + .query_async::<()>(&mut async_conn) .await?; } else { pipe() .atomic() .set(full_key.as_slice(), val) .zadd(db_zkey, key.as_ref(), i64::MAX) - .query_async::<_, ()>(&mut async_conn) + .query_async::<()>(&mut async_conn) .await?; } } @@ -457,6 +497,7 @@ impl RedisStorageDB { Ok(()) } + /// Internal method to remove a key #[inline] async fn _remove(&self, key: K) -> Result<()> where @@ -466,7 +507,7 @@ impl RedisStorageDB { #[cfg(not(feature = "len"))] { - self.async_conn().del(full_key).await?; + let _: () = self.async_conn().del(full_key).await?; } #[cfg(feature = "len")] { @@ -476,7 +517,7 @@ impl RedisStorageDB { .atomic() .del(full_key.as_slice()) .zrem(db_zkey, key.as_ref()) - .query_async::<_, ()>(&mut async_conn) + .query_async::<()>(&mut async_conn) .await?; } Ok(()) @@ -488,6 +529,7 @@ impl StorageDB for RedisStorageDB { type MapType = RedisStorageMap; type ListType = RedisStorageList; + /// Creates a new map with optional expiration #[inline] async fn map + Sync + Send>( &self, @@ -501,22 +543,25 @@ impl StorageDB for RedisStorageDB { ) } + /// Removes a map #[inline] async fn map_remove(&self, name: K) -> Result<()> where K: AsRef<[u8]> + Sync + Send, { let map_full_name = self.make_map_full_name(name.as_ref()); - self.async_conn().del(map_full_name).await?; + let _: () = self.async_conn().del(map_full_name).await?; Ok(()) } + /// Checks if a map exists #[inline] async fn map_contains_key + Sync + Send>(&self, key: K) -> Result { let map_full_name = self.make_map_full_name(key.as_ref()); Ok(self.async_conn().exists(map_full_name).await?) } + /// Creates a new list with optional expiration #[inline] async fn list + Sync + Send>( &self, @@ -530,22 +575,25 @@ impl StorageDB for RedisStorageDB { ) } + /// Removes a list #[inline] async fn list_remove(&self, name: K) -> Result<()> where K: AsRef<[u8]> + Sync + Send, { let list_full_name = self.make_list_full_name(name.as_ref()); - self.async_conn().del(list_full_name).await?; + let _: () = self.async_conn().del(list_full_name).await?; Ok(()) } + /// Checks if a list exists #[inline] async fn list_contains_key + Sync + Send>(&self, key: K) -> Result { let list_full_name = self.make_list_full_name(key.as_ref()); Ok(self.async_conn().exists(list_full_name).await?) } + /// Inserts a key-value pair #[inline] async fn insert(&self, key: K, val: &V) -> Result<()> where @@ -555,6 +603,7 @@ impl StorageDB for RedisStorageDB { self._insert(key, val, None).await } + /// Gets a value by key #[inline] async fn get(&self, key: K) -> Result> where @@ -573,6 +622,7 @@ impl StorageDB for RedisStorageDB { } } + /// Removes a key #[inline] async fn remove(&self, key: K) -> Result<()> where @@ -581,6 +631,7 @@ impl StorageDB for RedisStorageDB { self._remove(key).await } + /// Batch insertion of key-value pairs #[inline] async fn batch_insert(&self, key_vals: Vec<(Key, V)>) -> Result<()> where @@ -600,6 +651,7 @@ impl StorageDB for RedisStorageDB { Ok(()) } + /// Batch removal of keys #[inline] async fn batch_remove(&self, keys: Vec) -> Result<()> { if !keys.is_empty() { @@ -608,6 +660,7 @@ impl StorageDB for RedisStorageDB { Ok(()) } + /// Increments a counter #[inline] async fn counter_incr(&self, key: K, increment: isize) -> Result<()> where @@ -616,6 +669,7 @@ impl StorageDB for RedisStorageDB { self._counter_incr(key, increment, None).await } + /// Decrements a counter #[inline] async fn counter_decr(&self, key: K, decrement: isize) -> Result<()> where @@ -624,6 +678,7 @@ impl StorageDB for RedisStorageDB { self._counter_decr(key, decrement, None).await } + /// Gets a counter value #[inline] async fn counter_get(&self, key: K) -> Result> where @@ -633,6 +688,7 @@ impl StorageDB for RedisStorageDB { Ok(self.async_conn().get::<_, Option>(full_key).await?) } + /// Sets a counter value #[inline] async fn counter_set(&self, key: K, val: isize) -> Result<()> where @@ -641,13 +697,14 @@ impl StorageDB for RedisStorageDB { self._counter_set(key, val, None).await } + /// Checks if a key exists #[inline] async fn contains_key + Sync + Send>(&self, key: K) -> Result { - //HEXISTS key field let full_key = self.make_full_key(key.as_ref()); Ok(self.async_conn().exists(full_key).await?) } + /// Gets the number of keys in the database #[inline] #[cfg(feature = "len")] async fn len(&self) -> Result { @@ -656,18 +713,19 @@ impl StorageDB for RedisStorageDB { let (_, count) = pipe() .zrembyscore(db_zkey.as_slice(), 0, timestamp_millis()) .zcard(db_zkey.as_slice()) - .query_async::<_, (i64, usize)>(&mut async_conn) + .query_async::<(i64, usize)>(&mut async_conn) .await?; Ok(count) } + /// Gets the total database size #[inline] async fn db_size(&self) -> Result { let mut async_conn = self.async_conn(); //DBSIZE let dbsize = redis::pipe() .cmd("DBSIZE") - .query_async::<_, redis::Value>(&mut async_conn) + .query_async::(&mut async_conn) .await?; let dbsize = dbsize.as_sequence().and_then(|vs| { vs.iter().next().and_then(|v| { @@ -681,6 +739,7 @@ impl StorageDB for RedisStorageDB { Ok(dbsize.unwrap_or(0) as usize) } + /// Sets expiration time for a key #[inline] #[cfg(feature = "ttl")] async fn expire_at(&self, key: K, at: TimestampMillis) -> Result @@ -704,12 +763,13 @@ impl StorageDB for RedisStorageDB { .atomic() .zadd(db_zkey, key.as_ref(), at) .pexpire_at(full_name.as_slice(), at) - .query_async::<_, (i64, bool)>(&mut async_conn) + .query_async::<(i64, bool)>(&mut async_conn) .await?; Ok(res) } } + /// Sets expiration duration for a key #[inline] #[cfg(feature = "ttl")] async fn expire(&self, key: K, dur: TimestampMillis) -> Result @@ -731,12 +791,13 @@ impl StorageDB for RedisStorageDB { .atomic() .zadd(db_zkey, key.as_ref(), timestamp_millis() + dur) .pexpire(full_name.as_slice(), dur) - .query_async::<_, (i64, bool)>(&mut async_conn) + .query_async::<(i64, bool)>(&mut async_conn) .await?; Ok(res) } } + /// Gets time-to-live for a key #[inline] #[cfg(feature = "ttl")] async fn ttl(&self, key: K) -> Result> @@ -753,6 +814,7 @@ impl StorageDB for RedisStorageDB { } } + /// Creates an iterator for all maps #[inline] async fn map_iter<'a>( &'a mut self, @@ -765,6 +827,7 @@ impl StorageDB for RedisStorageDB { Ok(Box::new(iter)) } + /// Creates an iterator for all lists #[inline] async fn list_iter<'a>( &'a mut self, @@ -777,6 +840,7 @@ impl StorageDB for RedisStorageDB { Ok(Box::new(iter)) } + /// Creates an iterator for keys matching a pattern async fn scan<'a, P>( &'a mut self, pattern: P, @@ -796,12 +860,13 @@ impl StorageDB for RedisStorageDB { Ok(Box::new(iter)) } + /// Gets database information #[inline] async fn info(&self) -> Result { let mut conn = self.async_conn(); let dbsize = redis::pipe() .cmd("dbsize") - .query_async::<_, redis::Value>(&mut conn) + .query_async::(&mut conn) .await?; let dbsize = dbsize.as_sequence().and_then(|vs| { vs.iter().next().and_then(|v| { @@ -819,17 +884,24 @@ impl StorageDB for RedisStorageDB { } } +/// Redis-backed map storage implementation #[derive(Clone)] pub struct RedisStorageMap { + /// Name of the map name: Key, + /// Full key name with prefix full_name: Key, + /// Optional expiration time in milliseconds #[allow(dead_code)] expire: Option, + /// Flag indicating if the map is empty empty: Arc, + /// Reference to the parent database pub(crate) db: RedisStorageDB, } impl RedisStorageMap { + /// Creates a new map without expiration #[inline] pub(crate) fn new(name: Key, full_name: Key, db: RedisStorageDB) -> Self { Self { @@ -841,6 +913,7 @@ impl RedisStorageMap { } } + /// Creates a new map with expiration #[inline] pub(crate) async fn new_expire( name: Key, @@ -863,19 +936,21 @@ impl RedisStorageMap { }) } + /// Gets a clone of the async connection #[inline] fn async_conn(&self) -> RedisConnection { self.db.async_conn() } + /// Gets a mutable reference to the async connection #[inline] fn async_conn_mut(&mut self) -> &mut RedisConnection { self.db.async_conn_mut() } + /// Checks if the map is empty #[inline] async fn _is_empty(async_conn: &mut RedisConnection, full_name: &[u8]) -> Result { - //HSCAN key cursor [MATCH pattern] [COUNT count] let res = async_conn .hscan::<_, Vec>(full_name) .await? @@ -885,6 +960,7 @@ impl RedisStorageMap { Ok(res) } + /// Internal method to insert with expiration handling #[inline] async fn _insert_expire(&self, key: &[u8], val: Vec) -> Result<()> { let mut async_conn = self.async_conn(); @@ -893,9 +969,7 @@ impl RedisStorageMap { #[cfg(feature = "ttl")] if self.empty.load(Ordering::SeqCst) { if let Some(expire) = self.expire.as_ref() { - //HSET key field value - //PEXPIRE key ms - redis::pipe() + let _: () = redis::pipe() .atomic() .hset(name, key, val) .pexpire(name, *expire) @@ -906,11 +980,11 @@ impl RedisStorageMap { } } - //HSET key field value - async_conn.hset(name, key.as_ref(), val).await?; + let _: () = async_conn.hset(name, key.as_ref(), val).await?; Ok(()) } + /// Internal method for batch insertion with expiration #[inline] async fn _batch_insert_expire(&self, key_vals: Vec<(Key, Vec)>) -> Result<()> { let mut async_conn = self.async_conn(); @@ -919,9 +993,7 @@ impl RedisStorageMap { #[cfg(feature = "ttl")] if self.empty.load(Ordering::SeqCst) { if let Some(expire) = self.expire.as_ref() { - //HMSET key field value - //PEXPIRE key ms - redis::pipe() + let _: () = redis::pipe() .atomic() .hset_multiple(name, key_vals.as_slice()) .pexpire(name, *expire) @@ -933,19 +1005,20 @@ impl RedisStorageMap { } } - //HSET key field value - async_conn.hset_multiple(name, key_vals.as_slice()).await?; + let _: () = async_conn.hset_multiple(name, key_vals.as_slice()).await?; Ok(()) } } #[async_trait] impl Map for RedisStorageMap { + /// Gets the map name #[inline] fn name(&self) -> &[u8] { self.name.as_slice() } + /// Inserts a key-value pair into the map #[inline] async fn insert(&self, key: K, val: &V) -> Result<()> where @@ -956,13 +1029,13 @@ impl Map for RedisStorageMap { .await } + /// Gets a value from the map #[inline] async fn get(&self, key: K) -> Result> where K: AsRef<[u8]> + Sync + Send, V: DeserializeOwned + Sync + Send, { - //HSET key field value let res: Option> = self .async_conn() .hget(self.full_name.as_slice(), key.as_ref()) @@ -974,21 +1047,22 @@ impl Map for RedisStorageMap { } } + /// Removes a key from the map #[inline] async fn remove(&self, key: K) -> Result<()> where K: AsRef<[u8]> + Sync + Send, { - //HDEL key field [field ...] - self.async_conn() + let _: () = self + .async_conn() .hdel(self.full_name.as_slice(), key.as_ref()) .await?; Ok(()) } + /// Checks if a key exists in the map #[inline] async fn contains_key + Sync + Send>(&self, key: K) -> Result { - //HEXISTS key field let res = self .async_conn() .hexists(self.full_name.as_slice(), key.as_ref()) @@ -996,16 +1070,16 @@ impl Map for RedisStorageMap { Ok(res) } + /// Gets the number of elements in the map #[cfg(feature = "map_len")] #[inline] async fn len(&self) -> Result { - //HLEN key Ok(self.async_conn().hlen(self.full_name.as_slice()).await?) } + /// Checks if the map is empty #[inline] async fn is_empty(&self) -> Result { - //HSCAN key cursor [MATCH pattern] [COUNT count] let res = self .async_conn() .hscan::<_, Vec>(self.full_name.as_slice()) @@ -1016,22 +1090,21 @@ impl Map for RedisStorageMap { Ok(res) } + /// Clears all elements from the map #[inline] async fn clear(&self) -> Result<()> { - //DEL key [key ...] - self.async_conn().del(self.full_name.as_slice()).await?; + let _: () = self.async_conn().del(self.full_name.as_slice()).await?; self.empty.store(true, Ordering::SeqCst); Ok(()) } + /// Removes and returns a value from the map #[inline] async fn remove_and_fetch(&self, key: K) -> Result> where K: AsRef<[u8]> + Sync + Send, V: DeserializeOwned + Sync + Send, { - //HSET key field value - //HDEL key field [field ...] let name = self.full_name.as_slice(); let mut conn = self.async_conn(); let (res, _): (Option>, isize) = redis::pipe() @@ -1048,6 +1121,7 @@ impl Map for RedisStorageMap { } } + /// Removes all keys with a given prefix #[inline] async fn remove_with_prefix(&self, prefix: K) -> Result<()> where @@ -1065,18 +1139,19 @@ impl Map for RedisStorageMap { .next_item() .await { - removeds.push(key); + removeds.push(key?); if removeds.len() > 20 { - conn2.hdel(name, removeds.as_slice()).await?; + let _: () = conn2.hdel(name, removeds.as_slice()).await?; removeds.clear(); } } if !removeds.is_empty() { - conn.hdel(name, removeds).await?; + let _: () = conn.hdel(name, removeds).await?; } Ok(()) } + /// Batch insertion of key-value pairs #[inline] async fn batch_insert(&self, key_vals: Vec<(Key, V)>) -> Result<()> where @@ -1097,16 +1172,19 @@ impl Map for RedisStorageMap { Ok(()) } + /// Batch removal of keys #[inline] async fn batch_remove(&self, keys: Vec) -> Result<()> { if !keys.is_empty() { - self.async_conn() + let _: () = self + .async_conn() .hdel(self.full_name.as_slice(), keys) .await?; } Ok(()) } + /// Creates an iterator over key-value pairs #[inline] async fn iter<'a, V>( &'a mut self, @@ -1125,6 +1203,7 @@ impl Map for RedisStorageMap { Ok(Box::new(iter)) } + /// Creates an iterator over keys #[inline] async fn key_iter<'a>( &'a mut self, @@ -1139,6 +1218,7 @@ impl Map for RedisStorageMap { Ok(Box::new(iter)) } + /// Creates an iterator over key-value pairs with a prefix #[inline] async fn prefix_iter<'a, P, V>( &'a mut self, @@ -1161,6 +1241,7 @@ impl Map for RedisStorageMap { Ok(Box::new(iter)) } + /// Sets expiration time for the map #[cfg(feature = "ttl")] async fn expire_at(&self, at: TimestampMillis) -> Result { let res = self @@ -1170,6 +1251,7 @@ impl Map for RedisStorageMap { Ok(res) } + /// Sets expiration duration for the map #[cfg(feature = "ttl")] async fn expire(&self, dur: TimestampMillis) -> Result { let res = self @@ -1179,6 +1261,7 @@ impl Map for RedisStorageMap { Ok(res) } + /// Gets time-to-live for the map #[cfg(feature = "ttl")] async fn ttl(&self) -> Result> { let mut async_conn = self.async_conn(); @@ -1193,17 +1276,24 @@ impl Map for RedisStorageMap { } } +/// Redis-backed list storage implementation #[derive(Clone)] pub struct RedisStorageList { + /// Name of the list name: Key, + /// Full key name with prefix full_name: Key, + /// Optional expiration time in milliseconds #[allow(dead_code)] expire: Option, + /// Flag indicating if the list is empty empty: Arc, + /// Reference to the parent database pub(crate) db: RedisStorageDB, } impl RedisStorageList { + /// Creates a new list without expiration #[inline] pub(crate) fn new(name: Key, full_name: Key, db: RedisStorageDB) -> Self { Self { @@ -1215,6 +1305,7 @@ impl RedisStorageList { } } + /// Creates a new list with expiration #[inline] pub(crate) async fn new_expire( name: Key, @@ -1237,16 +1328,19 @@ impl RedisStorageList { }) } + /// Gets a clone of the async connection #[inline] pub(crate) fn async_conn(&self) -> RedisConnection { self.db.async_conn() } + /// Checks if the list is empty #[inline] async fn _is_empty(async_conn: &mut RedisConnection, full_name: &[u8]) -> Result { Ok(async_conn.llen::<_, usize>(full_name).await? == 0) } + /// Internal method to push with expiration handling #[inline] async fn _push_expire(&self, val: Vec) -> Result<()> { let mut async_conn = self.async_conn(); @@ -1255,9 +1349,7 @@ impl RedisStorageList { #[cfg(feature = "ttl")] if self.empty.load(Ordering::SeqCst) { if let Some(expire) = self.expire.as_ref() { - //RPUSH key value [value ...] - //PEXPIRE key ms - redis::pipe() + let _: () = redis::pipe() .atomic() .rpush(name, val) .pexpire(name, *expire) @@ -1268,11 +1360,11 @@ impl RedisStorageList { } } - //RPUSH key value [value ...] - async_conn.rpush(name, val).await?; + let _: () = async_conn.rpush(name, val).await?; Ok(()) } + /// Internal method for batch push with expiration #[inline] async fn _pushs_expire(&self, vals: Vec>) -> Result<()> { let mut async_conn = self.async_conn(); @@ -1281,9 +1373,7 @@ impl RedisStorageList { if self.empty.load(Ordering::SeqCst) { if let Some(expire) = self.expire.as_ref() { let name = self.full_name.as_slice(); - //RPUSH key value [value ...] - //PEXPIRE key ms - redis::pipe() + let _: () = redis::pipe() .atomic() .rpush(name, vals) .pexpire(name, *expire) @@ -1294,11 +1384,11 @@ impl RedisStorageList { } } - //RPUSH key value [value ...] - async_conn.rpush(self.full_name.as_slice(), vals).await?; + let _: () = async_conn.rpush(self.full_name.as_slice(), vals).await?; Ok(()) } + /// Internal method for push with limit and expiration #[inline] async fn _push_limit_expire( &self, @@ -1314,7 +1404,7 @@ impl RedisStorageList { let name = self.full_name.as_slice(); let count = conn.llen::<_, usize>(name).await?; let res = if count < limit { - redis::pipe() + let _: () = redis::pipe() .atomic() .rpush(name, val) .pexpire(name, *expire) @@ -1343,6 +1433,7 @@ impl RedisStorageList { .await } + /// Internal method for push with limit #[inline] async fn _push_limit( &self, @@ -1355,7 +1446,7 @@ impl RedisStorageList { let count = async_conn.llen::<_, usize>(name).await?; if count < limit { - async_conn.rpush(name, val).await?; + let _: () = async_conn.rpush(name, val).await?; Ok(None) } else if pop_front_if_limited { let (poped, _): (Option>, Option<()>) = redis::pipe() @@ -1373,11 +1464,13 @@ impl RedisStorageList { #[async_trait] impl List for RedisStorageList { + /// Gets the list name #[inline] fn name(&self) -> &[u8] { self.name.as_slice() } + /// Pushes a value to the end of the list #[inline] async fn push(&self, val: &V) -> Result<()> where @@ -1386,12 +1479,12 @@ impl List for RedisStorageList { self._push_expire(bincode::serialize(val)?).await } + /// Pushes multiple values to the end of the list #[inline] async fn pushs(&self, vals: Vec) -> Result<()> where V: Serialize + Sync + Send, { - //RPUSH key value [value ...] let vals = vals .into_iter() .map(|v| bincode::serialize(&v).map_err(|e| anyhow!(e))) @@ -1399,6 +1492,7 @@ impl List for RedisStorageList { self._pushs_expire(vals).await } + /// Pushes a value with size limit handling #[inline] async fn push_limit( &self, @@ -1424,12 +1518,12 @@ impl List for RedisStorageList { } } + /// Pops a value from the front of the list #[inline] async fn pop(&self) -> Result> where V: DeserializeOwned + Sync + Send, { - //LPOP key let removed = self .async_conn() .lpop::<_, Option>>(self.full_name.as_slice(), None) @@ -1444,12 +1538,12 @@ impl List for RedisStorageList { Ok(removed) } + /// Gets all values in the list #[inline] async fn all(&self) -> Result> where V: DeserializeOwned + Sync + Send, { - //LRANGE key 0 -1 let all = self .async_conn() .lrange::<_, Vec>>(self.full_name.as_slice(), 0, -1) @@ -1459,12 +1553,12 @@ impl List for RedisStorageList { .collect::>>() } + /// Gets a value by index #[inline] async fn get_index(&self, idx: usize) -> Result> where V: DeserializeOwned + Sync + Send, { - //LINDEX key index let val = self .async_conn() .lindex::<_, Option>>(self.full_name.as_slice(), idx as isize) @@ -1477,24 +1571,27 @@ impl List for RedisStorageList { }) } + /// Gets the length of the list #[inline] async fn len(&self) -> Result { - //LLEN key Ok(self.async_conn().llen(self.full_name.as_slice()).await?) } + /// Checks if the list is empty #[inline] async fn is_empty(&self) -> Result { Ok(self.len().await? == 0) } + /// Clears the list #[inline] async fn clear(&self) -> Result<()> { - self.async_conn().del(self.full_name.as_slice()).await?; + let _: () = self.async_conn().del(self.full_name.as_slice()).await?; self.empty.store(true, Ordering::SeqCst); Ok(()) } + /// Creates an iterator over list values #[inline] async fn iter<'a, V>( &'a mut self, @@ -1508,6 +1605,7 @@ impl List for RedisStorageList { ))) } + /// Sets expiration time for the list #[cfg(feature = "ttl")] async fn expire_at(&self, at: TimestampMillis) -> Result { let res = self @@ -1517,6 +1615,7 @@ impl List for RedisStorageList { Ok(res) } + /// Sets expiration duration for the list #[cfg(feature = "ttl")] async fn expire(&self, dur: TimestampMillis) -> Result { let res = self @@ -1526,6 +1625,7 @@ impl List for RedisStorageList { Ok(res) } + /// Gets time-to-live for the list #[cfg(feature = "ttl")] async fn ttl(&self) -> Result> { let mut async_conn = self.async_conn(); @@ -1540,6 +1640,7 @@ impl List for RedisStorageList { } } +/// Iterator for list values pub struct AsyncListValIter<'a, V> { name: &'a [u8], conn: RedisConnection, @@ -1550,6 +1651,7 @@ pub struct AsyncListValIter<'a, V> { } impl<'a, V> AsyncListValIter<'a, V> { + /// Creates a new list value iterator fn new(name: &'a [u8], conn: RedisConnection) -> Self { let start = 0; let limit = 20; @@ -1565,7 +1667,7 @@ impl<'a, V> AsyncListValIter<'a, V> { } #[async_trait] -impl<'a, V> AsyncIterator for AsyncListValIter<'a, V> +impl AsyncIterator for AsyncListValIter<'_, V> where V: DeserializeOwned + Sync + Send + 'static, { @@ -1599,6 +1701,7 @@ where } } +/// Iterator for map entries pub struct AsyncIter<'a, V> { iter: redis::AsyncIter<'a, (Key, Vec)>, _m: std::marker::PhantomData, @@ -1612,82 +1715,96 @@ where type Item = IterItem; async fn next(&mut self) -> Option { - let item = self.iter.next_item().await; - item.map(|(key, v)| match bincode::deserialize::(v.as_ref()) { - Ok(v) => Ok((key, v)), - Err(e) => Err(anyhow::Error::new(e)), - }) + match self.iter.next_item().await { + None => None, + Some(Err(e)) => Some(Err(anyhow::Error::new(e))), + Some(Ok((key, v))) => match bincode::deserialize::(v.as_ref()) { + Ok(v) => Some(Ok((key, v))), + Err(e) => Some(Err(anyhow::Error::new(e))), + }, + } } } +/// Iterator for database keys pub struct AsyncDbKeyIter<'a> { prefix_len: usize, iter: redis::AsyncIter<'a, Key>, } #[async_trait] -impl<'a> AsyncIterator for AsyncDbKeyIter<'a> { +impl AsyncIterator for AsyncDbKeyIter<'_> { type Item = Result; async fn next(&mut self) -> Option { - self.iter - .next_item() - .await - .map(|key| Ok(key[self.prefix_len..].to_vec())) + match self.iter.next_item().await { + None => None, + Some(Err(e)) => Some(Err(anyhow::Error::new(e))), + Some(Ok(key)) => Some(Ok(key[self.prefix_len..].to_vec())), + } } } +/// Iterator for map keys pub struct AsyncKeyIter<'a> { iter: redis::AsyncIter<'a, (Key, ())>, } #[async_trait] -impl<'a> AsyncIterator for AsyncKeyIter<'a> { +impl AsyncIterator for AsyncKeyIter<'_> { type Item = Result; async fn next(&mut self) -> Option { - self.iter.next_item().await.map(|(key, _)| Ok(key)) + match self.iter.next_item().await { + None => None, + Some(Err(e)) => Some(Err(anyhow::Error::new(e))), + Some(Ok((key, _))) => Some(Ok(key)), + } } } +/// Iterator for maps pub struct AsyncMapIter<'a> { db: RedisStorageDB, iter: redis::AsyncIter<'a, Key>, } #[async_trait] -impl<'a> AsyncIterator for AsyncMapIter<'a> { +impl AsyncIterator for AsyncMapIter<'_> { type Item = Result; async fn next(&mut self) -> Option { - let full_name = self.iter.next_item().await; - if let Some(full_name) = full_name { - let name = self.db.map_full_name_to_key(full_name.as_slice()).to_vec(); - let m = RedisStorageMap::new(name, full_name, self.db.clone()); - Some(Ok(StorageMap::Redis(m))) - } else { - None - } + let full_name = match self.iter.next_item().await { + None => return None, + Some(Err(e)) => return Some(Err(anyhow::Error::new(e))), + Some(Ok(key)) => key, + }; + + let name = self.db.map_full_name_to_key(full_name.as_slice()).to_vec(); + let m = RedisStorageMap::new(name, full_name, self.db.clone()); + Some(Ok(StorageMap::Redis(m))) } } +/// Iterator for lists pub struct AsyncListIter<'a> { db: RedisStorageDB, iter: redis::AsyncIter<'a, Key>, } #[async_trait] -impl<'a> AsyncIterator for AsyncListIter<'a> { +impl AsyncIterator for AsyncListIter<'_> { type Item = Result; async fn next(&mut self) -> Option { - let full_name = self.iter.next_item().await; - if let Some(full_name) = full_name { - let name = self.db.list_full_name_to_key(full_name.as_slice()).to_vec(); - let l = RedisStorageList::new(name, full_name, self.db.clone()); - Some(Ok(StorageList::Redis(l))) - } else { - None - } + let full_name = match self.iter.next_item().await { + None => return None, + Some(Err(e)) => return Some(Err(anyhow::Error::new(e))), + Some(Ok(key)) => key, + }; + + let name = self.db.list_full_name_to_key(full_name.as_slice()).to_vec(); + let l = RedisStorageList::new(name, full_name, self.db.clone()); + Some(Ok(StorageList::Redis(l))) } } diff --git a/src/storage_redis_cluster.rs b/src/storage_redis_cluster.rs new file mode 100644 index 0000000..099889c --- /dev/null +++ b/src/storage_redis_cluster.rs @@ -0,0 +1,2092 @@ +//! Redis-based storage implementation for key-value, map, and list data structures. +//! +//! This module provides a Redis-backed storage system with support for: +//! - Key-value storage with expiration +//! - Map (hash) data structures +//! - List data structures +//! - Counters with atomic operations +//! - Iteration and scanning capabilities +//! - Cluster-aware operations + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::anyhow; +use async_trait::async_trait; +use redis::{ + aio::{ConnectionLike, ConnectionManager, ConnectionManagerConfig}, + cluster::ClusterClient, + cluster_async::ClusterConnection, + cluster_routing::get_slot, + pipe, AsyncCommands, Cmd, +}; +use serde::de::DeserializeOwned; +use serde::Deserialize; +use serde::Serialize; +use serde_json::Value; + +use crate::storage::{AsyncIterator, IterItem, Key, List, Map, StorageDB}; +use crate::{Result, StorageList, StorageMap}; + +#[allow(unused_imports)] +use crate::{timestamp_millis, TimestampMillis}; + +use crate::storage::{KEY_PREFIX, KEY_PREFIX_LEN, LIST_NAME_PREFIX, MAP_NAME_PREFIX, SEPARATOR}; + +/// Type alias for Redis cluster connection +type RedisConnection = ClusterConnection; + +/// Configuration for Redis storage +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RedisConfig { + /// Redis server URLs + pub urls: Vec, + + /// Key prefix for all storage operations + pub prefix: String, +} + +impl Default for RedisConfig { + fn default() -> Self { + RedisConfig { + urls: Vec::default(), + prefix: "__def".into(), + } + } +} + +/// Redis storage database implementation +#[derive(Clone)] +pub struct RedisStorageDB { + /// Prefix for all keys + prefix: Key, + /// Asynchronous connection to Redis cluster + async_conn: RedisConnection, + /// Connection managers for cluster nodes + nodes: Vec, + /// Last update time for cluster nodes + nodes_update_time: TimestampMillis, +} + +impl RedisStorageDB { + /// Creates a new Redis storage instance + #[inline] + pub(crate) async fn new(cfg: RedisConfig) -> Result { + let prefix = [cfg.prefix.as_bytes(), SEPARATOR].concat(); + + let client = ClusterClient::builder(cfg.urls) + .retry_wait_formula(2, 100) + .retries(2) + .connection_timeout(Duration::from_secs(15)) + .response_timeout(Duration::from_secs(10)) + .build()?; + + let async_conn = client.get_async_connection().await?; + + let mut db = Self { + prefix, + async_conn, + nodes: Vec::new(), + nodes_update_time: timestamp_millis(), + } + .cleanup(); + db.refresh_cluster_nodes().await?; + Ok(db) + } + + /// Starts background cleanup task + fn cleanup(self) -> Self { + let db = self.clone(); + tokio::spawn(async move { + loop { + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + let mut async_conn = db.async_conn(); + let db_zkey = db.make_len_sortedset_key(); + if let Err(e) = async_conn + .zrembyscore::<'_, _, _, _, ()>(db_zkey.as_slice(), 0, timestamp_millis()) + .await + { + log::error!("{:?}", e); + } + } + }); + self + } + + /// Gets a clone of the async connection + #[inline] + fn async_conn(&self) -> RedisConnection { + self.async_conn.clone() + } + + /// Gets a mutable reference to the async connection + #[inline] + fn async_conn_mut(&mut self) -> &mut RedisConnection { + &mut self.async_conn + } + + /// Refreshes cluster node information + #[inline] + async fn refresh_cluster_nodes(&mut self) -> Result<()> { + let slots = self + .async_conn() + .req_packed_command(Cmd::new().arg("CLUSTER").arg("SLOTS")) + .await?; + + let mut addrs = Vec::new(); + for slot in slots + .as_sequence() + .map(|arrs| { + arrs.iter() + .filter_map(|obj| obj.as_sequence()) + .collect::>() + }) + .iter() + .flatten() + .collect::>() + { + if let Some(addr_info) = slot.get(2) { + if let Some(addr_items) = addr_info.as_sequence() { + if addr_items.len() > 1 { + if let (redis::Value::BulkString(addr), redis::Value::Int(port)) = + (&addr_items[0], &addr_items[1]) + { + let addr = + format!("redis://{}:{}", String::from_utf8_lossy(addr), *port); + addrs.push(addr); + } + } + } + } + } + + // let mut nodes = BTreeMap::new(); + let mut nodes = Vec::new(); + for addr in addrs { + let client = match redis::Client::open(addr.as_str()) { + Ok(c) => c, + Err(e) => { + log::error!("open redis node error, addr is {}, {:?}", addr, e); + return Err(anyhow!(e)); + } + }; + let mgr_cfg = ConnectionManagerConfig::default() + .set_exponent_base(100) + .set_factor(2) + .set_number_of_retries(2) + .set_connection_timeout(Duration::from_secs(15)) + .set_response_timeout(Duration::from_secs(10)); + let conn = match client.get_connection_manager_with_config(mgr_cfg).await { + Ok(conn) => conn, + Err(e) => { + log::error!("get redis connection error, addr {:?}, {:?}", addr, e); + return Err(anyhow!(e)); + } + }; + nodes.push(conn); + } + self.nodes = nodes; + log::info!("nodes.len(): {:?}", self.nodes.len()); + Ok(()) + } + + /// Gets mutable reference to cluster nodes with refresh + #[inline] + async fn nodes_mut(&mut self) -> Result<&mut Vec> { + //Refresh after a certain interval. + if (timestamp_millis() - self.nodes_update_time) > 5000 { + self.refresh_cluster_nodes().await?; + } + Ok(&mut self.nodes) + } + + /// Creates key for length tracking sorted set + #[inline] + #[allow(dead_code)] + fn make_len_sortedset_key(&self) -> Key { + [KEY_PREFIX_LEN, self.prefix.as_slice()].concat() + } + + /// Creates full key with prefix + #[inline] + fn make_full_key(&self, key: K) -> Key + where + K: AsRef<[u8]>, + { + [KEY_PREFIX, self.prefix.as_slice(), key.as_ref()].concat() + } + + /// Creates scan pattern with prefix + #[inline] + fn make_scan_pattern_match>(&self, pattern: P) -> Key { + [KEY_PREFIX, self.prefix.as_slice(), pattern.as_ref()].concat() + } + + /// Creates full map name with prefix + #[inline] + fn make_map_full_name(&self, name: K) -> Key + where + K: AsRef<[u8]>, + { + [MAP_NAME_PREFIX, self.prefix.as_slice(), name.as_ref()].concat() + } + + /// Creates full list name with prefix + #[inline] + fn make_list_full_name(&self, name: K) -> Key + where + K: AsRef<[u8]>, + { + [LIST_NAME_PREFIX, self.prefix.as_slice(), name.as_ref()].concat() + } + + /// Creates map prefix pattern for scanning + #[inline] + fn make_map_prefix_match(&self) -> Key { + [MAP_NAME_PREFIX, self.prefix.as_slice(), b"*"].concat() + } + + /// Creates list prefix pattern for scanning + #[inline] + fn make_list_prefix_match(&self) -> Key { + [LIST_NAME_PREFIX, self.prefix.as_slice(), b"*"].concat() + } + + /// Extracts map key from full name + #[inline] + fn map_full_name_to_key<'a>(&self, full_name: &'a [u8]) -> &'a [u8] { + full_name[MAP_NAME_PREFIX.len() + self.prefix.len()..].as_ref() + } + + /// Extracts list key from full name + #[inline] + fn list_full_name_to_key<'a>(&self, full_name: &'a [u8]) -> &'a [u8] { + full_name[LIST_NAME_PREFIX.len() + self.prefix.len()..].as_ref() + } + + /// Gets full key name for a given key + #[inline] + async fn _get_full_name(&self, key: &[u8]) -> Result { + let map_full_name = self.make_map_full_name(key); + let mut async_conn = self.async_conn(); + let full_name = if async_conn.exists(map_full_name.as_slice()).await? { + map_full_name + } else { + let list_full_name = self.make_list_full_name(key); + if async_conn.exists(list_full_name.as_slice()).await? { + list_full_name + } else { + self.make_full_key(key) + } + }; + Ok(full_name) + } + + /// Internal method to insert a key-value pair + #[inline] + async fn _insert( + &self, + _key: K, + _val: &V, + _expire_interval: Option, + ) -> Result<()> + where + K: AsRef<[u8]> + Sync + Send, + V: serde::ser::Serialize + Sync + Send, + { + #[cfg(not(feature = "len"))] + { + let full_key = self.make_full_key(_key.as_ref()); + if let Some(expire_interval) = _expire_interval { + let mut async_conn = self.async_conn(); + pipe() + .atomic() + .set(full_key.as_slice(), bincode::serialize(_val)?) + .pexpire(full_key.as_slice(), expire_interval) + .query_async::<()>(&mut async_conn) + .await?; + } else { + let _: () = self + .async_conn() + .set(full_key, bincode::serialize(_val)?) + .await?; + } + } + #[cfg(feature = "len")] + { + return Err(anyhow!("unsupported!")); + //@TODO ... + #[allow(unreachable_code)] + { + let full_key = self.make_full_key(_key.as_ref()); + let db_zkey = self.make_len_sortedset_key(); + let mut async_conn = self.async_conn(); + if get_slot(db_zkey.as_slice()) == get_slot(full_key.as_slice()) { + if let Some(expire_interval) = _expire_interval { + pipe() + .atomic() + .set(full_key.as_slice(), bincode::serialize(_val)?) + .pexpire(full_key.as_slice(), expire_interval) + .zadd(db_zkey, _key.as_ref(), timestamp_millis() + expire_interval) + .query_async::<()>(&mut async_conn) + .await?; + } else { + pipe() + .atomic() + .set(full_key.as_slice(), bincode::serialize(_val)?) + .zadd(db_zkey, _key.as_ref(), i64::MAX) + .query_async::<()>(&mut async_conn) + .await?; + } + } else { + // + if let Some(expire_interval) = _expire_interval { + let _: () = pipe() + .atomic() + .set(full_key.as_slice(), bincode::serialize(_val)?) + .pexpire(full_key.as_slice(), expire_interval) + .query_async(&mut async_conn) + .await?; + let _: () = async_conn + .zadd(db_zkey, _key.as_ref(), timestamp_millis() + expire_interval) + .await?; + } else { + let _: () = async_conn + .set(full_key.as_slice(), bincode::serialize(_val)?) + .await?; + let _: () = async_conn.zadd(db_zkey, _key.as_ref(), i64::MAX).await?; + } + } + } + } + + Ok(()) + } + + /// Internal method for batch insertion + #[inline] + async fn _batch_insert( + &self, + _key_val_expires: Vec<(Key, Key, V, Option)>, + ) -> Result<()> + where + V: serde::ser::Serialize + Sync + Send, + { + #[cfg(not(feature = "len"))] + { + let keys_vals: Vec<(&Key, Vec)> = _key_val_expires + .iter() + .map(|(_, full_key, v, _)| { + bincode::serialize(&v) + .map(move |v| (full_key, v)) + .map_err(|e| anyhow!(e)) + }) + .collect::>>()?; + + let mut async_conn = self.async_conn(); + let mut p = pipe(); + let mut rpipe = p.atomic().mset(keys_vals.as_slice()); + for (_, full_key, _, at) in _key_val_expires { + if let Some(at) = at { + rpipe = rpipe.expire(full_key, at); + } + } + rpipe.query_async::<()>(&mut async_conn).await?; + Ok(()) + } + + #[cfg(feature = "len")] + { + return Err(anyhow!("unsupported!")); + //@TODO ... + #[allow(unreachable_code)] + { + for (k, _, v, expire) in _key_val_expires { + self._insert(k.as_slice(), &v, expire).await?; + } + Ok(()) + } + } + } + + /// Internal method for batch removal + #[inline] + async fn _batch_remove(&self, _keys: Vec) -> Result<()> { + #[cfg(not(feature = "len"))] + { + let full_keys = _keys + .iter() + .map(|k| self.make_full_key(k)) + .collect::>(); + let _: () = self.async_conn().del(full_keys).await?; + } + #[cfg(feature = "len")] + { + return Err(anyhow!("unsupported!")); + #[allow(unreachable_code)] + //@TODO ... + { + for key in _keys { + self._remove(key).await?; + } + } + } + Ok(()) + } + + /// Internal method to increment a counter + #[inline] + async fn _counter_incr( + &self, + _key: K, + _increment: isize, + _expire_interval: Option, + ) -> Result<()> + where + K: AsRef<[u8]> + Sync + Send, + { + #[cfg(not(feature = "len"))] + { + let full_key = self.make_full_key(_key.as_ref()); + if let Some(expire_interval) = _expire_interval { + let mut async_conn = self.async_conn(); + pipe() + .atomic() + .incr(full_key.as_slice(), _increment) + .pexpire(full_key.as_slice(), expire_interval) + .query_async::<()>(&mut async_conn) + .await?; + } else { + let _: () = self.async_conn().incr(full_key, _increment).await?; + } + } + #[cfg(feature = "len")] + { + return Err(anyhow!("unsupported!")); + //@TODO ... + #[allow(unreachable_code)] + { + let full_key = self.make_full_key(_key.as_ref()); + let db_zkey = self.make_len_sortedset_key(); + if get_slot(&db_zkey) == get_slot(full_key.as_slice()) { + let mut async_conn = self.async_conn(); + if let Some(expire_interval) = _expire_interval { + pipe() + .atomic() + .incr(full_key.as_slice(), _increment) + .pexpire(full_key.as_slice(), expire_interval) + .zadd(db_zkey, _key.as_ref(), timestamp_millis() + expire_interval) + .query_async::<()>(&mut async_conn) + .await?; + } else { + pipe() + .atomic() + .incr(full_key.as_slice(), _increment) + .zadd(db_zkey, _key.as_ref(), i64::MAX) + .query_async::<()>(&mut async_conn) + .await?; + } + } else { + let mut async_conn = self.async_conn(); + if let Some(expire_interval) = _expire_interval { + let _: () = pipe() + .atomic() + .incr(full_key.as_slice(), _increment) + .pexpire(full_key.as_slice(), expire_interval) + .query_async::<()>(&mut async_conn) + .await?; + let _: () = async_conn + .zadd(db_zkey, _key.as_ref(), timestamp_millis() + expire_interval) + .await?; + } else { + let _: () = async_conn.incr(full_key.as_slice(), _increment).await?; + let _: () = async_conn.zadd(db_zkey, _key.as_ref(), i64::MAX).await?; + } + } + } + } + Ok(()) + } + + /// Internal method to decrement a counter + #[inline] + async fn _counter_decr( + &self, + _key: K, + _decrement: isize, + _expire_interval: Option, + ) -> Result<()> + where + K: AsRef<[u8]> + Sync + Send, + { + #[cfg(not(feature = "len"))] + { + let full_key = self.make_full_key(_key.as_ref()); + if let Some(expire_interval) = _expire_interval { + let mut async_conn = self.async_conn(); + pipe() + .atomic() + .decr(full_key.as_slice(), _decrement) + .pexpire(full_key.as_slice(), expire_interval) + .query_async::<()>(&mut async_conn) + .await?; + } else { + let _: () = self.async_conn().decr(full_key, _decrement).await?; + } + } + #[cfg(feature = "len")] + { + return Err(anyhow!("unsupported!")); + //@TODO ... + #[allow(unreachable_code)] + { + let full_key = self.make_full_key(_key.as_ref()); + let db_zkey = self.make_len_sortedset_key(); + let mut async_conn = self.async_conn(); + if get_slot(&db_zkey) == get_slot(full_key.as_slice()) { + if let Some(expire_interval) = _expire_interval { + pipe() + .atomic() + .decr(full_key.as_slice(), _decrement) + .pexpire(full_key.as_slice(), expire_interval) + .zadd(db_zkey, _key.as_ref(), timestamp_millis() + expire_interval) + .query_async::<()>(&mut async_conn) + .await?; + } else { + pipe() + .atomic() + .decr(full_key.as_slice(), _decrement) + .zadd(db_zkey, _key.as_ref(), i64::MAX) + .query_async::<()>(&mut async_conn) + .await?; + } + } else { + // + if let Some(expire_interval) = _expire_interval { + let _: () = pipe() + .atomic() + .decr(full_key.as_slice(), _decrement) + .pexpire(full_key.as_slice(), expire_interval) + .query_async::<()>(&mut async_conn) + .await?; + let _: () = async_conn + .zadd(db_zkey, _key.as_ref(), timestamp_millis() + expire_interval) + .await?; + } else { + let _: () = async_conn.decr(full_key.as_slice(), _decrement).await?; + let _: () = async_conn.zadd(db_zkey, _key.as_ref(), i64::MAX).await?; + } + } + } + } + Ok(()) + } + + /// Internal method to set a counter value + #[inline] + async fn _counter_set( + &self, + _key: K, + _val: isize, + _expire_interval: Option, + ) -> Result<()> + where + K: AsRef<[u8]> + Sync + Send, + { + #[cfg(not(feature = "len"))] + { + let full_key = self.make_full_key(_key.as_ref()); + if let Some(expire_interval) = _expire_interval { + let mut async_conn = self.async_conn(); + pipe() + .atomic() + .set(full_key.as_slice(), _val) + .pexpire(full_key.as_slice(), expire_interval) + .query_async::<()>(&mut async_conn) + .await?; + } else { + let _: () = self.async_conn().set(full_key, _val).await?; + } + } + #[cfg(feature = "len")] + { + return Err(anyhow!("unsupported!")); + //@TODO ... + #[allow(unreachable_code)] + { + let full_key = self.make_full_key(_key.as_ref()); + let db_zkey = self.make_len_sortedset_key(); + let mut async_conn = self.async_conn(); + if get_slot(&db_zkey) == get_slot(full_key.as_slice()) { + if let Some(expire_interval) = _expire_interval { + pipe() + .atomic() + .set(full_key.as_slice(), _val) + .pexpire(full_key.as_slice(), expire_interval) + .zadd(db_zkey, _key.as_ref(), timestamp_millis() + expire_interval) + .query_async::<()>(&mut async_conn) + .await?; + } else { + pipe() + .atomic() + .set(full_key.as_slice(), _val) + .zadd(db_zkey, _key.as_ref(), i64::MAX) + .query_async::<()>(&mut async_conn) + .await?; + } + } else { + // + if let Some(expire_interval) = _expire_interval { + pipe() + .atomic() + .set(full_key.as_slice(), _val) + .pexpire(full_key.as_slice(), expire_interval) + .query_async::<()>(&mut async_conn) + .await?; + let _: () = async_conn + .zadd(db_zkey, _key.as_ref(), timestamp_millis() + expire_interval) + .await?; + } else { + let _: () = async_conn.set(full_key.as_slice(), _val).await?; + let _: () = async_conn.zadd(db_zkey, _key.as_ref(), i64::MAX).await?; + } + } + } + } + + Ok(()) + } + + /// Internal method to remove a key + #[inline] + async fn _remove(&self, _key: K) -> Result<()> + where + K: AsRef<[u8]> + Sync + Send, + { + #[cfg(not(feature = "len"))] + { + let full_key = self.make_full_key(_key.as_ref()); + let _: () = self.async_conn().del(full_key).await?; + Ok(()) + } + #[cfg(feature = "len")] + { + return Err(anyhow!("unsupported!")); + //@TODO ... + #[allow(unreachable_code)] + { + let full_key = self.make_full_key(_key.as_ref()); + let db_zkey = self.make_len_sortedset_key(); + + let mut async_conn = self.async_conn(); + if get_slot(&db_zkey) == get_slot(_key.as_ref()) { + pipe() + .atomic() + .del(full_key.as_slice()) + .zrem(db_zkey, _key.as_ref()) + .query_async::<()>(&mut async_conn) + .await?; + } else { + let _: () = async_conn.zrem(db_zkey, _key.as_ref()).await?; + let _: () = async_conn.del(full_key).await?; + } + Ok(()) + } + } + } +} + +#[async_trait] +impl StorageDB for RedisStorageDB { + type MapType = RedisStorageMap; + type ListType = RedisStorageList; + + /// Creates a new map with optional expiration + #[inline] + async fn map + Sync + Send>( + &self, + name: V, + expire: Option, + ) -> Result { + let full_name = self.make_map_full_name(name.as_ref()); + Ok( + RedisStorageMap::new_expire(name.as_ref().to_vec(), full_name, expire, self.clone()) + .await?, + ) + } + + /// Removes a map + #[inline] + async fn map_remove(&self, name: K) -> Result<()> + where + K: AsRef<[u8]> + Sync + Send, + { + let map_full_name = self.make_map_full_name(name.as_ref()); + let _: () = self.async_conn().del(map_full_name).await?; + Ok(()) + } + + /// Checks if a map exists + #[inline] + async fn map_contains_key + Sync + Send>(&self, key: K) -> Result { + let map_full_name = self.make_map_full_name(key.as_ref()); + Ok(self.async_conn().exists(map_full_name).await?) + } + + /// Creates a new list with optional expiration + #[inline] + async fn list + Sync + Send>( + &self, + name: V, + expire: Option, + ) -> Result { + let full_name = self.make_list_full_name(name.as_ref()); + Ok( + RedisStorageList::new_expire(name.as_ref().to_vec(), full_name, expire, self.clone()) + .await?, + ) + } + + /// Removes a list + #[inline] + async fn list_remove(&self, name: K) -> Result<()> + where + K: AsRef<[u8]> + Sync + Send, + { + let list_full_name = self.make_list_full_name(name.as_ref()); + let _: () = self.async_conn().del(list_full_name).await?; + Ok(()) + } + + /// Checks if a list exists + #[inline] + async fn list_contains_key + Sync + Send>(&self, key: K) -> Result { + let list_full_name = self.make_list_full_name(key.as_ref()); + Ok(self.async_conn().exists(list_full_name).await?) + } + + /// Inserts a key-value pair + #[inline] + async fn insert(&self, key: K, val: &V) -> Result<()> + where + K: AsRef<[u8]> + Sync + Send, + V: serde::ser::Serialize + Sync + Send, + { + self._insert(key, val, None).await + } + + /// Gets a value by key + #[inline] + async fn get(&self, key: K) -> Result> + where + K: AsRef<[u8]> + Sync + Send, + V: DeserializeOwned + Sync + Send, + { + let full_key = self.make_full_key(key); + if let Some(v) = self + .async_conn() + .get::<_, Option>>(full_key) + .await? + { + Ok(Some(bincode::deserialize::(v.as_ref())?)) + } else { + Ok(None) + } + } + + /// Removes a key + #[inline] + async fn remove(&self, key: K) -> Result<()> + where + K: AsRef<[u8]> + Sync + Send, + { + self._remove(key).await + } + + /// Batch insertion of key-value pairs + #[inline] + async fn batch_insert(&self, key_vals: Vec<(Key, V)>) -> Result<()> + where + V: Serialize + Sync + Send, + { + if !key_vals.is_empty() { + let mut slot_keys_vals_expires = key_vals + .into_iter() + .map(|(k, v)| { + let full_key = self.make_full_key(k.as_slice()); + (get_slot(&full_key), (k, full_key, v, None)) + }) + .collect::>(); + + if !slot_keys_vals_expires.is_empty() { + for keys_vals_expires in transform_by_slot(slot_keys_vals_expires) { + self._batch_insert(keys_vals_expires).await?; + } + } else if let Some((_, keys_vals_expires)) = slot_keys_vals_expires.pop() { + self._batch_insert(vec![keys_vals_expires]).await?; + } + } + Ok(()) + } + + /// Batch removal of keys + #[inline] + async fn batch_remove(&self, keys: Vec) -> Result<()> { + if !keys.is_empty() { + self._batch_remove(keys).await?; + } + Ok(()) + } + + /// Increments a counter + #[inline] + async fn counter_incr(&self, key: K, increment: isize) -> Result<()> + where + K: AsRef<[u8]> + Sync + Send, + { + self._counter_incr(key, increment, None).await + } + + /// Decrements a counter + #[inline] + async fn counter_decr(&self, key: K, decrement: isize) -> Result<()> + where + K: AsRef<[u8]> + Sync + Send, + { + self._counter_decr(key, decrement, None).await + } + + /// Gets a counter value + #[inline] + async fn counter_get(&self, key: K) -> Result> + where + K: AsRef<[u8]> + Sync + Send, + { + let full_key = self.make_full_key(key); + Ok(self.async_conn().get::<_, Option>(full_key).await?) + } + + /// Sets a counter value + #[inline] + async fn counter_set(&self, key: K, val: isize) -> Result<()> + where + K: AsRef<[u8]> + Sync + Send, + { + self._counter_set(key, val, None).await + } + + /// Checks if a key exists + #[inline] + async fn contains_key + Sync + Send>(&self, key: K) -> Result { + //HEXISTS key field + let full_key = self.make_full_key(key.as_ref()); + Ok(self.async_conn().exists(full_key).await?) + } + + /// Gets the number of keys in the database + #[inline] + #[cfg(feature = "len")] + async fn len(&self) -> Result { + return Err(anyhow!("unsupported!")); + //@TODO ... + #[allow(unreachable_code)] + { + let db_zkey = self.make_len_sortedset_key(); + let mut async_conn = self.async_conn(); + let (_, count) = pipe() + .zrembyscore(db_zkey.as_slice(), 0, timestamp_millis()) + .zcard(db_zkey.as_slice()) + .query_async::<(i64, usize)>(&mut async_conn) + .await?; + Ok(count) + } + } + + /// Gets the total database size + #[inline] + async fn db_size(&self) -> Result { + let mut dbsize = 0; + for mut async_conn in self.nodes.clone() { + //DBSIZE + let dbsize_val = redis::pipe() + .cmd("DBSIZE") + .query_async::(&mut async_conn) + .await?; + dbsize += dbsize_val + .as_sequence() + .and_then(|vs| { + vs.iter().next().and_then(|v| { + if let redis::Value::Int(v) = v { + Some(*v) + } else { + None + } + }) + }) + .unwrap_or_default(); + } + Ok(dbsize as usize) + } + + /// Sets expiration time for a key + #[inline] + #[cfg(feature = "ttl")] + async fn expire_at(&self, _key: K, _at: TimestampMillis) -> Result + where + K: AsRef<[u8]> + Sync + Send, + { + #[cfg(not(feature = "len"))] + { + let full_name = self.make_full_key(_key.as_ref()); + let res = self + .async_conn() + .pexpire_at::<_, bool>(full_name, _at) + .await?; + Ok(res) + } + #[cfg(feature = "len")] + { + return Err(anyhow!("unsupported!")); + //@TODO ... + #[allow(unreachable_code)] + { + let full_name = self.make_full_key(_key.as_ref()); + let db_zkey = self.make_len_sortedset_key(); + let mut async_conn = self.async_conn(); + if get_slot(&db_zkey) == get_slot(full_name.as_slice()) { + let (_, res) = pipe() + .atomic() + .zadd(db_zkey, _key.as_ref(), _at) + .pexpire_at(full_name.as_slice(), _at) + .query_async::<(i64, bool)>(&mut async_conn) + .await?; + Ok(res) + } else { + let res = async_conn.zadd(db_zkey, _key.as_ref(), _at).await?; + let _: () = async_conn.pexpire_at(full_name.as_slice(), _at).await?; + Ok(res) + } + } + } + } + + /// Sets expiration duration for a key + #[inline] + #[cfg(feature = "ttl")] + async fn expire(&self, _key: K, _dur: TimestampMillis) -> Result + where + K: AsRef<[u8]> + Sync + Send, + { + #[cfg(not(feature = "len"))] + { + let full_name = self.make_full_key(_key.as_ref()); + let res = self + .async_conn() + .pexpire::<_, bool>(full_name, _dur) + .await?; + Ok(res) + } + #[cfg(feature = "len")] + { + return Err(anyhow!("unsupported!")); + //@TODO ... + #[allow(unreachable_code)] + { + let full_name = self.make_full_key(_key.as_ref()); + let db_zkey = self.make_len_sortedset_key(); + let mut async_conn = self.async_conn(); + if get_slot(db_zkey.as_slice()) == get_slot(full_name.as_slice()) { + let (_, res) = pipe() + .atomic() + .zadd(db_zkey, _key.as_ref(), timestamp_millis() + _dur) + .pexpire(full_name.as_slice(), _dur) + .query_async::<(i64, bool)>(&mut async_conn) + .await?; + Ok(res) + } else { + let _: () = async_conn + .zadd(db_zkey, _key.as_ref(), timestamp_millis() + _dur) + .await?; + let res = async_conn + .pexpire::<_, bool>(full_name.as_slice(), _dur) + .await?; + Ok(res) + } + } + } + } + + /// Gets time-to-live for a key + #[inline] + #[cfg(feature = "ttl")] + async fn ttl(&self, key: K) -> Result> + where + K: AsRef<[u8]> + Sync + Send, + { + let mut async_conn = self.async_conn(); + let full_key = self.make_full_key(key.as_ref()); + let res = async_conn.pttl::<_, isize>(full_key).await?; + match res { + -2 => Ok(None), + -1 => Ok(Some(TimestampMillis::MAX)), + _ => Ok(Some(res as TimestampMillis)), + } + } + + /// Creates an iterator for all maps + #[inline] + async fn map_iter<'a>( + &'a mut self, + ) -> Result> + Send + 'a>> { + let db = self.clone(); + let pattern = self.make_map_prefix_match(); + + let mut iters = Vec::new(); + for conn in self.nodes_mut().await?.iter_mut() { + let iter = conn.scan_match::<_, Key>(pattern.as_slice()).await?; + iters.push(iter); + } + + let iter = AsyncMapIter { db, iters }; + Ok(Box::new(iter)) + } + + /// Creates an iterator for all lists + #[inline] + async fn list_iter<'a>( + &'a mut self, + ) -> Result> + Send + 'a>> { + let db = self.clone(); + let pattern = self.make_list_prefix_match(); + + let mut iters = Vec::new(); + for conn in self.nodes_mut().await?.iter_mut() { + let iter = conn.scan_match::<_, Key>(pattern.as_slice()).await?; + iters.push(iter); + } + + let iter = AsyncListIter { db, iters }; + Ok(Box::new(iter)) + } + + /// Creates an iterator for keys matching a pattern + async fn scan<'a, P>( + &'a mut self, + pattern: P, + ) -> Result> + Send + 'a>> + where + P: AsRef<[u8]> + Send + Sync, + { + let pattern = self.make_scan_pattern_match(pattern); + let prefix_len = KEY_PREFIX.len() + self.prefix.len(); + + let mut iters = Vec::new(); + for conn in self.nodes_mut().await?.iter_mut() { + let iter = conn.scan_match::<_, Key>(pattern.as_slice()).await?; + iters.push(iter); + } + Ok(Box::new(AsyncDbKeyIter { prefix_len, iters })) + } + + /// Gets database information + #[inline] + async fn info(&self) -> Result { + Ok(serde_json::json!({ + "storage_engine": "RedisCluster", + "dbsize": self.db_size().await?, + })) + } +} + +/// Redis-backed map storage implementation +#[derive(Clone)] +pub struct RedisStorageMap { + /// Name of the map + name: Key, + /// Full key name with prefix + full_name: Key, + /// Optional expiration time in milliseconds + #[allow(dead_code)] + expire: Option, + /// Flag indicating if the map is empty + empty: Arc, + /// Reference to the parent database + pub(crate) db: RedisStorageDB, +} + +impl RedisStorageMap { + /// Creates a new map without expiration + #[inline] + pub(crate) fn new(name: Key, full_name: Key, db: RedisStorageDB) -> Self { + Self { + name, + full_name, + expire: None, + empty: Arc::new(AtomicBool::new(true)), + db, + } + } + + /// Creates a new map with expiration + #[inline] + pub(crate) async fn new_expire( + name: Key, + full_name: Key, + expire: Option, + mut db: RedisStorageDB, + ) -> Result { + let empty = if expire.is_some() { + let empty = Self::_is_empty(&mut db.async_conn, full_name.as_slice()).await?; + Arc::new(AtomicBool::new(empty)) + } else { + Arc::new(AtomicBool::new(true)) + }; + Ok(Self { + name, + full_name, + expire, + empty, + db, + }) + } + + /// Gets a clone of the async connection + #[inline] + fn async_conn(&self) -> RedisConnection { + self.db.async_conn() + } + + /// Gets a mutable reference to the async connection + #[inline] + fn async_conn_mut(&mut self) -> &mut RedisConnection { + self.db.async_conn_mut() + } + + /// Checks if the map is empty + #[inline] + async fn _is_empty(async_conn: &mut RedisConnection, full_name: &[u8]) -> Result { + //HSCAN key cursor [MATCH pattern] [COUNT count] + let res = async_conn + .hscan::<_, Vec>(full_name) + .await? + .next_item() + .await + .is_none(); + Ok(res) + } + + /// Internal method to insert with expiration handling + #[inline] + async fn _insert_expire(&self, key: &[u8], val: Vec) -> Result<()> { + let mut async_conn = self.async_conn(); + let name = self.full_name.as_slice(); + + #[cfg(feature = "ttl")] + if self.empty.load(Ordering::SeqCst) { + if let Some(expire) = self.expire.as_ref() { + //HSET key field value + //PEXPIRE key ms + let _: () = redis::pipe() + .atomic() + .hset(name, key, val) + .pexpire(name, *expire) + .query_async(&mut async_conn) + .await?; + self.empty.store(false, Ordering::SeqCst); + return Ok(()); + } + } + + //HSET key field value + let _: () = async_conn.hset(name, key.as_ref(), val).await?; + Ok(()) + } + + /// Internal method for batch insertion with expiration + #[inline] + async fn _batch_insert_expire(&self, key_vals: Vec<(Key, Vec)>) -> Result<()> { + let mut async_conn = self.async_conn(); + let name = self.full_name.as_slice(); + + #[cfg(feature = "ttl")] + if self.empty.load(Ordering::SeqCst) { + if let Some(expire) = self.expire.as_ref() { + //HMSET key field value + //PEXPIRE key ms + let _: () = redis::pipe() + .atomic() + .hset_multiple(name, key_vals.as_slice()) + .pexpire(name, *expire) + .query_async(&mut async_conn) + .await?; + + self.empty.store(false, Ordering::SeqCst); + return Ok(()); + } + } + + //HSET key field value + let _: () = async_conn.hset_multiple(name, key_vals.as_slice()).await?; + Ok(()) + } +} + +#[async_trait] +impl Map for RedisStorageMap { + /// Gets the map name + #[inline] + fn name(&self) -> &[u8] { + self.name.as_slice() + } + + /// Inserts a key-value pair into the map + #[inline] + async fn insert(&self, key: K, val: &V) -> Result<()> + where + K: AsRef<[u8]> + Sync + Send, + V: Serialize + Sync + Send + ?Sized, + { + self._insert_expire(key.as_ref(), bincode::serialize(val)?) + .await + } + + /// Gets a value from the map + #[inline] + async fn get(&self, key: K) -> Result> + where + K: AsRef<[u8]> + Sync + Send, + V: DeserializeOwned + Sync + Send, + { + //HSET key field value + let res: Option> = self + .async_conn() + .hget(self.full_name.as_slice(), key.as_ref()) + .await?; + if let Some(res) = res { + Ok(Some(bincode::deserialize::(res.as_ref())?)) + } else { + Ok(None) + } + } + + /// Removes a key from the map + #[inline] + async fn remove(&self, key: K) -> Result<()> + where + K: AsRef<[u8]> + Sync + Send, + { + //HDEL key field [field ...] + let _: () = self + .async_conn() + .hdel(self.full_name.as_slice(), key.as_ref()) + .await?; + Ok(()) + } + + /// Checks if a key exists in the map + #[inline] + async fn contains_key + Sync + Send>(&self, key: K) -> Result { + //HEXISTS key field + let res = self + .async_conn() + .hexists(self.full_name.as_slice(), key.as_ref()) + .await?; + Ok(res) + } + + /// Gets the number of elements in the map + #[cfg(feature = "map_len")] + #[inline] + async fn len(&self) -> Result { + //HLEN key + Ok(self.async_conn().hlen(self.full_name.as_slice()).await?) + } + + /// Checks if the map is empty + #[inline] + async fn is_empty(&self) -> Result { + //HSCAN key cursor [MATCH pattern] [COUNT count] + let res = self + .async_conn() + .hscan::<_, Vec>(self.full_name.as_slice()) + .await? + .next_item() + .await + .is_none(); + Ok(res) + } + + /// Clears all elements from the map + #[inline] + async fn clear(&self) -> Result<()> { + //DEL key [key ...] + let _: () = self.async_conn().del(self.full_name.as_slice()).await?; + self.empty.store(true, Ordering::SeqCst); + Ok(()) + } + + /// Removes and returns a value from the map + #[inline] + async fn remove_and_fetch(&self, key: K) -> Result> + where + K: AsRef<[u8]> + Sync + Send, + V: DeserializeOwned + Sync + Send, + { + //HSET key field value + //HDEL key field [field ...] + let name = self.full_name.as_slice(); + let mut conn = self.async_conn(); + let (res, _): (Option>, isize) = redis::pipe() + .atomic() + .hget(name, key.as_ref()) + .hdel(name, key.as_ref()) + .query_async(&mut conn) + .await?; + + if let Some(res) = res { + Ok(Some(bincode::deserialize::(res.as_ref())?)) + } else { + Ok(None) + } + } + + /// Removes all keys with a given prefix + #[inline] + async fn remove_with_prefix(&self, prefix: K) -> Result<()> + where + K: AsRef<[u8]> + Sync + Send, + { + let name = self.full_name.as_slice(); + let mut conn = self.async_conn(); + let mut conn2 = conn.clone(); + let mut prefix = prefix.as_ref().to_vec(); + prefix.push(b'*'); + let mut removeds = Vec::new(); + while let Some(key) = conn + .hscan_match::<_, _, Vec>(name, prefix.as_slice()) + .await? + .next_item() + .await + { + removeds.push(key?); + if removeds.len() > 20 { + let _: () = conn2.hdel(name, removeds.as_slice()).await?; + removeds.clear(); + } + } + if !removeds.is_empty() { + let _: () = conn.hdel(name, removeds).await?; + } + Ok(()) + } + + /// Batch insertion of key-value pairs + #[inline] + async fn batch_insert(&self, key_vals: Vec<(Key, V)>) -> Result<()> + where + V: Serialize + Sync + Send, + { + if !key_vals.is_empty() { + let key_vals = key_vals + .into_iter() + .map(|(k, v)| { + bincode::serialize(&v) + .map(move |v| (k, v)) + .map_err(|e| anyhow!(e)) + }) + .collect::>>()?; + + self._batch_insert_expire(key_vals).await?; + } + Ok(()) + } + + /// Batch removal of keys + #[inline] + async fn batch_remove(&self, keys: Vec) -> Result<()> { + if !keys.is_empty() { + let _: () = self + .async_conn() + .hdel(self.full_name.as_slice(), keys) + .await?; + } + Ok(()) + } + + /// Creates an iterator over key-value pairs + #[inline] + async fn iter<'a, V>( + &'a mut self, + ) -> Result> + Send + 'a>> + where + V: DeserializeOwned + Sync + Send + 'a + 'static, + { + let name = self.full_name.clone(); + let iter = AsyncIter { + iter: self + .async_conn_mut() + .hscan::<_, (Key, Vec)>(name) + .await?, + _m: std::marker::PhantomData, + }; + Ok(Box::new(iter)) + } + + /// Creates an iterator over keys + #[inline] + async fn key_iter<'a>( + &'a mut self, + ) -> Result> + Send + 'a>> { + let iter = AsyncKeyIter { + iter: self + .db + .async_conn + .hscan::<_, (Key, ())>(self.full_name.as_slice()) + .await?, + }; + Ok(Box::new(iter)) + } + + /// Creates an iterator over key-value pairs with a prefix + #[inline] + async fn prefix_iter<'a, P, V>( + &'a mut self, + prefix: P, + ) -> Result> + Send + 'a>> + where + P: AsRef<[u8]> + Send + Sync, + V: DeserializeOwned + Sync + Send + 'a + 'static, + { + let name = self.full_name.clone(); + let mut prefix = prefix.as_ref().to_vec(); + prefix.push(b'*'); + let iter = AsyncIter { + iter: self + .async_conn_mut() + .hscan_match::<_, _, (Key, Vec)>(name, prefix.as_slice()) + .await?, + _m: std::marker::PhantomData, + }; + Ok(Box::new(iter)) + } + + /// Sets expiration time for the map + #[cfg(feature = "ttl")] + async fn expire_at(&self, at: TimestampMillis) -> Result { + let res = self + .async_conn() + .pexpire_at::<_, bool>(self.full_name.as_slice(), at) + .await?; + Ok(res) + } + + /// Sets expiration duration for the map + #[cfg(feature = "ttl")] + async fn expire(&self, dur: TimestampMillis) -> Result { + let res = self + .async_conn() + .pexpire::<_, bool>(self.full_name.as_slice(), dur) + .await?; + Ok(res) + } + + /// Gets time-to-live for the map + #[cfg(feature = "ttl")] + async fn ttl(&self) -> Result> { + let mut async_conn = self.async_conn(); + let res = async_conn + .pttl::<_, isize>(self.full_name.as_slice()) + .await?; + match res { + -2 => Ok(None), + -1 => Ok(Some(TimestampMillis::MAX)), + _ => Ok(Some(res as TimestampMillis)), + } + } +} + +/// Redis-backed list storage implementation +#[derive(Clone)] +pub struct RedisStorageList { + /// Name of the list + name: Key, + /// Full key name with prefix + full_name: Key, + /// Optional expiration time in milliseconds + #[allow(dead_code)] + expire: Option, + /// Flag indicating if the list is empty + empty: Arc, + /// Reference to the parent database + pub(crate) db: RedisStorageDB, +} + +impl RedisStorageList { + /// Creates a new list without expiration + #[inline] + pub(crate) fn new(name: Key, full_name: Key, db: RedisStorageDB) -> Self { + Self { + name, + full_name, + expire: None, + empty: Arc::new(AtomicBool::new(true)), + db, + } + } + + /// Creates a new list with expiration + #[inline] + pub(crate) async fn new_expire( + name: Key, + full_name: Key, + expire: Option, + mut db: RedisStorageDB, + ) -> Result { + let empty = if expire.is_some() { + let empty = Self::_is_empty(&mut db.async_conn, full_name.as_slice()).await?; + Arc::new(AtomicBool::new(empty)) + } else { + Arc::new(AtomicBool::new(true)) + }; + Ok(Self { + name, + full_name, + expire, + empty, + db, + }) + } + + /// Gets a clone of the async connection + #[inline] + pub(crate) fn async_conn(&self) -> RedisConnection { + self.db.async_conn() + } + + /// Checks if the list is empty + #[inline] + async fn _is_empty(async_conn: &mut RedisConnection, full_name: &[u8]) -> Result { + Ok(async_conn.llen::<_, usize>(full_name).await? == 0) + } + + /// Internal method to push with expiration handling + #[inline] + async fn _push_expire(&self, val: Vec) -> Result<()> { + let mut async_conn = self.async_conn(); + let name = self.full_name.as_slice(); + + #[cfg(feature = "ttl")] + if self.empty.load(Ordering::SeqCst) { + if let Some(expire) = self.expire.as_ref() { + //RPUSH key value [value ...] + //PEXPIRE key ms + let _: () = redis::pipe() + .atomic() + .rpush(name, val) + .pexpire(name, *expire) + .query_async(&mut async_conn) + .await?; + self.empty.store(false, Ordering::SeqCst); + return Ok(()); + } + } + + //RPUSH key value [value ...] + let _: () = async_conn.rpush(name, val).await?; + Ok(()) + } + + /// Internal method for batch push with expiration + #[inline] + async fn _pushs_expire(&self, vals: Vec>) -> Result<()> { + let mut async_conn = self.async_conn(); + + #[cfg(feature = "ttl")] + if self.empty.load(Ordering::SeqCst) { + if let Some(expire) = self.expire.as_ref() { + let name = self.full_name.as_slice(); + //RPUSH key value [value ...] + //PEXPIRE key ms + let _: () = redis::pipe() + .atomic() + .rpush(name, vals) + .pexpire(name, *expire) + .query_async(&mut async_conn) + .await?; + self.empty.store(false, Ordering::SeqCst); + return Ok(()); + } + } + + //RPUSH key value [value ...] + let _: () = async_conn.rpush(self.full_name.as_slice(), vals).await?; + Ok(()) + } + + /// Internal method for push with limit and expiration + #[inline] + async fn _push_limit_expire( + &self, + val: Vec, + limit: usize, + pop_front_if_limited: bool, + ) -> Result>> { + let mut conn = self.async_conn(); + + #[cfg(feature = "ttl")] + if self.empty.load(Ordering::SeqCst) { + if let Some(expire) = self.expire.as_ref() { + let name = self.full_name.as_slice(); + let count = conn.llen::<_, usize>(name).await?; + let res = if count < limit { + let _: () = redis::pipe() + .atomic() + .rpush(name, val) + .pexpire(name, *expire) + .query_async(&mut conn) + .await?; + Ok(None) + } else if pop_front_if_limited { + let (poped, _): (Option>, Option<()>) = redis::pipe() + .atomic() + .lpop(name, None) + .rpush(name, val) + .pexpire(name, *expire) + .query_async(&mut conn) + .await?; + + Ok(poped) + } else { + Err(anyhow::Error::msg("Is full")) + }; + self.empty.store(false, Ordering::SeqCst); + return res; + } + } + + self._push_limit(val, limit, pop_front_if_limited, &mut conn) + .await + } + + /// Internal method for push with limit + #[inline] + async fn _push_limit( + &self, + val: Vec, + limit: usize, + pop_front_if_limited: bool, + async_conn: &mut RedisConnection, + ) -> Result>> { + let name = self.full_name.as_slice(); + + let count = async_conn.llen::<_, usize>(name).await?; + if count < limit { + let _: () = async_conn.rpush(name, val).await?; + Ok(None) + } else if pop_front_if_limited { + let (poped, _): (Option>, Option<()>) = redis::pipe() + .atomic() + .lpop(name, None) + .rpush(name, val) + .query_async(async_conn) + .await?; + Ok(poped) + } else { + Err(anyhow::Error::msg("Is full")) + } + } +} + +#[async_trait] +impl List for RedisStorageList { + /// Gets the list name + #[inline] + fn name(&self) -> &[u8] { + self.name.as_slice() + } + + /// Pushes a value to the end of the list + #[inline] + async fn push(&self, val: &V) -> Result<()> + where + V: Serialize + Sync + Send, + { + self._push_expire(bincode::serialize(val)?).await + } + + /// Pushes multiple values to the end of the list + #[inline] + async fn pushs(&self, vals: Vec) -> Result<()> + where + V: Serialize + Sync + Send, + { + //RPUSH key value [value ...] + let vals = vals + .into_iter() + .map(|v| bincode::serialize(&v).map_err(|e| anyhow!(e))) + .collect::>>()?; + self._pushs_expire(vals).await + } + + /// Pushes a value with size limit handling + #[inline] + async fn push_limit( + &self, + val: &V, + limit: usize, + pop_front_if_limited: bool, + ) -> Result> + where + V: Serialize + Sync + Send, + V: DeserializeOwned, + { + let data = bincode::serialize(val)?; + + if let Some(res) = self + ._push_limit_expire(data, limit, pop_front_if_limited) + .await? + { + Ok(Some( + bincode::deserialize::(res.as_ref()).map_err(|e| anyhow!(e))?, + )) + } else { + Ok(None) + } + } + + /// Pops a value from the front of the list + #[inline] + async fn pop(&self) -> Result> + where + V: DeserializeOwned + Sync + Send, + { + //LPOP key + let removed = self + .async_conn() + .lpop::<_, Option>>(self.full_name.as_slice(), None) + .await?; + + let removed = if let Some(v) = removed { + Some(bincode::deserialize::(v.as_ref()).map_err(|e| anyhow!(e))?) + } else { + None + }; + + Ok(removed) + } + + /// Gets all values in the list + #[inline] + async fn all(&self) -> Result> + where + V: DeserializeOwned + Sync + Send, + { + //LRANGE key 0 -1 + let all = self + .async_conn() + .lrange::<_, Vec>>(self.full_name.as_slice(), 0, -1) + .await?; + all.iter() + .map(|v| bincode::deserialize::(v.as_ref()).map_err(|e| anyhow!(e))) + .collect::>>() + } + + /// Gets a value by index + #[inline] + async fn get_index(&self, idx: usize) -> Result> + where + V: DeserializeOwned + Sync + Send, + { + //LINDEX key index + let val = self + .async_conn() + .lindex::<_, Option>>(self.full_name.as_slice(), idx as isize) + .await?; + + Ok(if let Some(v) = val { + Some(bincode::deserialize::(v.as_ref()).map_err(|e| anyhow!(e))?) + } else { + None + }) + } + + /// Gets the length of the list + #[inline] + async fn len(&self) -> Result { + //LLEN key + Ok(self.async_conn().llen(self.full_name.as_slice()).await?) + } + + /// Checks if the list is empty + #[inline] + async fn is_empty(&self) -> Result { + Ok(self.len().await? == 0) + } + + /// Clears the list + #[inline] + async fn clear(&self) -> Result<()> { + let _: () = self.async_conn().del(self.full_name.as_slice()).await?; + self.empty.store(true, Ordering::SeqCst); + Ok(()) + } + + /// Creates an iterator over list values + #[inline] + async fn iter<'a, V>( + &'a mut self, + ) -> Result> + Send + 'a>> + where + V: DeserializeOwned + Sync + Send + 'a + 'static, + { + Ok(Box::new(AsyncListValIter::new( + self.full_name.as_slice(), + self.db.async_conn(), + ))) + } + + /// Sets expiration time for the list + #[cfg(feature = "ttl")] + async fn expire_at(&self, at: TimestampMillis) -> Result { + let res = self + .async_conn() + .pexpire_at::<_, bool>(self.full_name.as_slice(), at) + .await?; + Ok(res) + } + + /// Sets expiration duration for the list + #[cfg(feature = "ttl")] + async fn expire(&self, dur: TimestampMillis) -> Result { + let res = self + .async_conn() + .pexpire::<_, bool>(self.full_name.as_slice(), dur) + .await?; + Ok(res) + } + + /// Gets time-to-live for the list + #[cfg(feature = "ttl")] + async fn ttl(&self) -> Result> { + let mut async_conn = self.async_conn(); + let res = async_conn + .pttl::<_, isize>(self.full_name.as_slice()) + .await?; + match res { + -2 => Ok(None), + -1 => Ok(Some(TimestampMillis::MAX)), + _ => Ok(Some(res as TimestampMillis)), + } + } +} + +/// Iterator for list values +pub struct AsyncListValIter<'a, V> { + name: &'a [u8], + conn: RedisConnection, + start: isize, + limit: isize, + catch_vals: Vec>, + _m: std::marker::PhantomData, +} + +impl<'a, V> AsyncListValIter<'a, V> { + fn new(name: &'a [u8], conn: RedisConnection) -> Self { + let start = 0; + let limit = 20; + Self { + name, + conn, + start, + limit, + catch_vals: Vec::with_capacity((limit + 1) as usize), + _m: std::marker::PhantomData, + } + } +} + +#[async_trait] +impl AsyncIterator for AsyncListValIter<'_, V> +where + V: DeserializeOwned + Sync + Send, +{ + type Item = Result; + + async fn next(&mut self) -> Option { + if let Some(val) = self.catch_vals.pop() { + return Some(bincode::deserialize::(val.as_ref()).map_err(|e| anyhow!(e))); + } + + let vals = self + .conn + .lrange::<_, Vec>>(self.name, self.start, self.start + self.limit) + .await; + + match vals { + Err(e) => return Some(Err(anyhow!(e))), + Ok(vals) => { + if vals.is_empty() { + return None; + } + self.start += vals.len() as isize; + self.catch_vals = vals; + self.catch_vals.reverse(); + } + } + + self.catch_vals + .pop() + .map(|val| bincode::deserialize::(val.as_ref()).map_err(|e| anyhow!(e))) + } +} + +/// Iterator for map entries +pub struct AsyncIter<'a, V> { + iter: redis::AsyncIter<'a, (Key, Vec)>, + _m: std::marker::PhantomData, +} + +#[async_trait] +impl AsyncIterator for AsyncIter<'_, V> +where + V: DeserializeOwned + Sync + Send, +{ + type Item = IterItem; + + async fn next(&mut self) -> Option { + let item = match self.iter.next_item().await { + None => None, + Some(Err(e)) => return Some(Err(anyhow::Error::new(e))), + Some(Ok(item)) => Some(item), + }; + item.map(|(key, v)| match bincode::deserialize::(v.as_ref()) { + Ok(v) => Ok((key, v)), + Err(e) => Err(anyhow::Error::new(e)), + }) + } +} + +/// Iterator for database keys +pub struct AsyncDbKeyIter<'a> { + prefix_len: usize, + iters: Vec>, +} + +#[async_trait] +impl AsyncIterator for AsyncDbKeyIter<'_> { + type Item = Result; + async fn next(&mut self) -> Option { + while let Some(iter) = self.iters.last_mut() { + let item = match iter.next_item().await { + None => None, + Some(Err(e)) => Some(Err(anyhow::Error::new(e))), + Some(Ok(key)) => Some(Ok(key[self.prefix_len..].to_vec())), + }; + + if item.is_some() { + return item; + } + self.iters.pop(); + if self.iters.is_empty() { + return None; + } + } + None + } +} + +/// Iterator for map keys +pub struct AsyncKeyIter<'a> { + iter: redis::AsyncIter<'a, (Key, ())>, +} + +#[async_trait] +impl AsyncIterator for AsyncKeyIter<'_> { + type Item = Result; + + async fn next(&mut self) -> Option { + match self.iter.next_item().await { + None => None, + Some(Err(e)) => Some(Err(anyhow::Error::new(e))), + Some(Ok((key, _))) => Some(Ok(key)), + } + } +} + +/// Iterator for maps +pub struct AsyncMapIter<'a> { + db: RedisStorageDB, + iters: Vec>, +} + +#[async_trait] +impl AsyncIterator for AsyncMapIter<'_> { + type Item = Result; + + async fn next(&mut self) -> Option { + while let Some(iter) = self.iters.last_mut() { + let full_name = match iter.next_item().await { + None => None, + Some(Err(e)) => return Some(Err(anyhow::Error::new(e))), + Some(Ok(key)) => Some(key), + }; + + if let Some(full_name) = full_name { + let name = self.db.map_full_name_to_key(full_name.as_slice()).to_vec(); + let m = RedisStorageMap::new(name, full_name, self.db.clone()); + return Some(Ok(StorageMap::RedisCluster(m))); + } + self.iters.pop(); + if self.iters.is_empty() { + return None; + } + } + None + } +} + +/// Iterator for lists +pub struct AsyncListIter<'a> { + db: RedisStorageDB, + iters: Vec>, +} + +#[async_trait] +impl AsyncIterator for AsyncListIter<'_> { + type Item = Result; + + async fn next(&mut self) -> Option { + while let Some(iter) = self.iters.last_mut() { + let full_name = match iter.next_item().await { + None => None, + Some(Err(e)) => return Some(Err(anyhow::Error::new(e))), + Some(Ok(key)) => Some(key), + }; + + if let Some(full_name) = full_name { + let name = self.db.list_full_name_to_key(full_name.as_slice()).to_vec(); + let l = RedisStorageList::new(name, full_name, self.db.clone()); + return Some(Ok(StorageList::RedisCluster(l))); + } + self.iters.pop(); + if self.iters.is_empty() { + return None; + } + } + None + } +} + +/// Groups items by Redis slot +#[inline] +fn transform_by_slot(input: Vec<(u16, T)>) -> Vec> { + let mut grouped_data: BTreeMap> = BTreeMap::new(); + + for (group_key, item) in input { + grouped_data.entry(group_key).or_default().push(item); + } + + grouped_data.into_values().collect() +} diff --git a/src/storage_sled.rs b/src/storage_sled.rs index 8a6f103..73f0b42 100644 --- a/src/storage_sled.rs +++ b/src/storage_sled.rs @@ -1,3 +1,17 @@ +//! Sled-based persistent storage implementation +//! +//! This module provides a persistent storage solution backed by Sled (an embedded database). +//! It implements key-value storage, maps (dictionaries), and lists (queues) with support for: +//! - Atomic operations and transactions +//! - Asynchronous API +//! - TTL/expiration (optional feature) +//! - Counters +//! - Batch operations +//! - Iterators +//! +//! The implementation uses multiple sled trees for different data types and provides +//! a command-based interface with background processing for concurrent operations. + use core::fmt; use std::borrow::Cow; use std::fmt::Debug; @@ -7,10 +21,11 @@ use std::ops::Deref; use std::sync::atomic::{AtomicBool, AtomicIsize, Ordering}; use std::sync::Arc; -use anyhow::anyhow; +use anyhow::{anyhow, Error}; use async_trait::async_trait; use convert::Bytesize; use serde::de::DeserializeOwned; +use serde::Deserialize; use serde::Serialize; use serde_json::Value; @@ -31,32 +46,49 @@ use tokio::task::spawn_blocking; use crate::storage::{AsyncIterator, IterItem, Key, List, Map, StorageDB}; #[allow(unused_imports)] use crate::{timestamp_millis, TimestampMillis}; -use crate::{Error, Result, StorageList, StorageMap}; +use crate::{Result, StorageList, StorageMap}; +/// Byte separator used in composite keys const SEPARATOR: &[u8] = b"@"; +/// Tree name for key-value storage const KV_TREE: &[u8] = b"__kv_tree@"; +/// Tree name for map metadata const MAP_TREE: &[u8] = b"__map_tree@"; +/// Tree name for list metadata const LIST_TREE: &[u8] = b"__list_tree@"; +/// Tree for tracking expiration times (expire_at => key) const EXPIRE_KEYS_TREE: &[u8] = b"__expire_key_tree@"; +/// Tree for tracking key expiration (key => expire_at) const KEY_EXPIRE_TREE: &[u8] = b"__key_expire_tree@"; +/// Prefix for map keys const MAP_NAME_PREFIX: &[u8] = b"__map@"; +/// Separator between map name and item key const MAP_KEY_SEPARATOR: &[u8] = b"@__item@"; #[allow(dead_code)] +/// Suffix for map count keys const MAP_KEY_COUNT_SUFFIX: &[u8] = b"@__count@"; +/// Prefix for list keys const LIST_NAME_PREFIX: &[u8] = b"__list@"; +/// Suffix for list count keys const LIST_KEY_COUNT_SUFFIX: &[u8] = b"@__count@"; +/// Suffix for list content keys const LIST_KEY_CONTENT_SUFFIX: &[u8] = b"@__content@"; +/// Enum representing different key types in storage #[allow(dead_code)] #[derive(Debug, Clone, Copy, Deserialize, Serialize)] enum KeyType { + /// Key-value pair KV, + /// Map structure Map, + /// List structure List, } impl KeyType { + /// Encodes key type to a single byte #[inline] #[allow(dead_code)] fn encode(&self) -> &[u8] { @@ -67,6 +99,7 @@ impl KeyType { } } + /// Decodes key type from byte representation #[inline] #[allow(dead_code)] fn decode(v: &[u8]) -> Result { @@ -83,7 +116,9 @@ impl KeyType { } } +/// Enum representing all possible storage operations enum Command { + // Database operations DBInsert(SledStorageDB, Key, Vec, oneshot::Sender>), DBGet(SledStorageDB, IVec, oneshot::Sender>>), DBRemove(SledStorageDB, IVec, oneshot::Sender>), @@ -130,6 +165,7 @@ enum Command { DBLen(SledStorageDB, oneshot::Sender), DBSize(SledStorageDB, oneshot::Sender), + // Map operations MapInsert(SledStorageMap, IVec, IVec, oneshot::Sender>), MapGet(SledStorageMap, IVec, oneshot::Sender>>), MapRemove(SledStorageMap, IVec, oneshot::Sender>), @@ -160,6 +196,7 @@ enum Command { MapIsExpired(SledStorageMap, oneshot::Sender>), MapPrefixIter(SledStorageMap, Option, oneshot::Sender), + // List operations ListPush(SledStorageList, IVec, oneshot::Sender>), ListPushs(SledStorageList, Vec, oneshot::Sender>), ListPushLimit( @@ -193,6 +230,7 @@ enum Command { ListIsExpired(SledStorageList, oneshot::Sender>), ListPrefixIter(SledStorageList, oneshot::Sender), + // Iterator operation #[allow(clippy::type_complexity)] IterNext( sled::Iter, @@ -200,8 +238,10 @@ enum Command { ), } +/// Type alias for cleanup function signature pub type CleanupFun = fn(&SledStorageDB); +/// Default cleanup function that runs in background thread fn def_cleanup(_db: &SledStorageDB) { #[cfg(feature = "ttl")] { @@ -246,10 +286,14 @@ fn def_cleanup(_db: &SledStorageDB) { } } +/// Configuration for Sled storage backend #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SledConfig { + /// Path to database directory pub path: String, + /// Cache capacity in bytes pub cache_capacity: Bytesize, + /// Cleanup function for expired keys #[serde(skip, default = "SledConfig::cleanup_f_default")] pub cleanup_f: CleanupFun, } @@ -265,6 +309,7 @@ impl Default for SledConfig { } impl SledConfig { + /// Converts to Sled's native configuration #[inline] pub fn to_sled_config(&self) -> Result { if self.path.trim().is_empty() { @@ -277,12 +322,14 @@ impl SledConfig { Ok(sled_cfg) } + /// Returns default cleanup function #[inline] fn cleanup_f_default() -> CleanupFun { def_cleanup } } +/// Increments a counter value stored in bytes fn _increment(old: Option<&[u8]>) -> Option> { let number = match old { Some(bytes) => { @@ -299,6 +346,7 @@ fn _increment(old: Option<&[u8]>) -> Option> { Some(number.to_be_bytes().to_vec()) } +/// Decrements a counter value stored in bytes fn _decrement(old: Option<&[u8]>) -> Option> { let number = match old { Some(bytes) => { @@ -315,6 +363,7 @@ fn _decrement(old: Option<&[u8]>) -> Option> { Some(number.to_be_bytes().to_vec()) } +/// Pattern for matching keys with wildcards #[derive(Clone)] pub struct Pattern(Arc>); @@ -338,14 +387,19 @@ impl From<&[u8]> for Pattern { } } +/// Represents a single character in a pattern #[derive(Clone)] pub enum PatternChar { + /// Literal byte Literal(u8), + /// Wildcard matching zero or more characters Wildcard, + /// Matches any single character AnyChar, } impl Pattern { + /// Parses a byte pattern into PatternChar sequence pub fn parse(pattern: &[u8]) -> Self { let mut parsed_pattern = Vec::new(); let mut chars = pattern.bytes().peekable(); @@ -375,6 +429,7 @@ impl Pattern { } } +/// Checks if text matches the given pattern fn is_match>(pattern: P, text: &[u8]) -> bool { let pattern = pattern.into(); let text_chars = text; @@ -407,7 +462,9 @@ fn is_match>(pattern: P, text: &[u8]) -> bool { dp[pattern_len][text_len] } +/// Trait for byte replacement pub trait BytesReplace { + /// Replaces all occurrences of `from` with `to` in the byte slice fn replace(self, from: &[u8], to: &[u8]) -> Vec; } @@ -429,21 +486,31 @@ impl BytesReplace for &[u8] { } } +/// Main database handle for Sled storage #[derive(Clone)] pub struct SledStorageDB { + /// Underlying sled database pub(crate) db: Arc, + /// Tree for key-value storage pub(crate) kv_tree: sled::Tree, + /// Tree for map metadata pub(crate) map_tree: sled::Tree, + /// Tree for list metadata pub(crate) list_tree: sled::Tree, + /// Tree for tracking expiration times #[allow(dead_code)] - pub(crate) expire_key_tree: sled::Tree, //(key, val) => (expire_at, key) + pub(crate) expire_key_tree: sled::Tree, + /// Tree for tracking key expiration #[allow(dead_code)] - pub(crate) key_expire_tree: sled::Tree, //(key, val) => (key, expire_at) + pub(crate) key_expire_tree: sled::Tree, + /// Channel sender for commands cmd_tx: mpsc::Sender, - active_count: Arc, //Active Command Count + /// Count of active commands + active_count: Arc, } impl SledStorageDB { + /// Creates a new SledStorageDB instance #[inline] pub(crate) async fn new(cfg: SledConfig) -> Result { let sled_cfg = cfg.to_sled_config()?; @@ -661,6 +728,7 @@ impl SledStorageDB { Ok(db) } + /// Cleans up expired keys (TTL feature) #[cfg(feature = "ttl")] #[inline] pub fn cleanup(&self, limit: usize) -> usize { @@ -755,6 +823,7 @@ impl SledStorageDB { count } + /// Cleans up expired key-value pairs (TTL feature) #[cfg(feature = "ttl")] #[inline] pub fn cleanup_kvs(&self, limit: usize) -> usize { @@ -823,6 +892,7 @@ impl SledStorageDB { count } + /// Returns the count of active commands #[inline] pub fn active_count(&self) -> isize { self.active_count.load(Ordering::Relaxed) @@ -838,6 +908,7 @@ impl SledStorageDB { // self.list_tree.len() // } + /// Creates a map prefix name #[inline] fn make_map_prefix_name(name: K) -> Key where @@ -846,6 +917,7 @@ impl SledStorageDB { [MAP_NAME_PREFIX, name.as_ref(), SEPARATOR].concat() } + /// Creates a map item prefix name #[inline] fn make_map_item_prefix_name(name: K) -> Key where @@ -854,6 +926,7 @@ impl SledStorageDB { [MAP_NAME_PREFIX, name.as_ref(), MAP_KEY_SEPARATOR].concat() } + /// Creates a map count key name #[inline] fn make_map_count_key_name(name: K) -> Key where @@ -862,16 +935,19 @@ impl SledStorageDB { [MAP_NAME_PREFIX, name.as_ref(), MAP_KEY_COUNT_SUFFIX].concat() } + /// Extracts map name from count key #[inline] fn map_count_key_to_name(key: &[u8]) -> &[u8] { key[MAP_NAME_PREFIX.len()..key.as_ref().len() - MAP_KEY_COUNT_SUFFIX.len()].as_ref() } + /// Checks if a key is a map count key #[inline] fn is_map_count_key(key: &[u8]) -> bool { key.starts_with(MAP_NAME_PREFIX) && key.ends_with(MAP_KEY_COUNT_SUFFIX) } + /// Extracts map name from item key #[allow(dead_code)] #[inline] fn map_item_key_to_name(key: &[u8]) -> Option<&[u8]> { @@ -887,6 +963,7 @@ impl SledStorageDB { None } + /// Creates a list prefix #[inline] fn make_list_prefix(name: K) -> Key where @@ -895,21 +972,25 @@ impl SledStorageDB { [LIST_NAME_PREFIX, name.as_ref()].concat() } + /// Creates a list count key #[inline] fn make_list_count_key(name: &[u8]) -> Vec { [LIST_NAME_PREFIX, name, LIST_KEY_COUNT_SUFFIX].concat() } + /// Extracts list name from count key #[inline] fn list_count_key_to_name(key: &[u8]) -> &[u8] { key[LIST_NAME_PREFIX.len()..key.as_ref().len() - LIST_KEY_COUNT_SUFFIX.len()].as_ref() } + /// Checks if a key is a list count key #[inline] fn is_list_count_key(key: &[u8]) -> bool { key.starts_with(LIST_NAME_PREFIX) && key.ends_with(LIST_KEY_COUNT_SUFFIX) } + /// Checks if a key exists for a specific key type #[inline] fn _contains_key + Sync + Send>( &self, @@ -923,23 +1004,27 @@ impl SledStorageDB { } } + /// Checks if a key exists in key-value store #[inline] fn _kv_contains_key + Sync + Send>(kv: &Tree, key: K) -> Result { Ok(kv.contains_key(key.as_ref())?) } + /// Checks if a map exists #[inline] fn _map_contains_key + Sync + Send>(tree: &Tree, key: K) -> Result { let count_key = SledStorageDB::make_map_count_key_name(key.as_ref()); Ok(tree.contains_key(count_key)?) } + /// Checks if a list exists #[inline] fn _list_contains_key + Sync + Send>(tree: &Tree, name: K) -> Result { let count_key = SledStorageDB::make_list_count_key(name.as_ref()); Ok(tree.contains_key(count_key)?) } + /// Removes a map #[inline] fn _map_remove(&self, key: K) -> Result<()> where @@ -962,6 +1047,7 @@ impl SledStorageDB { Ok(()) } + /// Removes a list #[inline] fn _list_remove(&self, key: K) -> Result<()> where @@ -988,6 +1074,7 @@ impl SledStorageDB { Ok(()) } + /// Removes a key-value pair #[inline] fn _kv_remove(&self, key: K) -> Result<()> where @@ -1008,6 +1095,7 @@ impl SledStorageDB { Ok(()) } + /// Removes expiration key (TTL feature) #[cfg(feature = "ttl")] #[inline] fn _remove_expire_key(&self, key: &[u8]) -> Result<()> { @@ -1019,6 +1107,7 @@ impl SledStorageDB { Ok(()) } + /// Transactionally removes expiration key (TTL feature) #[cfg(feature = "ttl")] #[inline] fn _tx_remove_expire_key( @@ -1034,6 +1123,7 @@ impl SledStorageDB { Ok(()) } + /// Checks if a key is expired #[inline] fn _is_expired(&self, _key: K, _contains_key_f: F) -> Result where @@ -1052,6 +1142,7 @@ impl SledStorageDB { Ok(false) } + /// Gets time-to-live for a key #[inline] fn _ttl( &self, @@ -1067,6 +1158,7 @@ impl SledStorageDB { .map(|(expire_at, at_bytes)| (expire_at - timestamp_millis(), at_bytes))) } + /// Gets expiration time for a key #[inline] fn _ttl_at( &self, @@ -1100,6 +1192,7 @@ impl SledStorageDB { Ok(ttl_res) } + /// Inserts a key-value pair #[inline] fn _insert(&self, key: &[u8], val: &[u8]) -> Result<()> { #[cfg(not(feature = "ttl"))] @@ -1117,6 +1210,7 @@ impl SledStorageDB { Ok(()) } + /// Gets a value by key #[inline] fn _get(&self, key: &[u8]) -> Result> { let res = if self._is_expired(key.as_ref(), |k| Self::_kv_contains_key(&self.kv_tree, k))? { @@ -1127,28 +1221,42 @@ impl SledStorageDB { Ok(res) } + /// Checks if a map key exists #[inline] fn _self_map_contains_key(&self, key: &[u8]) -> Result { - // let this = self; - if self._is_expired(key, |k| Self::_map_contains_key(&self.map_tree, k))? { - Ok(false) - } else { - //Self::_map_contains_key(&self.map_tree, key) - Ok(true) + #[cfg(feature = "ttl")] + { + if self._is_expired(key, |k| Self::_map_contains_key(&self.map_tree, k))? { + Ok(false) + } else { + //Self::_map_contains_key(&self.map_tree, key) + Ok(true) + } } + + #[cfg(not(feature = "ttl"))] + Self::_map_contains_key(&self.map_tree, key) } + /// Checks if a list key exists #[inline] fn _self_list_contains_key(&self, key: &[u8]) -> Result { - let this = self; - if this._is_expired(key, |k| Self::_list_contains_key(&self.list_tree, k))? { - Ok(false) - } else { - // Self::_list_contains_key(&this.list_tree, key) - Ok(true) + #[cfg(feature = "ttl")] + { + let this = self; + if this._is_expired(key, |k| Self::_list_contains_key(&self.list_tree, k))? { + Ok(false) + } else { + // Self::_list_contains_key(&this.list_tree, key) + Ok(true) + } } + + #[cfg(not(feature = "ttl"))] + Self::_list_contains_key(&self.list_tree, key) } + /// Batch insert key-value pairs #[inline] fn _batch_insert(&self, key_vals: Vec<(Key, IVec)>) -> Result<()> { if key_vals.is_empty() { @@ -1195,6 +1303,7 @@ impl SledStorageDB { Ok(()) } + /// Batch remove keys #[inline] fn _batch_remove(&self, keys: Vec) -> Result<()> { if keys.is_empty() { @@ -1235,6 +1344,7 @@ impl SledStorageDB { Ok(()) } + /// Increments a counter #[inline] fn _counter_incr(&self, key: &[u8], increment: isize) -> Result<()> { self.kv_tree.fetch_and_update(key, |old: Option<&[u8]>| { @@ -1254,6 +1364,7 @@ impl SledStorageDB { Ok(()) } + /// Decrements a counter #[inline] fn _counter_decr(&self, key: &[u8], decrement: isize) -> Result<()> { self.kv_tree.fetch_and_update(key, |old: Option<&[u8]>| { @@ -1273,6 +1384,7 @@ impl SledStorageDB { Ok(()) } + /// Gets counter value #[inline] fn _counter_get(&self, key: &[u8]) -> Result> { let this = self; @@ -1285,6 +1397,7 @@ impl SledStorageDB { } } + /// Sets counter value #[inline] fn _counter_set(&self, key: &[u8], val: isize) -> Result<()> { let val = val.to_be_bytes().to_vec(); @@ -1306,17 +1419,24 @@ impl SledStorageDB { Ok(()) } + /// Checks if a key exists in key-value store #[inline] fn _self_contains_key(&self, key: &[u8]) -> Result { - let this = self; - if this._is_expired(key, |k| Self::_kv_contains_key(&self.kv_tree, k))? { - Ok(false) - } else { - // this._contains_key(key, KeyType::KV) - Ok(true) + #[cfg(feature = "ttl")] + { + let this = self; + if this._is_expired(key, |k| Self::_kv_contains_key(&self.kv_tree, k))? { + Ok(false) + } else { + // this._contains_key(key, KeyType::KV) + Ok(true) + } } + #[cfg(not(feature = "ttl"))] + Self::_kv_contains_key(&self.kv_tree, key) } + /// Sets expiration time for a key (TTL feature) #[inline] #[cfg(feature = "ttl")] fn _expire_at(&self, key: &[u8], at: TimestampMillis, key_type: KeyType) -> Result { @@ -1332,6 +1452,7 @@ impl SledStorageDB { } } + /// Transactionally sets expiration time (TTL feature) #[inline] #[cfg(feature = "ttl")] fn _tx_expire_at( @@ -1349,6 +1470,7 @@ impl SledStorageDB { Ok(res) } + /// Gets time-to-live for a key (TTL feature) #[inline] #[cfg(feature = "ttl")] fn _self_ttl(&self, key: &[u8]) -> Result> { @@ -1357,16 +1479,19 @@ impl SledStorageDB { .and_then(|(ttl, _)| if ttl > 0 { Some(ttl) } else { None })) } + /// Creates an iterator for map prefixes #[inline] fn _map_scan_prefix(&self) -> sled::Iter { self.map_tree.scan_prefix(MAP_NAME_PREFIX) } + /// Creates an iterator for list prefixes #[inline] fn _list_scan_prefix(&self) -> sled::Iter { self.list_tree.scan_prefix(LIST_NAME_PREFIX) } + /// Creates an iterator for database scan with pattern #[inline] fn _db_scan_prefix(&self, pattern: Vec) -> sled::Iter { let mut last_esc_char = false; @@ -1404,6 +1529,7 @@ impl SledStorageDB { iter } + /// Gets number of key-value pairs #[inline] fn _kv_len(&self) -> usize { #[cfg(feature = "ttl")] @@ -1418,11 +1544,13 @@ impl SledStorageDB { self.kv_tree.len() } + /// Gets total database size #[inline] fn _db_size(&self) -> usize { self.db.len() + self.kv_tree.len() + self.map_tree.len() + self.list_tree.len() } + /// Sends a command to the background processor #[inline] async fn cmd_send(&self, cmd: Command) -> Result<()> { self.active_count.fetch_add(1, Ordering::Relaxed); @@ -1434,11 +1562,13 @@ impl SledStorageDB { } } + /// Gets a map handle #[inline] fn _map>(&self, name: N) -> SledStorageMap { SledStorageMap::_new(name.as_ref().to_vec(), self.clone()) } + /// Gets a list handle #[inline] fn _list>(&self, name: V) -> SledStorageList { SledStorageList::_new(name.as_ref().to_vec(), self.clone()) @@ -1450,6 +1580,7 @@ impl StorageDB for SledStorageDB { type MapType = SledStorageMap; type ListType = SledStorageList; + /// Creates or gets a map with optional expiration #[inline] async fn map + Sync + Send>( &self, @@ -1459,6 +1590,7 @@ impl StorageDB for SledStorageDB { SledStorageMap::new_expire(name.as_ref().to_vec(), expire, self.clone()).await } + /// Removes a map #[inline] async fn map_remove(&self, name: K) -> Result<()> where @@ -1471,6 +1603,7 @@ impl StorageDB for SledStorageDB { Ok(()) } + /// Checks if a map exists #[inline] async fn map_contains_key + Sync + Send>(&self, key: K) -> Result { let (tx, rx) = oneshot::channel(); @@ -1483,6 +1616,7 @@ impl StorageDB for SledStorageDB { Ok(rx.await??) } + /// Creates or gets a list with optional expiration #[inline] async fn list + Sync + Send>( &self, @@ -1492,6 +1626,7 @@ impl StorageDB for SledStorageDB { SledStorageList::new_expire(name.as_ref().to_vec(), expire, self.clone()).await } + /// Removes a list #[inline] async fn list_remove(&self, name: K) -> Result<()> where @@ -1508,6 +1643,7 @@ impl StorageDB for SledStorageDB { Ok(()) } + /// Checks if a list exists #[inline] async fn list_contains_key + Sync + Send>(&self, key: K) -> Result { let (tx, rx) = oneshot::channel(); @@ -1520,6 +1656,7 @@ impl StorageDB for SledStorageDB { Ok(rx.await??) } + /// Inserts a key-value pair #[inline] async fn insert(&self, key: K, val: &V) -> Result<()> where @@ -1539,6 +1676,7 @@ impl StorageDB for SledStorageDB { Ok(()) } + /// Gets a value by key #[inline] async fn get(&self, key: K) -> Result> where @@ -1554,6 +1692,7 @@ impl StorageDB for SledStorageDB { } } + /// Removes a key-value pair #[inline] async fn remove(&self, key: K) -> Result<()> where @@ -1566,6 +1705,7 @@ impl StorageDB for SledStorageDB { Ok(()) } + /// Batch inserts key-value pairs #[inline] async fn batch_insert(&self, key_vals: Vec<(Key, V)>) -> Result<()> where @@ -1590,6 +1730,7 @@ impl StorageDB for SledStorageDB { Ok(rx.await??) } + /// Batch removes keys #[inline] async fn batch_remove(&self, keys: Vec) -> Result<()> { if keys.is_empty() { @@ -1602,6 +1743,7 @@ impl StorageDB for SledStorageDB { Ok(rx.await??) } + /// Increments a counter #[inline] async fn counter_incr(&self, key: K, increment: isize) -> Result<()> where @@ -1618,6 +1760,7 @@ impl StorageDB for SledStorageDB { Ok(rx.await??) } + /// Decrements a counter #[inline] async fn counter_decr(&self, key: K, decrement: isize) -> Result<()> where @@ -1634,6 +1777,7 @@ impl StorageDB for SledStorageDB { Ok(rx.await??) } + /// Gets counter value #[inline] async fn counter_get(&self, key: K) -> Result> where @@ -1645,6 +1789,7 @@ impl StorageDB for SledStorageDB { Ok(rx.await??) } + /// Sets counter value #[inline] async fn counter_set(&self, key: K, val: isize) -> Result<()> where @@ -1661,6 +1806,7 @@ impl StorageDB for SledStorageDB { Ok(rx.await??) } + /// Checks if a key exists #[inline] async fn contains_key + Sync + Send>(&self, key: K) -> Result { let (tx, rx) = oneshot::channel(); @@ -1673,6 +1819,7 @@ impl StorageDB for SledStorageDB { Ok(rx.await??) } + /// Gets number of key-value pairs (if enabled) #[inline] #[cfg(feature = "len")] async fn len(&self) -> Result { @@ -1681,6 +1828,7 @@ impl StorageDB for SledStorageDB { Ok(rx.await?) } + /// Gets total database size #[inline] async fn db_size(&self) -> Result { let (tx, rx) = oneshot::channel(); @@ -1688,6 +1836,7 @@ impl StorageDB for SledStorageDB { Ok(rx.await?) } + /// Sets expiration time for a key (TTL feature) #[inline] #[cfg(feature = "ttl")] async fn expire_at(&self, key: K, at: TimestampMillis) -> Result @@ -1705,6 +1854,7 @@ impl StorageDB for SledStorageDB { Ok(rx.await??) } + /// Sets time-to-live for a key (TTL feature) #[inline] #[cfg(feature = "ttl")] async fn expire(&self, key: K, dur: TimestampMillis) -> Result @@ -1715,6 +1865,7 @@ impl StorageDB for SledStorageDB { self.expire_at(key, at).await } + /// Gets time-to-live for a key (TTL feature) #[inline] #[cfg(feature = "ttl")] async fn ttl(&self, key: K) -> Result> @@ -1727,6 +1878,7 @@ impl StorageDB for SledStorageDB { Ok(rx.await??) } + /// Iterates over all maps #[inline] async fn map_iter<'a>( &'a mut self, @@ -1739,6 +1891,7 @@ impl StorageDB for SledStorageDB { Ok(iter) } + /// Iterates over all lists #[inline] async fn list_iter<'a>( &'a mut self, @@ -1754,6 +1907,7 @@ impl StorageDB for SledStorageDB { Ok(iter) } + /// Scans keys matching pattern async fn scan<'a, P>( &'a mut self, pattern: P, @@ -1775,6 +1929,7 @@ impl StorageDB for SledStorageDB { Ok(iter) } + /// Gets database information #[inline] async fn info(&self) -> Result { let active_count = self.active_count.load(Ordering::Relaxed); @@ -1836,17 +1991,25 @@ impl StorageDB for SledStorageDB { } } +/// Map structure for key-value storage within a namespace #[derive(Clone)] pub struct SledStorageMap { + /// Map name name: Key, + /// Prefix for map keys map_prefix_name: Key, + /// Prefix for map items map_item_prefix_name: Key, + /// Key for map count map_count_key_name: Key, + /// Flag indicating if map is empty empty: Arc, + /// Database handle pub(crate) db: SledStorageDB, } impl SledStorageMap { + /// Creates a new map with optional expiration #[inline] async fn new_expire( name: Key, @@ -1859,6 +2022,7 @@ impl SledStorageMap { rx.await? } + /// Internal method to create map with expiration #[inline] fn _new_expire( name: Key, @@ -1874,6 +2038,7 @@ impl SledStorageMap { Ok(m) } + /// Internal method to create map #[inline] fn _new(name: Key, db: SledStorageDB) -> Self { let map_prefix_name = SledStorageDB::make_map_prefix_name(name.as_slice()); @@ -1889,22 +2054,26 @@ impl SledStorageMap { } } + /// Gets the underlying tree #[inline] fn tree(&self) -> &sled::Tree { &self.db.map_tree } + /// Creates a full item key #[inline] fn make_map_item_key>(&self, key: K) -> Key { [self.map_item_prefix_name.as_ref(), key.as_ref()].concat() } + /// Gets map length (if enabled) #[cfg(feature = "map_len")] #[inline] fn _len_get(&self) -> Result { self._counter_get(self.map_count_key_name.as_slice()) } + /// Transactionally increments a counter #[inline] fn _tx_counter_inc>( tx: &TransactionalTree, @@ -1925,6 +2094,7 @@ impl SledStorageMap { Ok(()) } + /// Transactionally decrements a counter #[inline] fn _tx_counter_dec>( tx: &TransactionalTree, @@ -1949,6 +2119,7 @@ impl SledStorageMap { Ok(()) } + /// Transactionally gets counter value #[inline] fn _tx_counter_get, E>( tx: &TransactionalTree, @@ -1969,6 +2140,7 @@ impl SledStorageMap { } } + /// Transactionally sets counter value #[inline] fn _tx_counter_set, E>( tx: &TransactionalTree, @@ -1979,6 +2151,7 @@ impl SledStorageMap { Ok(()) } + /// Transactionally removes counter #[inline] fn _tx_counter_remove, E>( tx: &TransactionalTree, @@ -1988,6 +2161,7 @@ impl SledStorageMap { Ok(()) } + /// Gets counter value #[inline] fn _counter_get>(&self, key: K) -> Result { if let Some(v) = self.tree().get(key)? { @@ -1997,6 +2171,7 @@ impl SledStorageMap { } } + /// Initializes counter if not present #[inline] fn _counter_init(&self) -> Result<()> { let tree = self.tree(); @@ -2009,6 +2184,7 @@ impl SledStorageMap { Ok(()) } + /// Clears the map #[inline] fn _clear(&self) -> Result<()> { let batch = self._make_clear_batch(); @@ -2018,6 +2194,7 @@ impl SledStorageMap { Ok(()) } + /// Transactionally clears the map #[inline] fn _tx_clear( &self, @@ -2029,6 +2206,7 @@ impl SledStorageMap { Ok(()) } + /// Creates batch for clearing map #[inline] fn _make_clear_batch(&self) -> Batch { let mut batch = Batch::default(); @@ -2046,6 +2224,7 @@ impl SledStorageMap { batch } + /// Inserts a key-value pair into the map #[inline] fn _insert(&self, key: IVec, val: IVec) -> Result<()> { let item_key = self.make_map_item_key(key.as_ref()); @@ -2093,6 +2272,7 @@ impl SledStorageMap { Ok(()) } + /// Gets a value from the map #[inline] fn _get(&self, key: IVec) -> Result> { let this = self; @@ -2107,6 +2287,7 @@ impl SledStorageMap { Ok(res) } + /// Removes a key from the map #[inline] fn _remove(&self, key: IVec) -> Result<()> { let tree = self.tree(); @@ -2132,12 +2313,14 @@ impl SledStorageMap { Ok(()) } + /// Checks if key exists in map #[inline] fn _contains_key(&self, key: IVec) -> Result { let key = self.make_map_item_key(key.as_ref()); Ok(self.tree().contains_key(key)?) } + /// Gets map length (if enabled) #[cfg(feature = "map_len")] #[inline] fn _len(&self) -> Result { @@ -2154,6 +2337,7 @@ impl SledStorageMap { Ok(len as usize) } + /// Checks if map is empty #[inline] fn _is_empty(&self) -> Result { let this = self; @@ -2172,6 +2356,7 @@ impl SledStorageMap { Ok(res) } + /// Removes and returns a value #[inline] fn _remove_and_fetch(&self, key: IVec) -> Result> { let key = self.make_map_item_key(key.as_ref()); @@ -2206,6 +2391,7 @@ impl SledStorageMap { Ok(removed) } + /// Removes keys with prefix #[inline] fn _remove_with_prefix(&self, prefix: IVec) -> Result<()> { let tree = self.tree(); @@ -2256,6 +2442,7 @@ impl SledStorageMap { Ok(()) } + /// Batch inserts key-value pairs #[inline] fn _batch_insert(&self, key_vals: Vec<(IVec, IVec)>) -> Result<()> { for (k, v) in key_vals { @@ -2264,6 +2451,7 @@ impl SledStorageMap { Ok(()) } + /// Batch removes keys #[inline] fn _batch_remove(&self, keys: Vec) -> Result<()> { for k in keys { @@ -2272,12 +2460,14 @@ impl SledStorageMap { Ok(()) } + /// Sets expiration time (TTL feature) #[cfg(feature = "ttl")] #[inline] fn _expire_at(&self, at: TimestampMillis) -> Result { self.db._expire_at(self.name.as_slice(), at, KeyType::Map) } + /// Gets time-to-live (TTL feature) #[cfg(feature = "ttl")] #[inline] fn _ttl(&self) -> Result> { @@ -2290,6 +2480,7 @@ impl SledStorageMap { Ok(res) } + /// Checks if map is expired #[inline] fn _is_expired(&self) -> Result { self.db._is_expired(self.name.as_slice(), |k| { @@ -2297,6 +2488,7 @@ impl SledStorageMap { }) } + /// Checks if map is expired (async) #[inline] async fn call_is_expired(&self) -> Result { let (tx, rx) = oneshot::channel(); @@ -2306,6 +2498,7 @@ impl SledStorageMap { rx.await? } + /// Creates prefix iterator #[inline] fn _prefix_iter(&self, prefix: Option) -> sled::Iter { if let Some(prefix) = prefix { @@ -2317,6 +2510,7 @@ impl SledStorageMap { } } + /// Creates prefix iterator (async) #[inline] async fn call_prefix_iter(&self, prefix: Option) -> Result { let (tx, rx) = oneshot::channel(); @@ -2329,11 +2523,13 @@ impl SledStorageMap { #[async_trait] impl Map for SledStorageMap { + /// Gets map name #[inline] fn name(&self) -> &[u8] { self.name.as_slice() } + /// Inserts a key-value pair #[inline] async fn insert(&self, key: K, val: &V) -> Result<()> where @@ -2354,6 +2550,7 @@ impl Map for SledStorageMap { Ok(()) } + /// Gets a value by key #[inline] async fn get(&self, key: K) -> Result> where @@ -2371,6 +2568,7 @@ impl Map for SledStorageMap { } } + /// Removes a key #[inline] async fn remove(&self, key: K) -> Result<()> where @@ -2384,6 +2582,7 @@ impl Map for SledStorageMap { Ok(()) } + /// Checks if key exists #[inline] async fn contains_key + Sync + Send>(&self, key: K) -> Result { let (tx, rx) = oneshot::channel(); @@ -2397,6 +2596,7 @@ impl Map for SledStorageMap { Ok(rx.await??) } + /// Gets map length (if enabled) #[cfg(feature = "map_len")] #[inline] async fn len(&self) -> Result { @@ -2405,6 +2605,7 @@ impl Map for SledStorageMap { Ok(rx.await??) } + /// Checks if map is empty #[inline] async fn is_empty(&self) -> Result { let (tx, rx) = oneshot::channel(); @@ -2414,6 +2615,7 @@ impl Map for SledStorageMap { Ok(rx.await??) } + /// Clears the map #[inline] async fn clear(&self) -> Result<()> { let (tx, rx) = oneshot::channel(); @@ -2424,6 +2626,7 @@ impl Map for SledStorageMap { Ok(()) } + /// Removes and returns a value #[inline] async fn remove_and_fetch(&self, key: K) -> Result> where @@ -2445,6 +2648,7 @@ impl Map for SledStorageMap { } } + /// Removes keys with prefix #[inline] async fn remove_with_prefix(&self, prefix: K) -> Result<()> where @@ -2462,6 +2666,7 @@ impl Map for SledStorageMap { Ok(()) } + /// Batch inserts key-value pairs #[inline] async fn batch_insert(&self, key_vals: Vec<(Key, V)>) -> Result<()> where @@ -2484,6 +2689,7 @@ impl Map for SledStorageMap { Ok(()) } + /// Batch removes keys #[inline] async fn batch_remove(&self, keys: Vec) -> Result<()> { let keys = keys.into_iter().map(|k| k.into()).collect::>(); @@ -2496,6 +2702,7 @@ impl Map for SledStorageMap { Ok(()) } + /// Iterates over map items #[inline] async fn iter<'a, V>( &'a mut self, @@ -2526,6 +2733,7 @@ impl Map for SledStorageMap { Ok(res) } + /// Iterates over map keys #[inline] async fn key_iter<'a>( &'a mut self, @@ -2552,6 +2760,7 @@ impl Map for SledStorageMap { Ok(res) } + /// Iterates over items with prefix #[inline] async fn prefix_iter<'a, P, V>( &'a mut self, @@ -2585,6 +2794,7 @@ impl Map for SledStorageMap { Ok(res) } + /// Sets expiration time (TTL feature) #[cfg(feature = "ttl")] async fn expire_at(&self, at: TimestampMillis) -> Result { let (tx, rx) = oneshot::channel(); @@ -2594,12 +2804,14 @@ impl Map for SledStorageMap { Ok(rx.await??) } + /// Sets time-to-live (TTL feature) #[cfg(feature = "ttl")] async fn expire(&self, dur: TimestampMillis) -> Result { let at = timestamp_millis() + dur; self.expire_at(at).await } + /// Gets time-to-live (TTL feature) #[cfg(feature = "ttl")] async fn ttl(&self) -> Result> { let (tx, rx) = oneshot::channel(); @@ -2608,14 +2820,19 @@ impl Map for SledStorageMap { } } +/// List structure for queue-like storage within a namespace #[derive(Clone)] pub struct SledStorageList { + /// List name name: Key, + /// Prefix for list keys prefix_name: Key, + /// Database handle pub(crate) db: SledStorageDB, } impl SledStorageList { + /// Creates a new list with optional expiration #[inline] async fn new_expire( name: Key, @@ -2628,6 +2845,7 @@ impl SledStorageList { rx.await? } + /// Internal method to create list with expiration #[inline] fn _new_expire( name: Key, @@ -2642,6 +2860,7 @@ impl SledStorageList { Ok(l) } + /// Internal method to create list #[inline] fn _new(name: Key, db: SledStorageDB) -> Self { let prefix_name = SledStorageDB::make_list_prefix(name.as_slice()); @@ -2652,22 +2871,26 @@ impl SledStorageList { } } + /// Gets list name #[inline] pub(crate) fn name(&self) -> &[u8] { self.name.as_slice() } + /// Gets the underlying tree #[inline] pub(crate) fn tree(&self) -> &sled::Tree { &self.db.list_tree } + /// Creates list count key #[inline] fn make_list_count_key(&self) -> Vec { let list_count_key = [self.prefix_name.as_ref(), LIST_KEY_COUNT_SUFFIX].concat(); list_count_key } + /// Creates list content prefix #[inline] fn make_list_content_prefix(prefix_name: &[u8], idx: Option<&[u8]>) -> Vec { if let Some(idx) = idx { @@ -2677,6 +2900,7 @@ impl SledStorageList { } } + /// Creates list content key #[inline] fn make_list_content_key(&self, idx: usize) -> Vec { Self::make_list_content_prefix( @@ -2685,6 +2909,7 @@ impl SledStorageList { ) } + /// Creates batch of list content keys #[inline] fn make_list_content_keys(&self, start: usize, end: usize) -> Vec> { (start..end) @@ -2692,6 +2917,7 @@ impl SledStorageList { .collect() } + /// Transactionally gets list count #[inline] fn tx_list_count_get( tx: &TransactionalTree, @@ -2713,6 +2939,7 @@ impl SledStorageList { } } + /// Transactionally sets list count #[inline] fn tx_list_count_set( tx: &TransactionalTree, @@ -2733,6 +2960,7 @@ impl SledStorageList { Ok(()) } + /// Transactionally sets list content #[inline] fn tx_list_content_set( tx: &TransactionalTree, @@ -2747,6 +2975,7 @@ impl SledStorageList { Ok(()) } + /// Transactionally sets batch list content #[inline] fn tx_list_content_batch_set( tx: &TransactionalTree, @@ -2764,6 +2993,7 @@ impl SledStorageList { Ok(()) } + /// Clears the list #[inline] fn _clear(&self) -> Result<()> { let mut batch = Batch::default(); @@ -2789,6 +3019,7 @@ impl SledStorageList { Ok(()) } + /// Transactionally clears the list #[inline] fn _tx_clear( list_tree_tx: &TransactionalTree, @@ -2798,6 +3029,7 @@ impl SledStorageList { Ok(()) } + /// Creates batch for clearing list #[inline] fn _make_clear_batch(&self) -> Batch { let mut batch = Batch::default(); @@ -2817,6 +3049,7 @@ impl SledStorageList { batch } + /// Pushes value to list #[inline] fn _push(&self, data: IVec) -> Result<()> { let this = self; @@ -2856,6 +3089,7 @@ impl SledStorageList { Ok(()) } + /// Pushes multiple values to list #[inline] fn _pushs(&self, vals: Vec) -> Result<()> { if vals.is_empty() { @@ -2905,6 +3139,7 @@ impl SledStorageList { Ok(()) } + /// Pushes value with limit #[inline] fn _push_limit( &self, @@ -2974,6 +3209,7 @@ impl SledStorageList { Ok(removed) } + /// Pops value from list #[inline] fn _pop(&self) -> Result> { let this = self; @@ -3004,6 +3240,7 @@ impl SledStorageList { Ok(removed) } + /// Gets all values in list #[inline] fn _all(&self) -> Result> { let this = self; @@ -3025,6 +3262,7 @@ impl SledStorageList { Ok(res) } + /// Gets value by index #[inline] fn _get_index(&self, idx: usize) -> Result> { let this = self; @@ -3056,6 +3294,7 @@ impl SledStorageList { Ok(res) } + /// Gets list length #[inline] fn _len(&self) -> Result { let this = self; @@ -3077,6 +3316,7 @@ impl SledStorageList { Ok(res) } + /// Checks if list is empty #[inline] fn _is_empty(&self) -> Result { let this = self; @@ -3099,12 +3339,14 @@ impl SledStorageList { Ok(res) } + /// Sets expiration time (TTL feature) #[cfg(feature = "ttl")] #[inline] fn _expire_at(&self, at: TimestampMillis) -> Result { self.db._expire_at(self.name.as_slice(), at, KeyType::List) } + /// Gets time-to-live (TTL feature) #[cfg(feature = "ttl")] #[inline] fn _ttl(&self) -> Result> { @@ -3116,6 +3358,7 @@ impl SledStorageList { .and_then(|(at, _)| if at > 0 { Some(at) } else { None })) } + /// Checks if list is expired #[inline] fn _is_expired(&self) -> Result { self.db._is_expired(self.name.as_slice(), |k| { @@ -3123,6 +3366,7 @@ impl SledStorageList { }) } + /// Checks if list is expired (async) #[inline] async fn call_is_expired(&self) -> Result { let (tx, rx) = oneshot::channel(); @@ -3132,12 +3376,14 @@ impl SledStorageList { rx.await? } + /// Creates prefix iterator #[inline] fn _prefix_iter(&self) -> sled::Iter { let list_content_prefix = Self::make_list_content_prefix(self.prefix_name.as_slice(), None); self.tree().scan_prefix(list_content_prefix) } + /// Creates prefix iterator (async) #[inline] async fn call_prefix_iter(&self) -> Result { let (tx, rx) = oneshot::channel(); @@ -3150,11 +3396,13 @@ impl SledStorageList { #[async_trait] impl List for SledStorageList { + /// Gets list name #[inline] fn name(&self) -> &[u8] { self.name.as_slice() } + /// Pushes value to list #[inline] async fn push(&self, val: &V) -> Result<()> where @@ -3169,6 +3417,7 @@ impl List for SledStorageList { Ok(()) } + /// Pushes multiple values to list #[inline] async fn pushs(&self, vals: Vec) -> Result<()> where @@ -3195,6 +3444,7 @@ impl List for SledStorageList { Ok(()) } + /// Pushes value with limit #[inline] async fn push_limit( &self, @@ -3230,6 +3480,7 @@ impl List for SledStorageList { Ok(removed) } + /// Pops value from list #[inline] async fn pop(&self) -> Result> where @@ -3249,6 +3500,7 @@ impl List for SledStorageList { Ok(removed) } + /// Gets all values in list #[inline] async fn all(&self) -> Result> where @@ -3263,6 +3515,7 @@ impl List for SledStorageList { .collect::>>() } + /// Gets value by index #[inline] async fn get_index(&self, idx: usize) -> Result> where @@ -3280,6 +3533,7 @@ impl List for SledStorageList { }) } + /// Gets list length #[inline] async fn len(&self) -> Result { let (tx, rx) = oneshot::channel(); @@ -3287,6 +3541,7 @@ impl List for SledStorageList { Ok(rx.await??) } + /// Checks if list is empty #[inline] async fn is_empty(&self) -> Result { let (tx, rx) = oneshot::channel(); @@ -3296,6 +3551,7 @@ impl List for SledStorageList { Ok(rx.await??) } + /// Clears the list #[inline] async fn clear(&self) -> Result<()> { let (tx, rx) = oneshot::channel(); @@ -3305,6 +3561,7 @@ impl List for SledStorageList { Ok(rx.await??) } + /// Iterates over list values #[inline] async fn iter<'a, V>( &'a mut self, @@ -3334,6 +3591,7 @@ impl List for SledStorageList { Ok(res) } + /// Sets expiration time (TTL feature) #[cfg(feature = "ttl")] async fn expire_at(&self, at: TimestampMillis) -> Result { let (tx, rx) = oneshot::channel(); @@ -3343,12 +3601,14 @@ impl List for SledStorageList { Ok(rx.await??) } + /// Sets time-to-live (TTL feature) #[cfg(feature = "ttl")] async fn expire(&self, dur: TimestampMillis) -> Result { let at = timestamp_millis() + dur; self.expire_at(at).await } + /// Gets time-to-live (TTL feature) #[cfg(feature = "ttl")] async fn ttl(&self) -> Result> { let (tx, rx) = oneshot::channel(); @@ -3357,6 +3617,7 @@ impl List for SledStorageList { } } +/// Async iterator for map items pub struct AsyncIter<'a, V> { db: &'a SledStorageDB, prefix_len: usize, @@ -3364,14 +3625,14 @@ pub struct AsyncIter<'a, V> { _m: std::marker::PhantomData, } -impl<'a, V> Debug for AsyncIter<'a, V> { +impl Debug for AsyncIter<'_, V> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("AsyncIter .. ").finish() } } #[async_trait] -impl<'a, V> AsyncIterator for AsyncIter<'a, V> +impl AsyncIterator for AsyncIter<'_, V> where V: DeserializeOwned + Sync + Send + 'static, { @@ -3410,20 +3671,21 @@ where } } +/// Async iterator for map keys pub struct AsyncKeyIter<'a> { db: &'a SledStorageDB, prefix_len: usize, iter: Option, } -impl<'a> Debug for AsyncKeyIter<'a> { +impl Debug for AsyncKeyIter<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("AsyncKeyIter .. ").finish() } } #[async_trait] -impl<'a> AsyncIterator for AsyncKeyIter<'a> { +impl AsyncIterator for AsyncKeyIter<'_> { type Item = Result; async fn next(&mut self) -> Option { @@ -3454,20 +3716,21 @@ impl<'a> AsyncIterator for AsyncKeyIter<'a> { } } +/// Async iterator for list values pub struct AsyncListValIter<'a, V> { db: &'a SledStorageDB, iter: Option, _m: std::marker::PhantomData, } -impl<'a, V> Debug for AsyncListValIter<'a, V> { +impl Debug for AsyncListValIter<'_, V> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("AsyncListValIter .. ").finish() } } #[async_trait] -impl<'a, V> AsyncIterator for AsyncListValIter<'a, V> +impl AsyncIterator for AsyncListValIter<'_, V> where V: DeserializeOwned + Sync + Send + 'static, { @@ -3500,6 +3763,7 @@ where } } +/// Empty iterator pub struct AsyncEmptyIter { _m: std::marker::PhantomData, } @@ -3522,6 +3786,7 @@ where } } +/// Async iterator for maps pub struct AsyncMapIter<'a> { db: &'a SledStorageDB, iter: Option, @@ -3536,14 +3801,14 @@ impl<'a> AsyncMapIter<'a> { } } -impl<'a> Debug for AsyncMapIter<'a> { +impl Debug for AsyncMapIter<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("AsyncMapIter .. ").finish() } } #[async_trait] -impl<'a> AsyncIterator for AsyncMapIter<'a> { +impl AsyncIterator for AsyncMapIter<'_> { type Item = Result; async fn next(&mut self) -> Option { @@ -3579,19 +3844,20 @@ impl<'a> AsyncIterator for AsyncMapIter<'a> { } } +/// Async iterator for lists pub struct AsyncListIter<'a> { db: &'a SledStorageDB, iter: Option, } -impl<'a> Debug for AsyncListIter<'a> { +impl Debug for AsyncListIter<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("AsyncListIter .. ").finish() } } #[async_trait] -impl<'a> AsyncIterator for AsyncListIter<'a> { +impl AsyncIterator for AsyncListIter<'_> { type Item = Result; async fn next(&mut self) -> Option { @@ -3626,20 +3892,21 @@ impl<'a> AsyncIterator for AsyncListIter<'a> { } } +/// Async iterator for database keys with pattern matching pub struct AsyncDbKeyIter<'a> { db: &'a SledStorageDB, pattern: Pattern, iter: Option, } -impl<'a> Debug for AsyncDbKeyIter<'a> { +impl Debug for AsyncDbKeyIter<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("AsyncDbKeyIter .. ").finish() } } #[async_trait] -impl<'a> AsyncIterator for AsyncDbKeyIter<'a> { +impl AsyncIterator for AsyncDbKeyIter<'_> { type Item = Result; async fn next(&mut self) -> Option {