// Copyright 2018-2019 Mozilla // // Licensed under the Apache License, Version 2.0 (the "License"); you may not use // this file except in compliance with the License. You may obtain a copy of the // License at http://www.apache.org/licenses/LICENSE-2.0 // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License.
// TODO: change this back to `clippy::cognitive_complexity` when Clippy stable // deprecates `clippy::cyclomatic_complexity`. #![allow(clippy::complexity)]
use std::{
fs,
path::Path,
str,
sync::{Arc, RwLock},
thread,
};
use byteorder::{ByteOrder, LittleEndian}; use tempfile::Builder;
fn check_rkv(k: &Rkv<LmdbEnvironment>) { let _ = k
.open_single(None, StoreOptions::create())
.expect("created default");
let s = k.open_single("s", StoreOptions::create()).expect("opened"); let reader = k.read().expect("reader");
let result = s.get(&reader, "foo");
assert_eq!(None, result.expect("success but no value"));
}
// The default size is 1MB. const DEFAULT_SIZE: usize = 1024 * 1024;
/// We can't open a directory that doesn't exist. #[test] fn test_open_fails() { let root = Builder::new()
.prefix("test_open_fails")
.tempdir()
.expect("tempdir");
assert!(root.path().exists());
let nope = root.path().join("nope/");
assert!(!nope.exists());
let pb = nope.to_path_buf(); match Rkv::new::<Lmdb>(nope.as_path()).err() {
Some(StoreError::UnsuitableEnvironmentPath(p)) => {
assert_eq!(pb, p);
}
_ => panic!("expected error"),
};
}
let k = Rkv::with_capacity::<Lmdb>(root.path(), 1).expect("rkv");
check_rkv(&k);
// This errors with "opened: DbsFull" because we specified a capacity of one (database), // and check_rkv already opened one (plus the default database, which doesn't count // against the limit). let _zzz = k
.open_single("zzz", StoreOptions::create())
.expect("opened");
}
let k = Rkv::with_capacity::<Lmdb>(root.path(), 1).expect("rkv");
check_rkv(&k);
// This doesn't error with "opened: DbsFull" with because even though we specified a // capacity of one (database), and check_rkv already opened one, the default database // doesn't count against the limit. let _zzz = k.open_single(None, StoreOptions::create()).expect("opened");
}
let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); let sk = k
.open_single("test", StoreOptions::create())
.expect("opened");
// Writing a large enough value should cause LMDB to fail on MapFull. // We write a string that is larger than the default map size. let val = "x".repeat(get_larger_than_default_map_size_value()); letmut writer = k.write().expect("writer");
sk.put(&mut writer, "foo", &Value::Str(&val))
.expect("wrote");
}
letmut builder = Rkv::environment_builder::<Lmdb>(); // Set the map size to the size of the value we'll store in it + 100KiB, // which ensures that there's enough space for the value and metadata.
builder.set_map_size(
get_larger_than_default_map_size_value() + 100 * 1024, /* 100KiB */
);
builder.set_max_dbs(2); let k = Rkv::from_builder(root.path(), builder).unwrap(); let sk = k
.open_single("test", StoreOptions::create())
.expect("opened"); let val = "x".repeat(get_larger_than_default_map_size_value());
let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
// First create the store, and start a write transaction on it. let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); letmut writer = k.write().expect("writer");
sk.put(&mut writer, "foo", &Value::Str("bar"))
.expect("write");
// Open the same store for read, note that the write transaction is still in progress, // it should not block the reader though. let sk_readonly = k
.open_single("sk", StoreOptions::default())
.expect("opened");
writer.commit().expect("commit");
// Now the write transaction is committed, any followed reads should see its change. let reader = k.read().expect("reader");
assert_eq!(
sk_readonly.get(&reader, "foo").expect("read"),
Some(Value::Str("bar"))
);
}
#[test] #[should_panic(expected = "open a missing store")] fn test_open_a_missing_store() { let root = Builder::new()
.prefix("test_open_a_missing_store")
.tempdir()
.expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); let _sk = k
.open_single("sk", StoreOptions::default())
.expect("open a missing store");
}
#[test] #[should_panic(expected = "new failed: FileInvalid")] fn test_open_a_broken_store() { let root = Builder::new()
.prefix("test_open_a_missing_store")
.tempdir()
.expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let dbfile = root.path().join("data.mdb");
fs::write(dbfile, "bogus").expect("dbfile created");
let _ = Rkv::new::<Lmdb>(root.path()).expect("new failed");
}
let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
// First create the store let _sk = k.open_single("sk", StoreOptions::create()).expect("opened");
// Open a reader on this store let _reader = k.read().expect("reader");
// Open the same store for read while the reader is in progress will panic let store = k.open_single("sk", StoreOptions::default()); match store {
Err(StoreError::OpenAttemptedDuringTransaction(_thread_id)) => (),
_ => panic!("should panic"),
}
}
let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
// Test reading a number, modifying it, and then writing it back. // We have to be done with the Value::I64 before calling Writer::put, // as the Value::I64 borrows an immutable reference to the Writer. // So we extract and copy its primitive value.
let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
// Test reading a string, modifying it, and then writing it back. // We have to be done with the Value::Str before calling Writer::put, // as the Value::Str (and its underlying &str) borrows an immutable // reference to the Writer. So we copy it to a String.
{ let reader = k.read().unwrap();
assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234)));
}
// Establish a long-lived reader that outlasts a writer. let reader = k.read().expect("reader");
assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234)));
// The reader and writer are isolated.
assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234)));
assert_eq!(s.get(&writer, "foo").expect("read"), Some(Value::I64(999)));
// If we commit the writer, we still have isolation.
writer.commit().expect("committed");
assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234)));
// A new reader sees the committed value. Note that LMDB doesn't allow two // read transactions to exist in the same thread, so we abort the previous one.
reader.abort(); let reader = k.read().expect("reader");
assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(999)));
}
// When storing UTF-16 strings as blobs, we'll need to convert // their [u16] backing storage to [u8]. Test that converting, writing, // reading, and converting back works as expected. let u16_array = [1000, 10000, 54321, 65535];
assert_eq!(sk.get(&writer, "bar").expect("read"), None);
sk.put(&mut writer, "bar", &Value::Blob(&u16_to_u8(&u16_array)))
.expect("wrote"); let u8_array = match sk.get(&writer, "bar").expect("read") {
Some(Value::Blob(val)) => val,
_ => &[],
};
assert_eq!(u8_to_u16(u8_array), u16_array);
}
let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); for i in0..5 { let sk = k
.open_integer(&format!("sk{i}")[..], StoreOptions::create())
.expect("opened");
{ letmut writer = k.write().expect("writer");
sk.put(&mut writer, i, &Value::I64(i64::from(i)))
.expect("wrote");
writer.commit().expect("committed");
}
}
assert_eq!(k.stat().expect("stat").depth(), 1);
assert_eq!(k.stat().expect("stat").entries(), 5);
assert_eq!(k.stat().expect("stat").branch_pages(), 0);
assert_eq!(k.stat().expect("stat").leaf_pages(), 1);
}
// The default size is 1MB.
assert_eq!(info.map_size(), DEFAULT_SIZE); // Should greater than 0 after the write txn.
assert!(info.last_pgno() > 0); // A txn to open_single + a txn to write.
assert_eq!(info.last_txnid(), 2); // The default max readers is 126.
assert_eq!(info.max_readers(), 126);
assert_eq!(info.num_readers(), 0);
// A new reader should increment the reader counter. let _reader = k.read().expect("reader"); let info = k.info().expect("info");
let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
letmut writer = k.write().expect("writer");
sk.put(&mut writer, "foo", &Value::Str("bar"))
.expect("wrote");
writer.commit().expect("commited"); let ratio = k.load_ratio().expect("ratio").unwrap();
assert!(ratio > 0.0_f32 && ratio < 1.0_f32);
// Put data to database should increase the load ratio. letmut writer = k.write().expect("writer");
sk.put(
&mut writer, "bar",
&Value::Str(&"more-than-4KB".repeat(1000)),
)
.expect("wrote");
writer.commit().expect("commited"); let new_ratio = k.load_ratio().expect("ratio").unwrap();
assert!(new_ratio > ratio);
// Clear the database so that all the used pages should go to freelist, hence the ratio // should decrease. letmut writer = k.write().expect("writer");
sk.clear(&mut writer).expect("clear");
writer.commit().expect("commited"); let after_clear_ratio = k.load_ratio().expect("ratio").unwrap();
assert!(after_clear_ratio < new_ratio);
}
// Should be able to write. letmut writer = k.write().expect("writer");
sk.put(&mut writer, "foo", &Value::Str("bar"))
.expect("wrote");
writer.commit().expect("commited");
let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
// An iterator over an empty store returns no values.
{ let reader = k.read().unwrap(); letmut iter = sk.iter_start(&reader).unwrap();
assert!(iter.next().is_none());
}
// Reader.iter() returns (key, value) tuples ordered by key. letmut iter = sk.iter_start(&reader).unwrap(); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "bar");
assert_eq!(val, Value::Bool(true)); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "baz");
assert_eq!(val, Value::Str("héllo, yöu")); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "foo");
assert_eq!(val, Value::I64(1234)); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst");
assert_eq!(val, Value::Str("Emil.RuleZ!")); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "noo");
assert_eq!(val, Value::F64(1234.0.into())); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "你好,遊客");
assert_eq!(val, Value::Str("米克規則"));
assert!(iter.next().is_none());
// Iterators don't loop. Once one returns None, additional calls // to its next() method will always return None.
assert!(iter.next().is_none());
// Reader.iter_from() begins iteration at the first key equal to // or greater than the given key. letmut iter = sk.iter_from(&reader, "moo").unwrap(); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "noo");
assert_eq!(val, Value::F64(1234.0.into())); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "你好,遊客");
assert_eq!(val, Value::Str("米克規則"));
assert!(iter.next().is_none());
// Reader.iter_from() works as expected when the given key is a prefix // of a key in the store. letmut iter = sk.iter_from(&reader, "no").unwrap(); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "noo");
assert_eq!(val, Value::F64(1234.0.into())); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "你好,遊客");
assert_eq!(val, Value::Str("米克規則"));
assert!(iter.next().is_none());
}
#[test] fn test_iter_from_key_greater_than_existing() { let root = Builder::new()
.prefix("test_iter_from_key_greater_than_existing")
.tempdir()
.expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created"); let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); let s1 = k
.open_single("store_1", StoreOptions::create())
.expect("opened"); let s2 = k
.open_single("store_2", StoreOptions::create())
.expect("opened"); let s3 = k
.open_single("store_3", StoreOptions::create())
.expect("opened");
let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); let s1 = k
.open_single("store_1", StoreOptions::create())
.expect("opened"); let s2 = k
.open_single("store_2", StoreOptions::create())
.expect("opened");
// Iterate through the whole store in "s1" letmut iter = s1.iter_start(&reader).unwrap(); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "bar");
assert_eq!(val, Value::Bool(true)); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "baz");
assert_eq!(val, Value::Str("héllo, yöu")); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "foo");
assert_eq!(val, Value::I64(1234)); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst");
assert_eq!(val, Value::Str("Emil.RuleZ!")); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "noo");
assert_eq!(val, Value::F64(1234.0.into())); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "你好,遊客");
assert_eq!(val, Value::Str("米克規則"));
assert!(iter.next().is_none());
// Iterate through the whole store in "s2" letmut iter = s2.iter_start(&reader).unwrap(); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "bar");
assert_eq!(val, Value::Bool(true)); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "baz");
assert_eq!(val, Value::Str("héllo, yöu")); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "foo");
assert_eq!(val, Value::I64(1234)); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst");
assert_eq!(val, Value::Str("Emil.RuleZ!")); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "noo");
assert_eq!(val, Value::F64(1234.0.into())); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "你好,遊客");
assert_eq!(val, Value::Str("米克規則"));
assert!(iter.next().is_none());
// Iterate from a given key in "s1" letmut iter = s1.iter_from(&reader, "moo").unwrap(); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "noo");
assert_eq!(val, Value::F64(1234.0.into())); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "你好,遊客");
assert_eq!(val, Value::Str("米克規則"));
assert!(iter.next().is_none());
// Iterate from a given key in "s2" letmut iter = s2.iter_from(&reader, "moo").unwrap(); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "noo");
assert_eq!(val, Value::F64(1234.0.into())); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "你好,遊客");
assert_eq!(val, Value::Str("米克規則"));
assert!(iter.next().is_none());
// Iterate from a given prefix in "s1" letmut iter = s1.iter_from(&reader, "no").unwrap(); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "noo");
assert_eq!(val, Value::F64(1234.0.into())); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "你好,遊客");
assert_eq!(val, Value::Str("米克規則"));
assert!(iter.next().is_none());
// Iterate from a given prefix in "s2" letmut iter = s2.iter_from(&reader, "no").unwrap(); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "noo");
assert_eq!(val, Value::F64(1234.0.into())); let (key, val) = iter.next().unwrap().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "你好,遊客");
assert_eq!(val, Value::Str("米克規則"));
assert!(iter.next().is_none());
}
let rkv_arc = Arc::new(RwLock::new(
Rkv::new::<Lmdb>(root.path()).expect("new succeeded"),
)); let store = rkv_arc
.read()
.unwrap()
.open_single("test", StoreOptions::create())
.expect("opened");
let num_threads = 10; letmut write_handles = Vec::with_capacity(num_threads as usize); letmut read_handles = Vec::with_capacity(num_threads as usize);
// Note that this isn't intended to demonstrate a good use of threads. // For this shape of data, it would be more performant to write/read // all values using one transaction in a single thread. The point here // is just to confirm that a store can be shared by multiple threads.
// For each KV pair, spawn a thread that writes it to the store. for i in0..num_threads { let rkv_arc = rkv_arc.clone();
write_handles.push(thread::spawn(move || { let rkv = rkv_arc.write().expect("rkv"); letmut writer = rkv.write().expect("writer");
store
.put(&mut writer, i.to_string(), &Value::U64(i))
.expect("written");
writer.commit().unwrap();
}));
} for handle in write_handles {
handle.join().expect("joined");
}
// For each KV pair, spawn a thread that reads it from the store // and returns its value. for i in0..num_threads { let rkv_arc = rkv_arc.clone();
read_handles.push(thread::spawn(move || { let rkv = rkv_arc.read().expect("rkv"); let reader = rkv.read().expect("reader"); let value = match store.get(&reader, i.to_string()) {
Ok(Some(Value::U64(value))) => value,
Ok(Some(_)) => panic!("value type unexpected"),
Ok(None) => panic!("value not found"),
Err(err) => panic!("{}", err),
};
assert_eq!(value, i);
value
}));
}
// Sum the values returned from the threads and confirm that they're // equal to the sum of values written to the threads. let thread_sum: u64 = read_handles
.into_iter()
.map(|handle| handle.join().expect("value"))
.sum();
assert_eq!(thread_sum, (0..num_threads).sum());
}
#[test] fn test_use_value_as_key() { let root = Builder::new()
.prefix("test_use_value_as_key")
.tempdir()
.expect("tempdir"); let rkv = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); let store = rkv
.open_single("store", StoreOptions::create())
.expect("opened");
{ letmut writer = rkv.write().expect("writer");
store
.put(&mut writer, "foo", &Value::Str("bar"))
.expect("wrote");
store
.put(&mut writer, "bar", &Value::Str("baz"))
.expect("wrote");
writer.commit().expect("committed");
}
// It's possible to retrieve a value with a Reader and then use it // as a key with a Writer.
{ let reader = &rkv.read().unwrap(); iflet Some(Value::Str(key)) = store.get(reader, "foo").expect("read") { letmut writer = rkv.write().expect("writer");
store.delete(&mut writer, key).expect("deleted");
writer.commit().expect("committed");
}
}
// You can also retrieve a Value with a Writer and then use it as a key // with the same Writer if you copy the value to an owned type // so the Writer isn't still being borrowed by the retrieved value // when you try to borrow the Writer again to modify that value.
{ letmut writer = rkv.write().expect("writer"); iflet Some(Value::Str(value)) = store.get(&writer, "foo").expect("read") { let key = value.to_owned();
store.delete(&mut writer, key).expect("deleted");
writer.commit().expect("committed");
}
}
// You can also iterate (store, key) pairs to retrieve foreign keys, // then iterate those foreign keys to modify/delete them. // // You need to open the stores in advance, since opening a store // uses a write transaction internally, so opening them while a writer // is extant will hang. // // And you need to copy the values to an owned type so the Writer isn't // still being borrowed by a retrieved value when you try to borrow // the Writer again to modify another value. let fields = vec![
(
rkv.open_single("name1", StoreOptions::create())
.expect("opened"), "key1",
),
(
rkv.open_single("name2", StoreOptions::create())
.expect("opened"), "key2",
),
];
{ letmut foreignkeys = Vec::new(); letmut writer = rkv.write().expect("writer"); for (store, key) in fields.iter() { iflet Some(Value::Str(value)) = store.get(&writer, key).expect("read") {
foreignkeys.push((store, value.to_owned()));
}
} for (store, key) in foreignkeys.iter() {
store.delete(&mut writer, key).expect("deleted");
}
writer.commit().expect("committed");
}
}
Messung V0.5 in Prozent
¤ Dauer der Verarbeitung: 0.41 Sekunden
(vorverarbeitet am 2026-06-18)
¤
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.