diff --git a/chorus-lib/src/ip.rs b/chorus-lib/src/ip.rs index b1cf527..49a3743 100644 --- a/chorus-lib/src/ip.rs +++ b/chorus-lib/src/ip.rs @@ -24,6 +24,10 @@ impl HashedIp { ) } + pub fn from_bytes(bytes: &[u8]) -> HashedIp { + HashedIp(bytes[0..20].try_into().unwrap(), false) + } + pub fn is_loopback(&self) -> bool { self.1 } diff --git a/chorus-lib/src/store/lmdb/mod.rs b/chorus-lib/src/store/lmdb/mod.rs index e7f0555..9740ef3 100644 --- a/chorus-lib/src/store/lmdb/mod.rs +++ b/chorus-lib/src/store/lmdb/mod.rs @@ -409,6 +409,17 @@ impl Lmdb { Ok(()) } + pub fn dump_deleted(&self) -> Result, Error> { + let mut output: Vec = Vec::new(); + let txn = self.read_txn()?; + for i in self.deleted_ids.iter(&txn)? { + let (key, _val) = i?; + let id = Id(key[0..32].try_into().unwrap()); + output.push(id); + } + Ok(output) + } + pub fn get_ip_data(&self, ip: HashedIp) -> Result { let key = &ip.0; let txn = self.read_txn()?; @@ -428,6 +439,18 @@ impl Lmdb { Ok(()) } + pub fn dump_ip_data(&self) -> Result, Error> { + let mut output: Vec<(HashedIp, IpData)> = Vec::new(); + let txn = self.read_txn()?; + for i in self.ip_data.iter(&txn)? { + let (key, val) = i?; + let hashedip = HashedIp::from_bytes(key); + let data = IpData::read_from_buffer(val)?; + output.push((hashedip, data)); + } + Ok(output) + } + pub fn mark_event_approval(&self, id: Id, approval: bool) -> Result<(), Error> { let mut txn = self.write_txn()?; self.approved_events diff --git a/chorus-lib/src/store/mod.rs b/chorus-lib/src/store/mod.rs index 8a7c3be..371a8d5 100644 --- a/chorus-lib/src/store/mod.rs +++ b/chorus-lib/src/store/mod.rs @@ -12,6 +12,7 @@ use crate::ip::{HashedIp, IpData}; use crate::types::{Event, Filter, Id, Kind, Pubkey, Time}; use heed::RwTxn; use std::collections::BTreeSet; +use std::fs; #[derive(Debug)] pub struct Store { @@ -42,6 +43,88 @@ impl Store { Ok(store) } + pub fn rebuild(config: &Config) -> Result<(), Error> { + let dir = format!("{}/lmdb", &config.data_directory); + let dir_bak = format!("{}/lmdb.bak", &config.data_directory); + fs::rename(&dir, &dir_bak)?; + let old_lmdb = Lmdb::new(&dir_bak)?; + let new_lmdb = Lmdb::new(&dir)?; + + let old_align = old_lmdb.get_if_events_are_aligned()?; + + let event_map_file = format!("{}/event.map", &config.data_directory); + let event_map_file_bak = format!("{}/event.map.bak", &config.data_directory); + fs::rename(&event_map_file, &event_map_file_bak)?; + + let old_event_store = EventStore::new(event_map_file_bak, old_align)?; + let new_event_store = EventStore::new(event_map_file, true)?; + + let old_store = Store { + lmdb: old_lmdb, + events: old_event_store, + }; + + old_store.migrate()?; + + let new_store = Store { + lmdb: new_lmdb, + events: new_event_store, + }; + + new_store.migrate()?; + + log::info!("Copying data..."); + + let old_txn = old_store.lmdb.read_txn()?; + let mut new_txn = new_store.lmdb.write_txn()?; + + new_store + .lmdb + .set_if_events_are_aligned(&mut new_txn, true)?; + + // Iterate through all IDs and copy and index all of those events + for i in old_store.lmdb.i_iter(&old_txn)? { + let (_key, val) = i?; + //let id = Id(key[0..32].try_into().unwrap()); + let old_offset: usize = val; + let event = old_store.events.get_event_by_offset(old_offset)?; + let new_offset = new_store.events.store_event(&event)?; + new_store.lmdb.index(&mut new_txn, &event, new_offset)?; + } + + // Copy deleted IDs + let mut deleted = old_store.lmdb.dump_deleted()?; + for id in deleted.drain(..) { + new_store.lmdb.mark_deleted(&mut new_txn, id)?; + } + + new_txn.commit()?; + + // Copy approved events + let mut approvals = old_store.dump_event_approvals()?; + for (id, approval) in approvals.drain(..) { + new_store.mark_event_approval(id, approval)?; + } + + // Copy approved pubkeys + let mut approvals = old_store.dump_pubkey_approvals()?; + for (pubkey, approval) in approvals.drain(..) { + new_store.mark_pubkey_approval(pubkey, approval)?; + } + + // Copy ip data + let mut ipdata = old_store.lmdb.dump_ip_data()?; + for (hashedip, ipdata) in ipdata.drain(..) { + new_store.update_ip_data(hashedip, &ipdata)?; + } + + new_store.lmdb.sync()?; + + log::info!("done."); + + Ok(()) + } + /// Sync the data to disk. This happens periodically, but sometimes it's useful to force /// it. pub fn sync(&self) -> Result<(), Error> { diff --git a/sample/.gitignore b/sample/.gitignore index 2c1ff88..bd82380 100644 --- a/sample/.gitignore +++ b/sample/.gitignore @@ -1,2 +1,4 @@ lmdb/ -event.map \ No newline at end of file +lmdb.bak/ +event.map +event.map.bak \ No newline at end of file