From 437782e770d8d63ae4c0ac37aced46ee21e4f8a9 Mon Sep 17 00:00:00 2001 From: Mike Dilger Date: Tue, 5 Mar 2024 08:27:42 +1300 Subject: [PATCH] Two new config variables for allowing short scrapes --- chorus-bin/src/bin/dump.rs | 2 +- chorus-bin/src/main.rs | 2 +- chorus-lib/src/config.rs | 10 ++++++++++ chorus-lib/src/store/mod.rs | 18 ++++++++++++------ contrib/chorus.toml | 22 ++++++++++++++++++++++ docs/CONFIG.md | 16 ++++++++++++++++ sample/sample.config.toml | 2 ++ 7 files changed, 64 insertions(+), 8 deletions(-) diff --git a/chorus-bin/src/bin/dump.rs b/chorus-bin/src/bin/dump.rs index 9c062b1..0b4bfcb 100644 --- a/chorus-bin/src/bin/dump.rs +++ b/chorus-bin/src/bin/dump.rs @@ -37,7 +37,7 @@ fn main() -> Result<(), Error> { config.allow_scraping = true; // Setup store - let store = Store::new(&config.data_directory, config.allow_scraping)?; + let store = Store::new(&config)?; let mut buffer: [u8; 128] = [0; 128]; let (_incount, _outcount, filter) = Filter::from_json(b"{}", &mut buffer)?; diff --git a/chorus-bin/src/main.rs b/chorus-bin/src/main.rs index b2e5209..b00d846 100644 --- a/chorus-bin/src/main.rs +++ b/chorus-bin/src/main.rs @@ -63,7 +63,7 @@ async fn main() -> Result<(), Error> { log::debug!(target: "Server", "Loaded config file."); // Setup store - let store = Store::new(&config.data_directory, config.allow_scraping)?; + let store = Store::new(&config)?; let _ = GLOBALS.store.set(store); // TLS setup diff --git a/chorus-lib/src/config.rs b/chorus-lib/src/config.rs index 2126edb..a7a918c 100644 --- a/chorus-lib/src/config.rs +++ b/chorus-lib/src/config.rs @@ -20,6 +20,8 @@ pub struct FriendlyConfig { pub user_hex_keys: Vec, pub verify_events: bool, pub allow_scraping: bool, + pub allow_scrape_if_limited_to: u32, + pub allow_scrape_if_max_seconds: u64, pub max_subscriptions: usize, pub serve_ephemeral: bool, pub serve_relay_lists: bool, @@ -45,6 +47,8 @@ impl Default for FriendlyConfig { user_hex_keys: vec![], verify_events: true, allow_scraping: false, + allow_scrape_if_limited_to: 100, + allow_scrape_if_max_seconds: 3600, max_subscriptions: 32, serve_ephemeral: true, serve_relay_lists: true, @@ -72,6 +76,8 @@ impl FriendlyConfig { user_hex_keys, verify_events, allow_scraping, + allow_scrape_if_limited_to, + allow_scrape_if_max_seconds, max_subscriptions, serve_ephemeral, serve_relay_lists, @@ -115,6 +121,8 @@ impl FriendlyConfig { user_hex_keys, verify_events, allow_scraping, + allow_scrape_if_limited_to, + allow_scrape_if_max_seconds, max_subscriptions, serve_ephemeral, serve_relay_lists, @@ -142,6 +150,8 @@ pub struct Config { pub user_hex_keys: Vec, pub verify_events: bool, pub allow_scraping: bool, + pub allow_scrape_if_limited_to: u32, + pub allow_scrape_if_max_seconds: u64, pub max_subscriptions: usize, pub serve_ephemeral: bool, pub serve_relay_lists: bool, diff --git a/chorus-lib/src/store/mod.rs b/chorus-lib/src/store/mod.rs index ac401a2..323abc5 100644 --- a/chorus-lib/src/store/mod.rs +++ b/chorus-lib/src/store/mod.rs @@ -3,6 +3,7 @@ pub use event_store::EventStore; mod migrations; +use crate::config::Config; use crate::error::{ChorusError, Error}; use crate::ip::IpData; use crate::types::{Event, Filter, Id, Kind, Pubkey, Time}; @@ -29,11 +30,13 @@ pub struct Store { deleted_events: Database, Unit>, ip_data: Database, UnalignedSlice>, allow_scraping: bool, + allow_scrape_if_limited_to: u32, + allow_scrape_if_max_seconds: u64, } impl Store { /// Setup persistent storage - pub fn new(data_directory: &str, allow_scraping: bool) -> Result { + pub fn new(config: &Config) -> Result { let mut builder = EnvOpenOptions::new(); unsafe { builder.flags(EnvFlags::NO_TLS); @@ -41,7 +44,7 @@ impl Store { builder.max_dbs(32); builder.map_size(1048576 * 1024 * 24); // 24 GB - let dir = format!("{}/lmdb", data_directory); + let dir = format!("{}/lmdb", &config.data_directory); fs::create_dir_all(&dir)?; let env = match builder.open(&dir) { @@ -111,7 +114,7 @@ impl Store { txn.commit()?; - let event_map_file = format!("{}/event.map", data_directory); + let event_map_file = format!("{}/event.map", &config.data_directory); let events = EventStore::new(event_map_file)?; let store = Store { @@ -126,7 +129,9 @@ impl Store { deleted_offsets, deleted_events, ip_data, - allow_scraping, + allow_scraping: config.allow_scraping, + allow_scrape_if_limited_to: config.allow_scrape_if_limited_to, + allow_scrape_if_max_seconds: config.allow_scrape_if_max_seconds, }; // This is in migrations.rs @@ -465,8 +470,9 @@ impl Store { } else { // SCRAPE: let maxtime = filter.until().0.min(Time::now().0); - let allow = - self.allow_scraping || filter.limit() <= 100 || (maxtime - filter.since().0) < 3600; + let allow = self.allow_scraping || + filter.limit() <= self.allow_scrape_if_limited_to || + (maxtime - filter.since().0) < self.allow_scrape_if_max_seconds; if !allow { return Err(ChorusError::Scraper.into()); } diff --git a/contrib/chorus.toml b/contrib/chorus.toml index 398670b..b25bfd7 100644 --- a/contrib/chorus.toml +++ b/contrib/chorus.toml @@ -136,6 +136,28 @@ verify_events = true allow_scraping = false +# This is a u32 count of events indicating a filter `limit` value under which a scrape is +# allowed, irrespective of the `allow_scraping` setting. Such scrapes are not expensive due +# to the limit. +# +# See `allow_scraping` to learn the definition of a scrape. +# +# The default is 100. +# +allow_scrape_if_limited_to = 100 + + +# This is a u64 number of seconds indicating a filter time range under which a scrape is +# allowed, irrespective of the `allow_scraping` setting. Such scrapes are rarely expensive +# due to the short time period. +# +# See `allow_scraping` to learn the definition of a scrape. +# +# The default is 3600. +# +allow_scrape_if_max_seconds = 100 + + # This is an integer indicating the maximum number of subscriptions a connection can have open # at a given time. # diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 5667e88..d5f0e5f 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -121,6 +121,22 @@ The purpose of this setting is as a temporary setting that allows you to dump ev Default is false. +### allow_scrape_if_limited_to + +This is a u32 count of events indicating a filter `limit` value under which a scrape is allowed, irrespective of the `allow_scraping` setting. Such scrapes are not expensive due to the limit. + +See `allow_scraping` to learn the definition of a scrape. + +The default is 100. + +### allow_scrape_if_max_seconds + +This is a u64 number of seconds indicating a filter time range under which a scrape is allowed, irrespective of the `allow_scraping` setting. Such scrapes are rarely expensive due to the short time period. + +See `allow_scraping` to learn the definition of a scrape. + +The default is 3600. + ### max_subscriptions This is an integer indicating the maximum number of subscriptions a connection can have open at a given time. diff --git a/sample/sample.config.toml b/sample/sample.config.toml index 34e06cd..1f1f07d 100644 --- a/sample/sample.config.toml +++ b/sample/sample.config.toml @@ -14,6 +14,8 @@ user_hex_keys = [ ] verify_events = true allow_scraping = true +allow_scrape_if_limited_to = 100 +allow_scrape_if_max_seconds = 3600 max_subscriptions = 32 serve_ephemeral = true serve_relay_lists = true