daemon: introduce an SQLite database

With a schema similar to revaultd, but a different interface.
This commit is contained in:
Antoine Poinsot 2022-07-22 11:12:38 +02:00
parent 989a2cf8fd
commit b0196ab529
No known key found for this signature in database
GPG Key ID: E13FC145CD3F4304
7 changed files with 600 additions and 3 deletions

97
Cargo.lock generated
View File

@ -17,6 +17,17 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "ahash"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
dependencies = [
"getrandom",
"once_cell",
"version_check",
]
[[package]]
name = "backtrace"
version = "0.3.66"
@ -97,6 +108,18 @@ dependencies = [
"winapi",
]
[[package]]
name = "fallible-iterator"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "fern"
version = "0.6.1"
@ -123,6 +146,24 @@ version = "0.26.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
dependencies = [
"ahash",
]
[[package]]
name = "hashlink"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d452c155cb93fecdfb02a73dd57b5d8e442c2063bd7aac72f1bc5e4263a43086"
dependencies = [
"hashbrown",
]
[[package]]
name = "itoa"
version = "1.0.2"
@ -135,6 +176,17 @@ version = "0.2.126"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"
[[package]]
name = "libsqlite3-sys"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8664486da51de68fbb3331d37c2a0fff4b60e988f284670a6a0833a8e6406ad"
dependencies = [
"cc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "log"
version = "0.4.17"
@ -159,6 +211,7 @@ dependencies = [
"fern",
"log",
"miniscript",
"rusqlite",
"serde",
"serde_json",
"toml",
@ -192,6 +245,18 @@ dependencies = [
"memchr",
]
[[package]]
name = "once_cell"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1"
[[package]]
name = "pkg-config"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
[[package]]
name = "proc-macro2"
version = "1.0.40"
@ -230,6 +295,20 @@ dependencies = [
"thiserror",
]
[[package]]
name = "rusqlite"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a"
dependencies = [
"bitflags",
"fallible-iterator",
"fallible-streaming-iterator",
"hashlink",
"libsqlite3-sys",
"smallvec",
]
[[package]]
name = "rustc-demangle"
version = "0.1.21"
@ -292,6 +371,12 @@ dependencies = [
"serde",
]
[[package]]
name = "smallvec"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1"
[[package]]
name = "syn"
version = "1.0.98"
@ -338,6 +423,18 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15c61ba63f9235225a22310255a29b806b907c9b8c964bcbd0a2c70f3f2deea7"
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "version_check"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"

View File

@ -37,3 +37,5 @@ fern = "0.6"
# stdlib does not have a programmatic interface yet
# to work with our custom panic hook.
backtrace = "0.3"
rusqlite = { version = "0.28", features = ["bundled", "unlock_notify"] }

6
src/database/mod.rs Normal file
View File

@ -0,0 +1,6 @@
///! Database interface for Minisafe.
///!
///! Record wallet metadata, spent and unspent coins, ongoing transactions.
pub mod sqlite;
pub trait DatabaseInterface {}

235
src/database/sqlite/mod.rs Normal file
View File

@ -0,0 +1,235 @@
///! Implementation of the database interface using SQLite.
///!
///! We use a bundled SQLite that is compiled with SQLITE_THREADSAFE. Sqlite.org states:
///! > Multi-thread. In this mode, SQLite can be safely used by multiple threads provided that
///! > no single database connection is used simultaneously in two or more threads.
///!
///! We leverage SQLite's `unlock_notify` feature to synchronize writes accross connection. More
///! about it at https://sqlite.org/unlock_notify.html.
mod schema;
mod utils;
use schema::{DbTip, DbWallet};
use utils::{create_fresh_db, db_query};
use std::{convert::TryInto, fmt, io, path};
use miniscript::{bitcoin, Descriptor, DescriptorPublicKey};
const DB_VERSION: i64 = 0;
#[derive(Debug)]
pub enum SqliteDbError {
FileCreation(io::Error),
FileNotFound(path::PathBuf),
UnsupportedVersion(i64),
InvalidNetwork(bitcoin::Network),
DescriptorMismatch(Descriptor<DescriptorPublicKey>),
Rusqlite(rusqlite::Error),
}
impl std::fmt::Display for SqliteDbError {
fn fmt(&self, f: &mut fmt::Formatter) -> std::fmt::Result {
match self {
SqliteDbError::FileCreation(e) => {
write!(f, "Error when create SQLite database file: '{}'", e)
}
SqliteDbError::FileNotFound(p) => {
write!(f, "SQLite database file not found at '{}'.", p.display())
}
SqliteDbError::UnsupportedVersion(v) => {
write!(f, "Unsupported database version '{}'.", v)
}
SqliteDbError::InvalidNetwork(net) => {
write!(f, "Database was created for network '{}'.", net)
}
SqliteDbError::DescriptorMismatch(desc) => {
write!(f, "Database descriptor mismatch: '{}'.", desc)
}
SqliteDbError::Rusqlite(e) => write!(f, "SQLite error: '{}'", e),
}
}
}
impl std::error::Error for SqliteDbError {}
impl From<io::Error> for SqliteDbError {
fn from(e: io::Error) -> Self {
SqliteDbError::FileCreation(e)
}
}
impl From<rusqlite::Error> for SqliteDbError {
fn from(e: rusqlite::Error) -> Self {
SqliteDbError::Rusqlite(e)
}
}
#[derive(Debug, Clone)]
pub struct FreshDbOptions {
pub bitcoind_network: bitcoin::Network,
pub main_descriptor: Descriptor<DescriptorPublicKey>,
}
#[derive(Debug, Clone)]
pub struct SqliteDb {
db_path: path::PathBuf,
}
impl SqliteDb {
/// Instanciate an SQLite database either from an existing database file or by creating a fresh
/// one.
pub fn new(
db_path: path::PathBuf,
fresh_options: Option<FreshDbOptions>,
) -> Result<SqliteDb, SqliteDbError> {
// Create the database if needed, and make sure the db file exists.
if let Some(options) = fresh_options {
create_fresh_db(&db_path, options)?;
log::info!("Created a fresh database at {}.", db_path.display());
}
if !db_path.exists() {
return Err(SqliteDbError::FileNotFound(db_path.to_path_buf()));
}
Ok(SqliteDb { db_path })
}
/// Get a new connection to the database.
pub fn connection(&self) -> Result<SqliteConn, SqliteDbError> {
let conn = rusqlite::Connection::open(&self.db_path)?;
conn.busy_timeout(std::time::Duration::from_secs(60))?;
Ok(SqliteConn { conn })
}
/// Perform startup sanity checks.
pub fn sanity_check(
&self,
bitcoind_network: bitcoin::Network,
main_descriptor: &Descriptor<DescriptorPublicKey>,
) -> Result<(), SqliteDbError> {
let mut conn = self.connection()?;
// Check if there database isn't from the future.
// NOTE: we'll do migration there eventually. Until then be strict on the check.
let db_version = conn.db_version();
if db_version != DB_VERSION {
return Err(SqliteDbError::UnsupportedVersion(db_version));
}
// The config and the db should be on the same network.
let db_tip = conn.db_tip();
if db_tip.network != bitcoind_network {
return Err(SqliteDbError::InvalidNetwork(db_tip.network));
}
// The config and db descriptors must match!
let db_wallet = conn.db_wallet();
if &db_wallet.main_descriptor != main_descriptor {
return Err(SqliteDbError::DescriptorMismatch(db_wallet.main_descriptor));
}
Ok(())
}
}
pub struct SqliteConn {
conn: rusqlite::Connection,
}
impl SqliteConn {
pub fn db_version(&mut self) -> i64 {
db_query(
&mut self.conn,
"SELECT version FROM version",
rusqlite::params![],
|row| {
let version: i64 = row.get(0)?;
Ok(version)
},
)
.expect("db must not fail")
.pop()
.expect("There is always a row in the version table")
}
/// Get the network tip.
pub fn db_tip(&mut self) -> DbTip {
db_query(
&mut self.conn,
"SELECT * FROM tip",
rusqlite::params![],
|row| row.try_into(),
)
.expect("Db must not fail")
.pop()
.expect("There is always a row in the tip table")
}
/// Get the information about the wallet.
pub fn db_wallet(&mut self) -> DbWallet {
db_query(
&mut self.conn,
"SELECT * FROM wallets",
rusqlite::params![],
|row| row.try_into(),
)
.expect("Db must not fail")
.pop()
.expect("There is always a row in the wallet table")
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::{env, fs, path, process, str::FromStr, thread};
#[test]
fn db_startup_sanity_checks() {
let tmp_dir = env::temp_dir().join(format!(
"minisafed-unit-tests-{}-{:?}",
process::id(),
thread::current().id()
));
fs::create_dir_all(&tmp_dir).unwrap();
let db_path: path::PathBuf = [tmp_dir.as_path(), path::Path::new("minisafed.sqlite3")]
.iter()
.collect();
assert!(SqliteDb::new(db_path.clone(), None)
.unwrap_err()
.to_string()
.contains("database file not found"));
let desc_str = "wsh(andor(pk(03b506a1dbe57b4bf48c95e0c7d417b87dd3b4349d290d2e7e9ba72c912652d80a),older(10000),pk(0295e7f5d12a2061f1fd2286cefec592dff656a19f55f4f01305d6aa56630880ce)))";
let desc = Descriptor::<DescriptorPublicKey>::from_str(desc_str).unwrap();
let options = FreshDbOptions {
bitcoind_network: bitcoin::Network::Bitcoin,
main_descriptor: desc.clone(),
};
let db = SqliteDb::new(db_path.clone(), Some(options.clone())).unwrap();
db.sanity_check(bitcoin::Network::Testnet, &desc)
.unwrap_err()
.to_string()
.contains("Database was created for network");
fs::remove_file(&db_path).unwrap();
let other_desc_str = "wsh(andor(pk(037a27a76ebf33594c785e4fa41607860a960bb5aa3039654297b05bff57e4f9a9),older(10000),pk(0295e7f5d12a2061f1fd2286cefec592dff656a19f55f4f01305d6aa56630880ce)))";
let other_desc = Descriptor::<DescriptorPublicKey>::from_str(other_desc_str).unwrap();
let db = SqliteDb::new(db_path.clone(), Some(options.clone())).unwrap();
db.sanity_check(bitcoin::Network::Bitcoin, &other_desc)
.unwrap_err()
.to_string()
.contains("Database descriptor mismatch");
fs::remove_file(&db_path).unwrap();
// TODO: version check
let db = SqliteDb::new(db_path.clone(), Some(options.clone())).unwrap();
db.sanity_check(bitcoin::Network::Bitcoin, &desc).unwrap();
let db = SqliteDb::new(db_path.clone(), None).unwrap();
db.sanity_check(bitcoin::Network::Bitcoin, &desc).unwrap();
fs::remove_dir_all(&tmp_dir).unwrap();
}
}

View File

@ -0,0 +1,90 @@
use std::{convert::TryFrom, str::FromStr};
use miniscript::{
bitcoin::{self, consensus::encode, util::bip32},
Descriptor, DescriptorPublicKey,
};
pub const SCHEMA: &str = "\
CREATE TABLE version (
version INTEGER NOT NULL
);
/* About the Bitcoin network. */
CREATE TABLE tip (
network TEXT NOT NULL,
blockheight INTEGER,
blockhash BLOB
);
/* This stores metadata about our wallet. We only support single wallet for
* now (and the foreseeable future).
*/
CREATE TABLE wallets (
id INTEGER PRIMARY KEY NOT NULL,
timestamp INTEGER NOT NULL,
main_descriptor TEXT NOT NULL,
deposit_derivation_index INTEGER NOT NULL
);
";
/// A row in the "tip" table.
#[derive(Clone, Debug)]
pub struct DbTip {
pub network: bitcoin::Network,
pub block_height: Option<i32>,
pub block_hash: Option<bitcoin::BlockHash>,
}
impl TryFrom<&rusqlite::Row<'_>> for DbTip {
type Error = rusqlite::Error;
fn try_from(row: &rusqlite::Row) -> Result<Self, Self::Error> {
let network: String = row.get(0)?;
let network = bitcoin::Network::from_str(&network)
.expect("Insane database: can't parse network string");
let block_height: Option<i32> = row.get(1)?;
let block_hash: Option<Vec<u8>> = row.get(2)?;
let block_hash: Option<bitcoin::BlockHash> = block_hash
.map(|h| encode::deserialize(&h).expect("Insane database: can't parse network string"));
Ok(DbTip {
network,
block_height,
block_hash,
})
}
}
/// A row in the "wallets" table.
#[derive(Clone, Debug)]
pub struct DbWallet {
pub id: i64,
pub timestamp: u32,
pub main_descriptor: Descriptor<DescriptorPublicKey>,
pub deposit_derivation_index: bip32::ChildNumber,
}
impl TryFrom<&rusqlite::Row<'_>> for DbWallet {
type Error = rusqlite::Error;
fn try_from(row: &rusqlite::Row) -> Result<Self, Self::Error> {
let id = row.get(0)?;
let timestamp = row.get(1)?;
let desc_str: String = row.get(2)?;
let main_descriptor = Descriptor::<DescriptorPublicKey>::from_str(&desc_str)
.expect("Insane database: can't parse deposit descriptor");
let der_idx: u32 = row.get(3)?;
let deposit_derivation_index = bip32::ChildNumber::from(der_idx);
Ok(DbWallet {
id,
timestamp,
main_descriptor,
deposit_derivation_index,
})
}
}

View File

@ -0,0 +1,94 @@
use crate::database::sqlite::{schema::SCHEMA, FreshDbOptions, SqliteDbError, DB_VERSION};
use std::{convert::TryInto, fs, path, time};
/// Perform a set of modifications to the database inside a single transaction
pub fn db_exec<F>(conn: &mut rusqlite::Connection, modifications: F) -> Result<(), rusqlite::Error>
where
F: FnOnce(&rusqlite::Transaction) -> rusqlite::Result<()>,
{
let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
modifications(&tx)?;
tx.commit()
}
/// Internal helper for queries boilerplate
pub fn db_query<P, F, T>(
conn: &mut rusqlite::Connection,
stmt_str: &str,
params: P,
f: F,
) -> Result<Vec<T>, rusqlite::Error>
where
P: IntoIterator + rusqlite::Params,
P::Item: rusqlite::ToSql,
F: FnMut(&rusqlite::Row<'_>) -> rusqlite::Result<T>,
{
// rustc says 'borrowed value does not live long enough'
let x = conn
.prepare(stmt_str)?
.query_map(params, f)?
.collect::<rusqlite::Result<Vec<T>>>();
x
}
// Sqlite supports up to i64, thus rusqlite prevents us from inserting u64's.
// We use this to panic rather than inserting a truncated integer into the database (as we'd have
// done by using `n as u32`).
fn timestamp_to_u32(n: u64) -> u32 {
n.try_into()
.expect("Is this the year 2106 yet? Misconfigured system clock.")
}
// Create the db file with RW permissions only for the user
pub fn create_db_file(db_path: &path::Path) -> Result<(), std::io::Error> {
let mut options = fs::OpenOptions::new();
let options = options.read(true).write(true).create_new(true);
#[cfg(unix)]
return {
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600).open(db_path)?;
Ok(())
};
#[cfg(not(unix))]
return {
// TODO: permissions for Windows...
options.open(db_path)?;
Ok(())
};
}
pub fn create_fresh_db(db_path: &path::Path, options: FreshDbOptions) -> Result<(), SqliteDbError> {
create_db_file(db_path)?;
let timestamp = time::SystemTime::now()
.duration_since(time::UNIX_EPOCH)
.map(|dur| timestamp_to_u32(dur.as_secs()))
.expect("System clock went backward the epoch?");
let mut conn = rusqlite::Connection::open(db_path)?;
db_exec(&mut conn, |tx| {
tx.execute_batch(SCHEMA)?;
tx.execute(
"INSERT INTO version (version) VALUES (?1)",
rusqlite::params![DB_VERSION],
)?;
tx.execute(
"INSERT INTO tip (network, blockheight, blockhash) VALUES (?1, NULL, NULL)",
rusqlite::params![options.bitcoind_network.to_string()],
)?;
tx.execute(
"INSERT INTO wallets (timestamp, main_descriptor, deposit_derivation_index) \
VALUES (?1, ?2, ?3)",
rusqlite::params![timestamp, options.main_descriptor.to_string(), 0,],
)?;
Ok(())
})?;
Ok(())
}

View File

@ -1,10 +1,17 @@
pub mod config;
mod database;
use crate::config::{config_folder_path, Config};
use crate::{
config::{config_folder_path, Config},
database::sqlite::{FreshDbOptions, SqliteDb, SqliteDbError},
};
use std::{error, fmt, fs, io, panic, path, process};
use std::{error, fmt, fs, io, path};
#[cfg(not(test))]
use std::{panic, process};
// A panic in any thread should stop the main thread, and print the panic.
#[cfg(not(test))]
fn setup_panic_hook() {
panic::set_hook(Box::new(move |panic_info| {
let file = panic_info
@ -39,6 +46,7 @@ pub enum StartupError {
Io(io::Error),
DefaultDataDirNotFound,
DatadirCreation(path::PathBuf, io::Error),
Database(SqliteDbError),
}
impl fmt::Display for StartupError {
@ -53,6 +61,7 @@ impl fmt::Display for StartupError {
f,
"Could not create data directory at '{}': '{}'", dir_path.display(), e
),
Self::Database(e) => write!(f, "Error initializing database: '{}'.", e)
}
}
}
@ -65,6 +74,12 @@ impl From<io::Error> for StartupError {
}
}
impl From<SqliteDbError> for StartupError {
fn from(e: SqliteDbError) -> Self {
Self::Database(e)
}
}
fn create_datadir(datadir_path: &path::Path) -> Result<(), StartupError> {
#[cfg(unix)]
return {
@ -95,6 +110,7 @@ impl DaemonHandle {
/// **Note**: we internally use threads, and set a panic hook. A downstream application must
/// not overwrite this panic hook.
pub fn start(config: Config) -> Result<Self, StartupError> {
#[cfg(not(test))]
setup_panic_hook();
// First, check the data directory
@ -102,11 +118,26 @@ impl DaemonHandle {
.data_dir
.unwrap_or(config_folder_path().ok_or(StartupError::DefaultDataDirNotFound)?);
data_dir.push(config.bitcoind_config.network.to_string());
if !data_dir.as_path().exists() {
let fresh_data_dir = !data_dir.as_path().exists();
if fresh_data_dir {
create_datadir(&data_dir)?;
log::info!("Created a new data directory at '{}'", data_dir.display());
}
let db_path: path::PathBuf = [data_dir.as_path(), path::Path::new("minisafed.sqlite3")]
.iter()
.collect();
let options = if fresh_data_dir {
Some(FreshDbOptions {
bitcoind_network: config.bitcoind_config.network,
main_descriptor: config.main_descriptor.clone(),
})
} else {
None
};
let db = SqliteDb::new(db_path, options)?;
db.sanity_check(config.bitcoind_config.network, &config.main_descriptor)?;
Ok(Self {})
}
@ -114,3 +145,45 @@ impl DaemonHandle {
/// Shut down the Minisafe daemon.
pub fn shutdown(self) {}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::BitcoindConfig;
use miniscript::{bitcoin, Descriptor, DescriptorPublicKey};
use std::{env, fs, net, path, process, str::FromStr, thread, time};
#[test]
fn daemon_startup() {
let tmp_dir = env::temp_dir().join(format!(
"minisafed-unit-tests-{}-{:?}",
process::id(),
thread::current().id()
));
fs::create_dir_all(&tmp_dir).unwrap();
let data_dir: path::PathBuf = [tmp_dir.as_path(), path::Path::new("datadir")]
.iter()
.collect();
let desc_str = "wsh(andor(pk(03b506a1dbe57b4bf48c95e0c7d417b87dd3b4349d290d2e7e9ba72c912652d80a),older(10000),pk(0295e7f5d12a2061f1fd2286cefec592dff656a19f55f4f01305d6aa56630880ce)))";
let desc = Descriptor::<DescriptorPublicKey>::from_str(desc_str).unwrap();
let config = Config {
bitcoind_config: BitcoindConfig {
network: bitcoin::Network::Bitcoin,
cookie_path: path::PathBuf::new(),
addr: net::SocketAddr::new(net::IpAddr::V4(net::Ipv4Addr::LOCALHOST), 0),
poll_interval_secs: time::Duration::from_secs(1),
},
data_dir: Some(data_dir.clone()),
daemon: None,
log_level: log::LevelFilter::Debug,
main_descriptor: desc,
};
let handle = DaemonHandle::start(config).unwrap();
handle.shutdown();
fs::remove_dir_all(&tmp_dir).unwrap();
}
}