nik/chorus
1
0
Fork 0
mirror of https://github.com/mikedilger/chorus.git synced 2026-05-03 06:51:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Modify Filter structure to allow 2^32 long filters (we only needed slightly more than 2^16)

Browse Source
This commit is contained in:
Mike Dilger 2024-02-20 10:54:23 +13:00
parent 4dbc738181
commit 254c0a9afc
2 changed files with 41 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
src/types/filter/json_filter.rs
View File

@ -1,3 +1,8 @@
use super::{
ARRAYS_OFFSET, ID_SIZE, KIND_SIZE, LIMIT_OFFSET, NUM_AUTHORS_OFFSET, NUM_IDS_OFFSET,
NUM_KINDS_OFFSET, PUBKEY_SIZE, SINCE_OFFSET, UNTIL_OFFSET,
};
use crate::error::{ChorusError, Error}; use crate::error::{ChorusError, Error};
use crate::types::parse::json_escape::json_unescape; use crate::types::parse::json_escape::json_unescape;
use crate::types::parse::json_parse::*; use crate::types::parse::json_parse::*;
@ -40,18 +45,18 @@ pub fn parse_json_filter(input: &[u8], output: &mut [u8]) -> Result<(usize, usiz
output, output,
0, 0,
&[ &[
0, 0, // length (we will fill it in later) 0, 0, 0, 0, // length (we will fill it in later)
0, 0, // 0 ids 0, 0, // 0 ids
0, 0, // 0 authors 0, 0, // 0 authors
0, 0, // 0 kinds 0, 0, // 0 kinds
0, 0, // padding
255, 255, 255, 255, // max limit 255, 255, 255, 255, // max limit
0, 0, 0, 0, // padding
0, 0, 0, 0, 0, 0, 0, 0, // since 1970 0, 0, 0, 0, 0, 0, 0, 0, // since 1970
255, 255, 255, 255, 255, 255, 255, 255, // until max unixtime 255, 255, 255, 255, 255, 255, 255, 255, // until max unixtime
], ],
)?; )?;
let mut end: usize = 32; let mut end: usize = ARRAYS_OFFSET;
// We just store the position of ids, authors, kinds, and tags // We just store the position of ids, authors, kinds, and tags
// and come back to parse them properly again at the end, // and come back to parse them properly again at the end,
@ -148,7 +153,7 @@ pub fn parse_json_filter(input: &[u8], output: &mut [u8]) -> Result<(usize, usiz
eat_colon_with_whitespace(input, &mut inpos)?; eat_colon_with_whitespace(input, &mut inpos)?;
let since = read_u64(input, &mut inpos)?; let since = read_u64(input, &mut inpos)?;
put(output, 16, since.to_ne_bytes().as_slice())?; put(output, SINCE_OFFSET, since.to_ne_bytes().as_slice())?;
found |= HAVE_SINCE; found |= HAVE_SINCE;
} else if inpos + 6 <= input.len() && &input[inpos..inpos + 6] == b"until\"" { } else if inpos + 6 <= input.len() && &input[inpos..inpos + 6] == b"until\"" {
@ -160,7 +165,7 @@ pub fn parse_json_filter(input: &[u8], output: &mut [u8]) -> Result<(usize, usiz
eat_colon_with_whitespace(input, &mut inpos)?; eat_colon_with_whitespace(input, &mut inpos)?;
let until = read_u64(input, &mut inpos)?; let until = read_u64(input, &mut inpos)?;
put(output, 24, until.to_ne_bytes().as_slice())?; put(output, UNTIL_OFFSET, until.to_ne_bytes().as_slice())?;
found |= HAVE_UNTIL; found |= HAVE_UNTIL;
} else if inpos + 6 <= input.len() && &input[inpos..inpos + 6] == b"limit\"" { } else if inpos + 6 <= input.len() && &input[inpos..inpos + 6] == b"limit\"" {
@ -173,7 +178,7 @@ pub fn parse_json_filter(input: &[u8], output: &mut [u8]) -> Result<(usize, usiz
eat_colon_with_whitespace(input, &mut inpos)?; eat_colon_with_whitespace(input, &mut inpos)?;
let limit = read_u64(input, &mut inpos)?; let limit = read_u64(input, &mut inpos)?;
let limit: u32 = limit as u32; let limit: u32 = limit as u32;
put(output, 8, limit.to_ne_bytes().as_slice())?; put(output, LIMIT_OFFSET, limit.to_ne_bytes().as_slice())?;
found |= HAVE_LIMIT; found |= HAVE_LIMIT;
} else if inpos + 3 <= input.len() } else if inpos + 3 <= input.len()
@ -219,11 +224,11 @@ pub fn parse_json_filter(input: &[u8], output: &mut [u8]) -> Result<(usize, usiz
} }
read_id(input, &mut inpos, &mut output[end..])?; read_id(input, &mut inpos, &mut output[end..])?;
num_ids += 1; num_ids += 1;
end += 32; end += ID_SIZE;
} }
// Write num_ids // Write num_ids
put(output, 2, num_ids.to_ne_bytes().as_slice())?; put(output, NUM_IDS_OFFSET, num_ids.to_ne_bytes().as_slice())?;
} }
// Copy authors // Copy authors
@ -237,11 +242,15 @@ pub fn parse_json_filter(input: &[u8], output: &mut [u8]) -> Result<(usize, usiz
} }
read_pubkey(input, &mut inpos, &mut output[end..])?; read_pubkey(input, &mut inpos, &mut output[end..])?;
num_authors += 1; num_authors += 1;
end += 32; end += PUBKEY_SIZE;
} }
// write num_authors // write num_authors
put(output, 4, num_authors.to_ne_bytes().as_slice())?; put(
output,
NUM_AUTHORS_OFFSET,
num_authors.to_ne_bytes().as_slice(),
)?;
} }
// Copy kinds // Copy kinds
@ -261,11 +270,11 @@ pub fn parse_json_filter(input: &[u8], output: &mut [u8]) -> Result<(usize, usiz
} }
put(output, end, (u as u16).to_ne_bytes().as_slice())?; put(output, end, (u as u16).to_ne_bytes().as_slice())?;
num_kinds += 1; num_kinds += 1;
end += 2; end += KIND_SIZE;
} }
// write num_kinds // write num_kinds
put(output, 6, num_kinds.to_ne_bytes().as_slice())?; put(output, NUM_KINDS_OFFSET, num_kinds.to_ne_bytes().as_slice())?;
} }
// Copy tags // Copy tags
@ -337,12 +346,12 @@ pub fn parse_json_filter(input: &[u8], output: &mut [u8]) -> Result<(usize, usiz
)?; )?;
} }
if end > 65535 { if end > u32::MAX as usize {
return Err(ChorusError::JsonBadFilter("Filter is too long", end).into()); return Err(ChorusError::JsonBadFilter("Filter is too long", end).into());
} }
// Write length of filter // Write length of filter
put(output, 0, (end as u16).to_ne_bytes().as_slice())?; put(output, 0, (end as u32).to_ne_bytes().as_slice())?;
Ok((inpos, end)) Ok((inpos, end))
} }
@ -363,12 +372,12 @@ mod test {
assert_eq!( assert_eq!(
&buffer[0..size], &buffer[0..size],
&[ &[
36, 0, // length 36, 0, 0, 0, // length
0, 0, // 0 ids 0, 0, // 0 ids
0, 0, // 0 authors 0, 0, // 0 authors
0, 0, // 0 kinds 0, 0, // 0 kinds
0, 0, // padding
255, 255, 255, 255, // max limit 255, 255, 255, 255, // max limit
0, 0, 0, 0, // padding
0, 0, 0, 0, 0, 0, 0, 0, // since 1970 0, 0, 0, 0, 0, 0, 0, 0, // since 1970
255, 255, 255, 255, 255, 255, 255, 255, // until max unixtime 255, 255, 255, 255, 255, 255, 255, 255, // until max unixtime
4, 0, 0, 0, // empty tags section 4, 0, 0, 0, // empty tags section
@ -388,12 +397,12 @@ mod test {
assert_eq!( assert_eq!(
&buffer[0..size], &buffer[0..size],
&[ &[
136, 0, // length 136, 0, 0, 0, // length
1, 0, // 0 ids 1, 0, // 0 ids
2, 0, // 0 authors 2, 0, // 0 authors
2, 0, // 0 kinds 2, 0, // 0 kinds
0, 0, // padding
10, 0, 0, 0, // max limit 10 10, 0, 0, 0, // max limit 10
0, 0, 0, 0, // padding
102, 232, 61, 100, 0, 0, 0, 0, // since 1681778790 102, 232, 61, 100, 0, 0, 0, 0, // since 1681778790
116, 156, 148, 101, 0, 0, 0, 0, // until 1704238196 116, 156, 148, 101, 0, 0, 0, 0, // until 1704238196
// First ID: // First ID:
@ -423,12 +432,12 @@ mod test {
assert_eq!( assert_eq!(
&buffer[0..size], &buffer[0..size],
&[ &[
209, 0, // length 209, 0, 0, 0, // length
1, 0, // 0 ids 1, 0, // 0 ids
2, 0, // 0 authors 2, 0, // 0 authors
2, 0, // 0 kinds 2, 0, // 0 kinds
0, 0, // padding
10, 0, 0, 0, // max limit 10 10, 0, 0, 0, // max limit 10
0, 0, 0, 0, // padding
102, 232, 61, 100, 0, 0, 0, 0, // since 1681778790 102, 232, 61, 100, 0, 0, 0, 0, // since 1681778790
116, 156, 148, 101, 0, 0, 0, 0, // until 1704238196 116, 156, 148, 101, 0, 0, 0, 0, // until 1704238196
// First ID: // First ID:

24
src/types/filter/mod.rs
View File

@ -6,12 +6,12 @@ mod json_filter;
use json_filter::parse_json_filter; use json_filter::parse_json_filter;
/* /*
* 0 [2 bytes] length of entire structure * 0 [4 bytes] length of entire structure
* 2 [2 bytes] num_ids * 4 [2 bytes] num_ids
* 4 [2 bytes] num_authors * 6 [2 bytes] num_authors
* 6 [2 bytes] num_kinds * 8 [2 bytes] num_kinds
* 8 [4 bytes] limit u32. Set to u32::max if limit was not set. * 10 [2 bytes] PADDING
* 12 [4 bytes] PADDING * 12 [4 bytes] limit u32. Set to u32::max if limit was not set.
* 16 [8 bytes] since u64. Set to 0 if since was not set. * 16 [8 bytes] since u64. Set to 0 if since was not set.
* 24 [8 bytes] until u64. Set to u64::max if until was not set. * 24 [8 bytes] until u64. Set to u64::max if until was not set.
* 32 [ID] array * 32 [ID] array
@ -20,10 +20,10 @@ use json_filter::parse_json_filter;
* [Tags] object starts at 32 + num_ids*32 + num_authors*32 * num_kinds*2 * [Tags] object starts at 32 + num_ids*32 + num_authors*32 * num_kinds*2
*/ */
const NUM_IDS_OFFSET: usize = 2; const NUM_IDS_OFFSET: usize = 4;
const NUM_AUTHORS_OFFSET: usize = 4; const NUM_AUTHORS_OFFSET: usize = 6;
const NUM_KINDS_OFFSET: usize = 6; const NUM_KINDS_OFFSET: usize = 8;
const LIMIT_OFFSET: usize = 8; const LIMIT_OFFSET: usize = 12;
const SINCE_OFFSET: usize = 16; const SINCE_OFFSET: usize = 16;
const UNTIL_OFFSET: usize = 24; const UNTIL_OFFSET: usize = 24;
const ARRAYS_OFFSET: usize = 32; const ARRAYS_OFFSET: usize = 32;
@ -463,12 +463,12 @@ mod test {
.unwrap(); .unwrap();
let data: Vec<u8> = vec![ let data: Vec<u8> = vec![
211, 0, // length of structure 211, 0, 0, 0, // length of structure
2, 0, // number of IDs 2, 0, // number of IDs
2, 0, // number of authors 2, 0, // number of authors
3, 0, // number of kinds 3, 0, // number of kinds
0, 0, // padding
255, 255, 255, 255, // limit 255, 255, 255, 255, // limit
0, 0, 0, 0, // padding
0xC1, 0xEB, 0x74, 0x65, 0, 0, 0, 0, // since 0xC1, 0xEB, 0x74, 0x65, 0, 0, 0, 0, // since
255, 255, 255, 255, 255, 255, 255, 255, // until 255, 255, 255, 255, 255, 255, 255, 255, // until
0x6b, 0x43, 0xbc, 0x2e, 0x37, 0x3b, 0x6d, 0x93, 0x30, 0xff, 0x57, 0x1f, 0x3f, 0x4e, 0x6b, 0x43, 0xbc, 0x2e, 0x37, 0x3b, 0x6d, 0x93, 0x30, 0xff, 0x57, 0x1f, 0x3f, 0x4e,