Merge pull request #5 from Vagabond/adt/more-improvements

Further improvements

Author: Elzor

crates/bloom/Cargo.lock

@@ -11,3 +11,4 @@ crate-type = ["dylib"]
 [dependencies]
 rustler = ">=0.13.0"
 bloomfilter = "1"
+siphasher = "*"

crates/bloom/Cargo.toml

@@ -1,12 +1,16 @@
 extern crate bloomfilter;
 #[macro_use]
 extern crate rustler;
+extern crate siphasher;
 
 use bloomfilter::Bloom;
-use rustler::{Encoder, Env, NifResult, Term, OwnedBinary, Binary};
 use rustler::resource::ResourceArc;
-use std::sync::RwLock;
+use rustler::{Binary, Encoder, Env, NifResult, OwnedBinary, Term};
+use siphasher::sip::SipHasher13;
+use std::hash::Hash;
+use std::hash::Hasher;
 use std::io::Write;
+use std::sync::RwLock;
 
 mod atoms {
     rustler_atoms! {
@@ -15,7 +19,7 @@ mod atoms {
 }
 
 struct FilterResource {
-    filter: RwLock<Bloom <Vec<u8>>>
+    filter: RwLock<Bloom<[u8]>>,
 }
 
 rustler_export_nifs!(
@@ -26,7 +30,8 @@ rustler_export_nifs!(
         ("serialize", 1, serialize),
         ("deserialize", 7, deserialize),
         ("set", 2, set),
-        ("check", 2, check),
+        ("check_nif", 2, check),
+        ("check_nif", 8, check_ro),
         ("check_and_set", 2, check_and_set),
         ("clear", 1, clear),
     ],
@@ -43,9 +48,7 @@ fn new<'a>(env: Env<'a>, args: &[Term<'a>]) -> NifResult<Term<'a>> {
     let items_count: i64 = args[1].decode()?;
 
     let resource = ResourceArc::new(FilterResource {
-        filter: RwLock::new(
-            Bloom::new(bitmap_size as usize, items_count as usize)
-        )
+        filter: RwLock::new(Bloom::new(bitmap_size as usize, items_count as usize)),
     });
 
     Ok((atoms::ok(), resource.encode(env)).encode(env))
@@ -56,15 +59,12 @@ fn new_for_fp_rate<'a>(env: Env<'a>, args: &[Term<'a>]) -> NifResult<Term<'a>> {
     let fp_p: f64 = args[1].decode()?;
 
     let resource = ResourceArc::new(FilterResource {
-        filter: RwLock::new(
-            Bloom::new_for_fp_rate(items_count as usize, fp_p)
-        )
+        filter: RwLock::new(Bloom::new_for_fp_rate(items_count as usize, fp_p)),
     });
 
     Ok((atoms::ok(), resource).encode(env))
 }
 
-
 fn serialize<'a>(env: Env<'a>, args: &[Term<'a>]) -> NifResult<Term<'a>> {
     let resource: ResourceArc<FilterResource> = args[0].decode()?;
 
@@ -74,11 +74,17 @@ fn serialize<'a>(env: Env<'a>, args: &[Term<'a>]) -> NifResult<Term<'a>> {
     let mut binary = OwnedBinary::new(bitmap.len()).unwrap();
     binary.as_mut_slice().write_all(&bitmap).unwrap();
 
-    Ok((atoms::ok(), (Binary::from_owned(binary, env),
-                      filter.number_of_bits(),
-                      filter.number_of_hash_functions(),
-                      sips[0],
-                      sips[1])).encode(env))
+    Ok((
+        atoms::ok(),
+        (
+            Binary::from_owned(binary, env),
+            filter.number_of_bits(),
+            filter.number_of_hash_functions(),
+            sips[0],
+            sips[1],
+        ),
+    )
+        .encode(env))
 }
 
 fn deserialize<'a>(env: Env<'a>, args: &[Term<'a>]) -> NifResult<Term<'a>> {
@@ -96,15 +102,19 @@ fn deserialize<'a>(env: Env<'a>, args: &[Term<'a>]) -> NifResult<Term<'a>> {
             num_bits,
             num_funs,
             [(sip00, sip01), (sip10, sip11)],
-        ))
+        )),
     });
 
     Ok((atoms::ok(), resource).encode(env))
 }
 
 fn set<'a>(env: Env<'a>, args: &[Term<'a>]) -> NifResult<Term<'a>> {
     let resource: ResourceArc<FilterResource> = args[0].decode()?;
-    let key: Vec<u8> = args[1].decode()?;
+    let key: Binary = if args[1].is_binary() {
+        args[1].decode()?
+    } else {
+        Binary::from_owned(args[1].to_binary(), env)
+    };
 
     let mut filter = resource.filter.write().unwrap();
     (*filter).set(&key);
@@ -114,16 +124,72 @@ fn set<'a>(env: Env<'a>, args: &[Term<'a>]) -> NifResult<Term<'a>> {
 
 fn check<'a>(env: Env<'a>, args: &[Term<'a>]) -> NifResult<Term<'a>> {
     let resource: ResourceArc<FilterResource> = args[0].decode()?;
-    let key: Vec<u8> = args[1].decode()?;
+    let key: Binary = if args[1].is_binary() {
+        args[1].decode()?
+    } else {
+        Binary::from_owned(args[1].to_binary(), env)
+    };
 
     let filter = resource.filter.read().unwrap();
 
     Ok(filter.check(&key).encode(env))
 }
 
+// check a serialized bloom for key membership without fully deserializing the bloom
+// specifically we want to avoid the very slow bitvec deserialization and simply compute
+// the hash keys manually and check them inside the Erlang binary by hand
+// for a 50mb bloom, this improves checking a serialized bloom from 25 seconds to 35 microseconds
+fn check_ro<'a>(env: Env<'a>, args: &[Term<'a>]) -> NifResult<Term<'a>> {
+    let bitmap: Binary = args[0].decode()?;
+    let num_bits: u64 = args[1].decode()?;
+    let num_funs: u32 = args[2].decode()?;
+    let sip00: u64 = args[3].decode()?;
+    let sip01: u64 = args[4].decode()?;
+    let sip10: u64 = args[5].decode()?;
+    let sip11: u64 = args[6].decode()?;
+    let key: Binary = if args[7].is_binary() {
+        args[7].decode()?
+    } else {
+        Binary::from_owned(args[7].to_binary(), env)
+    };
+
+    let sips = [
+        SipHasher13::new_with_keys(sip00, sip01),
+        SipHasher13::new_with_keys(sip10, sip11),
+    ];
+
+    let mut hashes = [0u64, 0u64];
+    for k_i in 0..num_funs {
+        let bit_offset = (bloom_hash(&mut hashes, &key, k_i, &sips) % num_bits) as usize;
+        let byte_offset = bit_offset / 8;
+        let bit = 7 - (bit_offset % 8);
+        if (bitmap[byte_offset] >> bit) & 1 != 1 {
+            return Ok(false.encode(env));
+        }
+    }
+    Ok(true.encode(env))
+}
+
+// helper for check_ro, extracted from the bloom crate source code
+fn bloom_hash(hashes: &mut [u64; 2], item: &[u8], k_i: u32, sips: &[SipHasher13; 2]) -> u64 {
+    if k_i < 2 {
+        let mut sip = sips[k_i as usize];
+        item.hash(&mut sip);
+        let hash = sip.finish();
+        hashes[k_i as usize] = hash;
+        hash
+    } else {
+        hashes[0].wrapping_add((u64::from(k_i)).wrapping_mul(hashes[1]) % 0xffff_ffff_ffff_ffc5)
+    }
+}
+
 fn check_and_set<'a>(env: Env<'a>, args: &[Term<'a>]) -> NifResult<Term<'a>> {
     let resource: ResourceArc<FilterResource> = args[0].decode()?;
-    let key: Vec<u8> = args[1].decode()?;
+    let key: Binary = if args[1].is_binary() {
+        args[1].decode()?
+    } else {
+        Binary::from_owned(args[1].to_binary(), env)
+    };
 
     let mut filter = resource.filter.write().unwrap();
 

crates/bloom/src/lib.rs

@@ -8,6 +8,8 @@
          serialize/1,
          deserialize/1,
          deserialize/7,
+         to_bin/1,
+         from_bin/1,
          set/2,
          check/2,
          check_and_set/2,
@@ -18,6 +20,9 @@
 -export([load/0]).
 -on_load(load/0).
 
+%% rev this if the internal structure of the bloom filter changes
+-define(ERBLOOM_VERSION1, 1).
+
 -type sip_keys() :: {non_neg_integer(), non_neg_integer()}.
 
 -type serialized_bloom() :: {Bitmap :: binary(), NumBits :: pos_integer(), NumFuns :: pos_integer(), sip_keys(), sip_keys()}.
@@ -36,11 +41,26 @@ new(_BitmapSize, _ItemsCount) ->
 new_for_fp_rate(_ItemsCount, _FP_Rate) ->
     not_loaded(?LINE).
 
-%% @doc Serialize a bloom filter to Erlang terms.
+%% @doc Serialize a bloom filter to Erlang terms. `check/2' can be used against this serialized form efficently.
 -spec serialize(Bloom :: bloom()) -> {ok, serialized_bloom()}.
 serialize(_Ref) ->
     not_loaded(?LINE).
 
+%% @doc Serialize a bloom filter and encode it as a versioned binary. `check/2' can be used against this binary form efficently.
+-spec to_bin(bloom()) -> binary().
+to_bin(Ref) ->
+    {ok, {Bitmap,NumBits,NumFuns,{Sv00,Sv01},{Sv10,Sv11}}} = serialize(Ref),
+    <<?ERBLOOM_VERSION1:8/integer, NumBits:64/integer-unsigned-little, NumFuns:32/integer-unsigned-little,
+      Sv00:64/integer-unsigned-little, Sv01:64/integer-unsigned-little,
+      Sv10:64/integer-unsigned-little, Sv11:64/integer-unsigned-little, Bitmap/binary>>.
+
+%% @doc Deserialize a versioned binary into a bloom filter reference.
+-spec from_bin(binary()) -> bloom().
+from_bin(<<?ERBLOOM_VERSION1:8/integer, NumBits:64/integer-unsigned-little, NumFuns:32/integer-unsigned-little,
+      Sv00:64/integer-unsigned-little, Sv01:64/integer-unsigned-little,
+      Sv10:64/integer-unsigned-little, Sv11:64/integer-unsigned-little, Bitmap/binary>>) ->
+    deserialize(Bitmap, NumBits, NumFuns, Sv00, Sv01, Sv10, Sv11).
+
 %% @doc Deserialize a previously serialized bloom filter back into a bloom filter reference.
 -spec deserialize(serialized_bloom()) -> {ok, bloom()}.
 deserialize({Bitmap,NumBits,NumFuns,{Sv00,Sv01},{Sv10,Sv11}}) ->
@@ -56,9 +76,18 @@ set(_Ref, _Key) ->
     not_loaded(?LINE).
 
 %% @doc Check for the presence of `Key' in `Bloom'.
--spec check(Bloom :: bloom(), Key :: term()) -> boolean().
-check(_Ref, _Key) ->
-    not_loaded(?LINE).
+%% Serialized and binary encoded bloom filters can be used with this
+%% function when you wish to check for the key and do not need to use set
+%% (eg. a static bloom filter stored in a database).
+-spec check(bloom() | serialized_bloom() | binary(), term()) -> boolean().
+check(Bloom, Key) when is_reference(Bloom) ->
+    check_nif(Bloom, Key);
+check(<<?ERBLOOM_VERSION1:8/integer, NumBits:64/integer-unsigned-little, NumFuns:32/integer-unsigned-little,
+      Sv00:64/integer-unsigned-little, Sv01:64/integer-unsigned-little,
+      Sv10:64/integer-unsigned-little, Sv11:64/integer-unsigned-little, Bitmap/binary>>, Key) ->
+    check_nif(Bitmap, NumBits, NumFuns, Sv00, Sv01, Sv10, Sv11, Key);
+check({Bitmap,NumBits,NumFuns,{Sv00,Sv01},{Sv10,Sv11}}, Key) ->
+    check_nif(Bitmap, NumBits, NumFuns, Sv00, Sv01, Sv10, Sv11, Key).
 
 %% @doc Record the presence of `Key' in `Bloom' and return whether it was present before.
 -spec check_and_set(Bloom :: bloom(), Key :: term()) -> boolean().
@@ -70,6 +99,12 @@ check_and_set(_Ref, _Key) ->
 clear(_Ref) ->
     not_loaded(?LINE).
 
+check_nif(_Ref, _Key) ->
+    not_loaded(?LINE).
+
+check_nif(_Bitmap, _NumBits, _NumFuns, _Sv00, _Sv01, _Sv10, _Sv11, _Key) ->
+    not_loaded(?LINE).
+
 %% @private
 load() ->
     erlang:load_nif(filename:join(priv(), "libbloom"), none).

src/bloom.erl

@@ -2,7 +2,6 @@
  [{description, "Fast Bloom Filter"},
   {vsn, "0.1.0"},
   {registered, []},
-  {mod, { erbloom_app, []}},
   {applications,
    [kernel,
     stdlib