Format

.gitignore

@@ -1,4 +1,5 @@
 .*.swp
+.idea
 tags
 target
 *.lock
@@ -12,3 +13,4 @@ words
 dict
 test
 months
+.vscode

.travis.yml

@@ -1,6 +1,5 @@
 language: rust
 rust:
-  - 1.24.0
   - stable
   - beta
   - nightly

Cargo.toml

@@ -1,21 +1,16 @@
 [package]
-name = "fst"
-version = "0.3.5"  #:version
-authors = ["Andrew Gallant <jamslam@gmail.com>"]
+name = "tantivy-fst"
+version = "0.1.0"
+authors = ["Paul Masurel<paul.masurel@gmail.com>"] # forked from Andrew Gallant's work
 description = """
-Use finite state transducers to compactly represents sets or maps of many
-strings (> 1 billion is possible).
+This is a fork from the
 """
-documentation = "https://docs.rs/fst"
-homepage = "https://github.com/BurntSushi/fst"
-repository = "https://github.com/BurntSushi/fst"
+documentation = "https://docs.rs/tantivy-fst"
+repository = "https://github.com/tantivy-search/fst"
 readme = "README.md"
-keywords = ["search", "information", "retrieval", "dictionary", "map"]
+keywords = ["donotuseme"]
 license = "Unlicense/MIT"
-
-[features]
-mmap = ["memmap"]
-default = ["mmap"]
+edition = "2018"
 
 [[bench]]
 name = "build"
@@ -31,15 +26,16 @@ bench = true
 
 [dependencies]
 byteorder = "1"
-memmap = { version = "0.6.0", optional = true }
+regex-syntax = "0.4"
+utf8-ranges = "1"
+levenshtein_automata="0.1"
 
 [dev-dependencies]
 fnv = "1.0.5"
-fst-levenshtein = { version = "0.2", path = "fst-levenshtein" }
-fst-regex = { version = "0.2", path = "fst-regex" }
-lazy_static = "0.2.8"
+lazy_static = "1.4"
 quickcheck = { version = "0.7", default-features = false }
 rand = "0.5"
+proptest = "0.9.4"
 
 [profile.release]
 debug = true

LICENSE-MIT

@@ -1,6 +1,7 @@
 The MIT License (MIT)
 
 Copyright (c) 2015 Andrew Gallant
+Copyright (c) 2019 Paul Masurel
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,90 +1,11 @@
-fst
-===
-This crate provides a fast implementation of ordered sets and maps using finite
-state machines. In particular, it makes use of finite state transducers to map
-keys to values as the machine is executed. Using finite state machines as data
-structures enables us to store keys in a compact format that is also easily
-searchable. For example, this crate leverages memory maps to make range queries
-very fast.
-
-Check out my blog post
-[Index 1,600,000,000 Keys with Automata and
-Rust](http://blog.burntsushi.net/transducers/)
-for extensive background, examples and experiments.
-
-[![Linux build status](https://travis-ci.org/BurntSushi/fst.svg?branch=master)](https://travis-ci.org/BurntSushi/fst)
-[![Windows build status](https://ci.appveyor.com/api/projects/status/github/BurntSushi/fst?svg=true)](https://ci.appveyor.com/project/BurntSushi/fst)
-[![](http://meritbadge.herokuapp.com/fst)](https://crates.io/crates/fst)
-
-Dual-licensed under MIT or the [UNLICENSE](http://unlicense.org).
-
-
-### Documentation
-
-[Full API documentation and examples.](http://burntsushi.net/rustdoc/fst/)
-
-The
-[`fst-regex`](https://docs.rs/fst-regex)
-and
-[`fst-levenshtein`](https://docs.rs/fst-levenshtein)
-crates provide regular expression matching and fuzzy searching on FSTs,
-respectively.
-
-
-### Installation
-
-Simply add a corresponding entry to your `Cargo.toml` dependency list:
+[![Build Status](https://travis-ci.org/tantivy-search/fst.svg?branch=master)](https://travis-ci.org/tantivy-search/fst)
 
-```toml,ignore
-[dependencies]
-fst = "0.3"
-```
-
-And add this to your crate root:
-
-```rust,ignore
-extern crate fst;
-```
-
-
-### Example
-
-This example demonstrates building a set in memory and executing a fuzzy query
-against it. You'll need `fst = "0.3"` and `fst-levenshtein = "0.2"` in your
-`Cargo.toml`.
-
-```rust
-extern crate fst;
-extern crate fst_levenshtein;
-
-use std::error::Error;
-use std::process;
-
-use fst::{IntoStreamer, Set};
-use fst_levenshtein::Levenshtein;
-
-fn try_main() -> Result<(), Box<Error>> {
-  // A convenient way to create sets in memory.
-  let keys = vec!["fa", "fo", "fob", "focus", "foo", "food", "foul"];
-  let set = Set::from_iter(keys)?;
-
-  // Build our fuzzy query.
-  let lev = Levenshtein::new("foo", 1)?;
+tantivy-fst
+===
 
-  // Apply our fuzzy query to the set we built.
-  let stream = set.search(lev).into_stream();
 
-  let keys = stream.into_strs()?;
-  assert_eq!(keys, vec!["fo", "fob", "foo", "food"]);
-  Ok(())
-}
+# WARNING: This is not the crate you are looking for.
 
-fn main() {
-  if let Err(err) = try_main() {
-    eprintln!("{}", err);
-    process::exit(1);
-  }
-}
-```
+This crate is a fork of the `fst` crate to better fit the need of tantivy. 
+You are probably looking for the  [fst](https://github.com/BurntSushi/fst) crate
 
-Check out the documentation for a lot more examples!

benches/build.rs

@@ -1,11 +1,11 @@
 #![feature(test)]
 
-extern crate fst;
+extern crate tantivy_fst;
 extern crate test;
 
 use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 
-use fst::raw::{Builder, Fst};
+use tantivy_fst::raw::{Builder, Fst};
 use test::Bencher;
 
 const WORDS: &'static str = include_str!("./../data/words-10000");
@@ -15,7 +15,10 @@ fn get_words() -> Vec<String> {
 }
 
 fn get_words_outputs() -> Vec<(String, u64)> {
-    WORDS.lines().map(|s| (s.to_owned(), s.len() as u64)).collect()
+    WORDS
+        .lines()
+        .map(|s| (s.to_owned(), s.len() as u64))
+        .collect()
 }
 
 #[bench]
@@ -27,7 +30,7 @@ fn build_fst_set(b: &mut Bencher) {
         for word in &words {
             bfst.add(word).unwrap();
         }
-        Fst::from_bytes(bfst.into_inner().unwrap()).unwrap();
+        Fst::new(bfst.into_inner().unwrap()).unwrap();
     });
 }
 
@@ -40,7 +43,7 @@ fn build_fst_map(b: &mut Bencher) {
         for &(ref word, len) in &words {
             bfst.insert(word, len).unwrap();
         }
-        Fst::from_bytes(bfst.into_inner().unwrap()).unwrap();
+        Fst::new(bfst.into_inner().unwrap()).unwrap();
     });
 }
 

benches/search.rs

@@ -1,8 +1,9 @@
 #![feature(test)]
 
 extern crate fnv;
-extern crate fst;
-#[macro_use] extern crate lazy_static;
+extern crate tantivy_fst;
+#[macro_use]
+extern crate lazy_static;
 extern crate test;
 
 const STR_WORDS: &'static str = include_str!("./../data/words-100000");
@@ -24,7 +25,7 @@ macro_rules! search {
             use std::hash::BuildHasherDefault;
 
             use fnv::FnvHasher;
-            use fst::raw::{Builder, Fst};
+            use tantivy_fst::raw::{Builder, Fst};
             use test::Bencher;
 
             #[bench]
@@ -36,7 +37,7 @@ macro_rules! search {
                             bfst.add(word).unwrap();
                         }
                         let bytes = bfst.into_inner().unwrap();
-                        Fst::from_bytes(bytes).unwrap()
+                        Fst::new(bytes).unwrap()
                     };
                 }
                 let mut i = 0;
@@ -46,13 +47,36 @@ macro_rules! search {
                 })
             }
 
+            #[bench]
+            fn fst_streams(b: &mut Bencher) {
+                use tantivy_fst::{IntoStreamer, Streamer};
+                lazy_static! {
+                    static ref FST: Fst = {
+                        let mut bfst = Builder::memory();
+                        for word in $keys.iter() {
+                            bfst.add(word).unwrap();
+                        }
+                        let bytes = bfst.into_inner().unwrap();
+                        Fst::new(bytes).unwrap()
+                    };
+                }
+                b.iter(|| {
+                    let start = 1000;
+                    let stop = 2000;
+                    let mut stream = FST.range().ge(&$keys[start]).lt(&$keys[stop]).into_stream();
+                    let mut count = 0;
+                    while stream.next().is_some() {
+                        count += 1;
+                    }
+                    assert_eq!(count, stop - start);
+                })
+            }
+
             #[bench]
             fn hash_fnv_contains(b: &mut Bencher) {
                 type Fnv = BuildHasherDefault<FnvHasher>;
                 lazy_static! {
-                    static ref SET: HashSet<String, Fnv> = {
-                        $keys.clone().into_iter().collect()
-                    };
+                    static ref SET: HashSet<String, Fnv> = { $keys.clone().into_iter().collect() };
                 }
                 let mut i = 0;
                 b.iter(|| {
@@ -64,9 +88,7 @@ macro_rules! search {
             #[bench]
             fn hash_sip_contains(b: &mut Bencher) {
                 lazy_static! {
-                    static ref SET: HashSet<String> = {
-                        $keys.clone().into_iter().collect()
-                    };
+                    static ref SET: HashSet<String> = { $keys.clone().into_iter().collect() };
                 }
                 let mut i = 0;
                 b.iter(|| {
@@ -78,9 +100,7 @@ macro_rules! search {
             #[bench]
             fn btree_contains(b: &mut Bencher) {
                 lazy_static! {
-                    static ref SET: BTreeSet<String> = {
-                        $keys.clone().into_iter().collect()
-                    };
+                    static ref SET: BTreeSet<String> = { $keys.clone().into_iter().collect() };
                 }
                 let mut i = 0;
                 b.iter(|| {
@@ -89,8 +109,8 @@ macro_rules! search {
                 })
             }
         }
-    }
+    };
 }
 
-search!(words, ::WORDS);
-search!(wiki_urls, ::WIKI_URLS);
+search!(words, crate::WORDS);
+search!(wiki_urls, crate::WIKI_URLS);

ci/script.sh

@@ -4,7 +4,6 @@ set -ex
 
 cargo doc --verbose
 cargo build --verbose
-cargo build --verbose --manifest-path fst-bin/Cargo.toml
 
 # If we're testing on an older version of Rust, then only check that we
 # can build the crate. This is because the dev dependencies might be updated
@@ -16,7 +15,3 @@ if [ "$TRAVIS_RUST_VERSION" = "1.20.0" ]; then
 fi
 
 cargo test --verbose
-cargo test --verbose  --lib --no-default-features
-if [ "$TRAVIS_RUST_VERSION" = "nightly" ]; then
-  cargo bench --verbose --no-run
-fi