Merge branch 'master' into pull/97

.travis.yml

@@ -5,7 +5,7 @@ sudo: required
 
 matrix:
   include:
-    - rust: 1.27.2
+    - rust: 1.34.0
     - rust: stable
     - rust: beta
     - rust: nightly
@@ -17,16 +17,28 @@ matrix:
       install:
         # For test coverage. In install step so that it can use cache.
         - cargo tarpaulin --version || RUSTFLAGS="--cfg procmacro2_semver_exempt" cargo install --force cargo-tarpaulin
-  allow_failures:
-    - rust: nightly
+        - cargo +nightly install cargo-fuzz
+
+    # no_std
+    - rust: stable
+      env: TARGET="--target thumbv6m-none-eabi" FEATURES="--no-default-features --features alloc"
+      install:
+        - rustup target add thumbv6m-none-eabi
 
 cache: cargo
 
+env:
+  # prevent cargo fuzz list from printing with color
+  - TERM=dumb
+
 script:
-  - cargo build --verbose
-  - cargo test --verbose
-  - cargo doc --verbose
+  - cargo build --verbose $TARGET --no-default-features 
+  - cargo build --verbose $TARGET $FEATURES
+  - 'if [[ -z "$TARGET" ]]; then cargo test --verbose; fi'
+  - 'if [[ -z "$TARGET" ]]; then cargo doc --verbose; fi'
   - 'if [[ "$TRAVIS_RUST_VERSION" = nightly ]]; then cargo bench --no-run; fi'
+  # run for just a second to confirm that it can build and run ok
+  - 'if [[ "$TRAVIS_RUST_VERSION" = nightly ]]; then cargo fuzz list | xargs -L 1 -I FUZZER cargo fuzz run FUZZER -- -max_total_time=1; fi'
 
 after_success: |
   if [[ "$TRAVIS_RUST_VERSION" = nightly ]]; then

Cargo.toml

@@ -1,25 +1,29 @@
 [package]
 name = "base64"
-version = "0.10.1"
+version = "0.11.0"
 authors = ["Alice Maz <alice@alicemaz.com>", "Marshall Pierce <marshall@mpierce.org>"]
 description = "encodes and decodes base64 as bytes or utf8"
-repository = "https://github.com/alicemaz/rust-base64"
-documentation = "https://github.com/alicemaz/rust-base64/blob/master/README.md"
+repository = "https://github.com/marshallpierce/rust-base64"
+documentation = "https://docs.rs/base64"
 readme = "README.md"
-keywords = ["base64", "utf8", "encode", "decode"]
+keywords = ["base64", "utf8", "encode", "decode", "no_std"]
 categories = ["encoding"]
 license = "MIT/Apache-2.0"
+edition = "2018"
 
 [[bench]]
 name = "benchmarks"
 harness = false
 
-[dependencies]
-byteorder = "1.2.6"
-
 [dev-dependencies]
-criterion = "0.2"
+criterion = "0.3"
 rand = "0.6.1"
+doc-comment = "0.3"
+
+[features]
+default = ["std"]
+alloc = []
+std = []
 
 [profile.bench]
 # Useful for better disassembly when using `perf record` and `perf annotate`

README.md

@@ -1,14 +1,16 @@
 [base64](https://crates.io/crates/base64)
 ===
 
-[![](https://img.shields.io/crates/v/base64.svg)](https://crates.io/crates/base64) [![Docs](https://docs.rs/base64/badge.svg)](https://docs.rs/base64) [![Build](https://travis-ci.org/alicemaz/rust-base64.svg?branch=master)](https://travis-ci.org/alicemaz/rust-base64) [![codecov](https://codecov.io/gh/alicemaz/rust-base64/branch/master/graph/badge.svg)](https://codecov.io/gh/alicemaz/rust-base64)
+[![](https://img.shields.io/crates/v/base64.svg)](https://crates.io/crates/base64) [![Docs](https://docs.rs/base64/badge.svg)](https://docs.rs/base64) [![Build](https://travis-ci.org/marshallpierce/rust-base64.svg?branch=master)](https://travis-ci.org/marshallpierce/rust-base64) [![codecov](https://codecov.io/gh/marshallpierce/rust-base64/branch/master/graph/badge.svg)](https://codecov.io/gh/marshallpierce/rust-base64) [![unsafe forbidden](https://img.shields.io/badge/unsafe-forbidden-success.svg)](https://github.com/rust-secure-code/safety-dance/)
 
 <a href="https://www.jetbrains.com/?from=rust-base64"><img src="/icon_CLion.svg" height="40px"/></a>
 
 Made with CLion. Thanks to JetBrains for supporting open source!
 
 It's base64. What more could anyone want?
 
+This library's goals are to be *correct* and *fast*. It's thoroughly tested and widely used. It exposes functionality at multiple levels of abstraction so you can choose the level of convenience vs performance that you want, e.g. `decode_config_slice` decodes into an existing `&mut [u8]` and is pretty fast (2.6GiB/s for a 3 KiB input), whereas `decode_config` allocates a new `Vec<u8>` and returns it, which might be more convenient in some cases, but is slower (although still fast enough for almost any purpose) at 2.1 GiB/s.
+
 Example
 ---
 
@@ -31,43 +33,48 @@ See the [docs](https://docs.rs/base64) for all the details.
 Rust version compatibility
 ---
 
-The minimum required Rust version is 1.27.2.
+The minimum required Rust version is 1.34.0.
 
 Developing
 ---
 
 Benchmarks are in `benches/`. Running them requires nightly rust, but `rustup` makes it easy:
 
-```
+```bash
 rustup run nightly cargo bench
 ```
 
 Decoding is aided by some pre-calculated tables, which are generated by:
 
-```
+```bash
 cargo run --example make_tables > src/tables.rs.tmp && mv src/tables.rs.tmp src/tables.rs
 ```
 
+no_std
+---
+
+This crate supports no_std. By default the crate targets std via the `std` feature. You can deactivate the `default-features` to target core instead. In that case you lose out on all the functionality revolving around `std::io`, `std::error::Error` and heap allocations. There is an additional `alloc` feature that you can activate to bring back the support for heap allocations.
+
 Profiling
 ---
 
 On Linux, you can use [perf](https://perf.wiki.kernel.org/index.php/Main_Page) for profiling. Then compile the benchmarks with `rustup nightly run cargo bench --no-run`.
 
 Run the benchmark binary with `perf` (shown here filtering to one particular benchmark, which will make the results easier to read). `perf` is only available to the root user on most systems as it fiddles with event counters in your CPU, so use `sudo`. We need to run the actual benchmark binary, hence the path into `target`. You can see the actual full path with `rustup run nightly cargo bench -v`; it will print out the commands it runs. If you use the exact path that `bench` outputs, make sure you get the one that's for the benchmarks, not the tests. You may also want to `cargo clean` so you have only one `benchmarks-` binary (they tend to accumulate).
 
-```
+```bash
 sudo perf record target/release/deps/benchmarks-* --bench decode_10mib_reuse
 ```
 
 Then analyze the results, again with perf:
 
-```
+```bash
 sudo perf annotate -l
 ```
 
 You'll see a bunch of interleaved rust source and assembly like this. The section with `lib.rs:327` is telling us that 4.02% of samples saw the `movzbl` aka bit shift as the active instruction. However, this percentage is not as exact as it seems due to a phenomenon called *skid*. Basically, a consequence of how fancy modern CPUs are is that this sort of instruction profiling is inherently inaccurate, especially in branch-heavy code.
 
-```
+```text
  lib.rs:322    0.70 :     10698:       mov    %rdi,%rax
     2.82 :        1069b:       shr    $0x38,%rax
          :                  if morsel == decode_tables::INVALID_VALUE {
@@ -93,10 +100,11 @@ Fuzzing
 
 This uses [cargo-fuzz](https://github.com/rust-fuzz/cargo-fuzz). See `fuzz/fuzzers` for the available fuzzing scripts. To run, use an invocation like these:
 
-```
+```bash
 cargo +nightly fuzz run roundtrip
 cargo +nightly fuzz run roundtrip_no_pad
 cargo +nightly fuzz run roundtrip_random_config -- -max_len=10240
+cargo +nightly fuzz run decode_random
 ```
 
 

RELEASE-NOTES.md

@@ -1,11 +1,15 @@
 # Next
 
-- TBD
+- A `Read` implementation (`DecoderReader`) to let users transparently decoded data from a b64 input source
+
+# 0.11.0
+- Minimum rust version 1.34.0
+- `no_std` is now supported via the two new features `alloc` and `std`.
 
 # 0.10.1
 
 - Minimum rust version 1.27.2
-- Fix bug in streaming encoding ([#90](https://github.com/alicemaz/rust-base64/pull/90)): if the underlying writer didn't write all the bytes given to it, the remaining bytes would not be retried later. See the docs on `EncoderWriter::write`.
+- Fix bug in streaming encoding ([#90](https://github.com/marshallpierce/rust-base64/pull/90)): if the underlying writer didn't write all the bytes given to it, the remaining bytes would not be retried later. See the docs on `EncoderWriter::write`.
 - Make it configurable whether or not to return an error when decoding detects excess trailing bits.
 
 # 0.10.0

benches/benchmarks.rs

@@ -11,7 +11,7 @@ use base64::{
 
 use criterion::{black_box, Bencher, Criterion, ParameterizedBenchmark, Throughput};
 use rand::{FromEntropy, Rng};
-use std::io::Write;
+use std::io::{self, Read, Write};
 
 const TEST_CONFIG: Config = base64::STANDARD;
 
@@ -52,6 +52,24 @@ fn do_decode_bench_slice(b: &mut Bencher, &size: &usize) {
     });
 }
 
+fn do_decode_bench_stream(b: &mut Bencher, &size: &usize) {
+    let mut v: Vec<u8> = Vec::with_capacity(size * 3 / 4);
+    fill(&mut v);
+    let encoded = encode(&v);
+
+    let mut buf = Vec::new();
+    buf.resize(size, 0);
+    buf.truncate(0);
+
+    b.iter(|| {
+        let mut cursor = io::Cursor::new(&encoded[..]);
+        let mut decoder = base64::read::DecoderReader::new(&mut cursor, TEST_CONFIG);
+        decoder.read_to_end(&mut buf).unwrap();
+        buf.clear();
+        black_box(&buf);
+    });
+}
+
 fn do_encode_bench(b: &mut Bencher, &size: &usize) {
     let mut v: Vec<u8> = Vec::with_capacity(size);
     fill(&mut v);
@@ -124,7 +142,7 @@ fn encode_benchmarks(byte_sizes: &[usize]) -> ParameterizedBenchmark<usize> {
     ParameterizedBenchmark::new("encode", do_encode_bench, byte_sizes.iter().cloned())
         .warm_up_time(std::time::Duration::from_millis(500))
         .measurement_time(std::time::Duration::from_secs(3))
-        .throughput(|s| Throughput::Bytes(*s as u32))
+        .throughput(|s| Throughput::Bytes(*s as u64))
         .with_function("encode_display", do_encode_bench_display)
         .with_function("encode_reuse_buf", do_encode_bench_reuse_buf)
         .with_function("encode_slice", do_encode_bench_slice)
@@ -135,9 +153,10 @@ fn decode_benchmarks(byte_sizes: &[usize]) -> ParameterizedBenchmark<usize> {
     ParameterizedBenchmark::new("decode", do_decode_bench, byte_sizes.iter().cloned())
         .warm_up_time(std::time::Duration::from_millis(500))
         .measurement_time(std::time::Duration::from_secs(3))
-        .throughput(|s| Throughput::Bytes(*s as u32))
+        .throughput(|s| Throughput::Bytes(*s as u64))
         .with_function("decode_reuse_buf", do_decode_bench_reuse_buf)
         .with_function("decode_slice", do_decode_bench_slice)
+        .with_function("decode_stream", do_decode_bench_stream)
 }
 
 fn bench(c: &mut Criterion) {

examples/make_tables.rs

@@ -58,9 +58,9 @@ fn main() {
 }
 
 fn print_encode_table(alphabet: &[u8], const_name: &str, indent_depth: usize) {
-    println!("#[cfg_attr(rustfmt, rustfmt_skip)]");
+    println!("#[rustfmt::skip]");
     println!(
-        "{:width$}pub const {}: &'static [u8; 64] = &[",
+        "{:width$}pub const {}: &[u8; 64] = &[",
         "",
         const_name,
         width = indent_depth
@@ -92,9 +92,9 @@ fn print_decode_table(alphabet: &[u8], const_name: &str, indent_depth: usize) {
         let _ = input_to_morsel.insert(*ascii_byte, morsel as u8);
     }
 
-    println!("#[cfg_attr(rustfmt, rustfmt_skip)]");
+    println!("#[rustfmt::skip]");
     println!(
-        "{:width$}pub const {}: &'static [u8; 256] = &[",
+        "{:width$}pub const {}: &[u8; 256] = &[",
         "",
         const_name,
         width = indent_depth

fuzz/fuzzers/roundtrip_no_pad.rs

@@ -3,7 +3,7 @@
 extern crate base64;
 
 fuzz_target!(|data: &[u8]| {
-    let config = base64::Config::new(base64::CharacterSet::Standard, false, false);
+    let config = base64::Config::new(base64::CharacterSet::Standard, false);
 
     let encoded = base64::encode_config(&data, config);
     let decoded = base64::decode_config(&encoded, config).unwrap();

src/chunked_encoder.rs

@@ -1,6 +1,12 @@
-use encode::{add_padding, encode_to_slice};
-use std::{cmp, str};
-use Config;
+use crate::{
+    encode::{add_padding, encode_to_slice},
+    Config,
+};
+#[cfg(any(feature = "alloc", feature = "std", test))]
+use alloc::string::String;
+use core::cmp;
+#[cfg(any(feature = "alloc", feature = "std", test))]
+use core::str;
 
 /// The output mechanism for ChunkedEncoder's encoded bytes.
 pub trait Sink {
@@ -78,16 +84,19 @@ fn max_input_length(encoded_buf_len: usize, config: Config) -> usize {
 }
 
 // A really simple sink that just appends to a string
+#[cfg(any(feature = "alloc", feature = "std", test))]
 pub(crate) struct StringSink<'a> {
     string: &'a mut String,
 }
 
+#[cfg(any(feature = "alloc", feature = "std", test))]
 impl<'a> StringSink<'a> {
     pub(crate) fn new(s: &mut String) -> StringSink {
         StringSink { string: s }
     }
 }
 
+#[cfg(any(feature = "alloc", feature = "std", test))]
 impl<'a> Sink for StringSink<'a> {
     type Error = ();
 
@@ -100,14 +109,13 @@ impl<'a> Sink for StringSink<'a> {
 
 #[cfg(test)]
 pub mod tests {
-    extern crate rand;
-
     use super::*;
-    use tests::random_config;
-    use *;
+    use crate::{encode_config_buf, tests::random_config, CharacterSet, STANDARD};
 
-    use self::rand::distributions::{Distribution, Uniform};
-    use self::rand::{FromEntropy, Rng};
+    use rand::{
+        distributions::{Distribution, Uniform},
+        FromEntropy, Rng,
+    };
 
     #[test]
     fn chunked_encode_empty() {
@@ -170,7 +178,7 @@ pub mod tests {
 
     #[test]
     fn max_input_length_cant_use_extra_single_encoded_byte() {
-        let config = Config::new(CharacterSet::Standard, false);
+        let config = Config::new(crate::CharacterSet::Standard, false);
         assert_eq!(300, max_input_length(401, config));
     }