Merge branch 'master' into think-lazysmp

Cargo.toml

@@ -11,16 +11,16 @@ futures = { version = "0.3", features = ["std"] }
 yaml-rust = "0.4"
 spin = "0.7"
 clap = "4.2"
-rayon = "1.5.3"
+rayon = "1.5"
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0.40"
-tokio = { version = "1.2" , features = ["full"] }
+serde_json = "1.0"
+tokio = { version = "1.36" , features = ["full"] }
 hyper = { version = "0.14", features = ["server"] }
-reqwest = { version = "0.11.12", features = ["json"], default-features = false }
-crc64 = "2.0.0"
+reqwest = { version = "0.11", features = ["json"], default-features = false }
+crc64 = "2.0"
 anyhow = "1.0"
 thiserror = "1.0"
-arrayvec = "0.7.4"
-num_cpus = "1.16.0"
+arrayvec = "0.7"
+num_cpus = "1.16"
 async-recursion = "1.0"
-dashmap = "5.5.3"
+dashmap = "5.5"

src/compression.rs

@@ -0,0 +1,100 @@
+#[cfg(test)]
+mod test;
+use crate::serializer::*;
+use clap::ArgMatches;
+use std::fs::File;
+use std::io::{BufRead, BufReader, BufWriter, Write};
+
+const PACKED_SCALE: i32 = 256;
+
+fn compress_word(data: i16) -> Vec<bool> {
+    if data == 0 {
+        vec![false, false]
+    } else {
+        let sign = data < 0;
+        let data_abs = (data as i32).abs();
+        if data_abs <= 8 {
+            let mut result = vec![false, true, false, sign];
+            let bits = data_abs - 1;
+            result.append(&mut encode_bits(bits, 3));
+            result
+        } else if data_abs <= 24 {
+            let mut result = vec![false, true, true, sign];
+            let bits = data_abs - 9;
+            result.append(&mut encode_bits(bits, 4));
+            result
+        } else if data_abs <= 56 {
+            let mut result = vec![true, false, false, sign];
+            let bits = data_abs - 25;
+            result.append(&mut encode_bits(bits, 5));
+            result
+        } else if data_abs <= 120 {
+            let mut result = vec![true, false, true, sign];
+            let bits = data_abs - 57;
+            result.append(&mut encode_bits(bits, 6));
+            result
+        } else if data_abs <= 248 {
+            let mut result = vec![true, true, false, sign];
+            let bits = data_abs - 121;
+            result.append(&mut encode_bits(bits, 7));
+            result
+        } else {
+            let mut result = vec![true, true, true, sign];
+            let bits = data_abs - 249;
+            result.append(&mut encode_bits(bits, 15));
+            result
+        }
+    }
+}
+
+fn compress(data: &[i16]) -> Vec<bool> {
+    let mut result_bits = Vec::new();
+    for &word in data {
+        result_bits.append(&mut compress_word(word));
+    }
+    result_bits
+}
+
+pub fn pack_weights(matches: &ArgMatches) {
+    let input_path = matches.get_one::<String>("INPUT").unwrap();
+    let output_path = matches.get_one::<String>("OUTPUT").unwrap();
+
+    let in_f = File::open(input_path).unwrap();
+    let mut reader = BufReader::new(in_f);
+
+    let out_f = File::create(output_path).unwrap();
+    let mut writer = BufWriter::new(out_f);
+
+    let mut input_line = String::new();
+    reader.read_line(&mut input_line).unwrap();
+    let num_weight = input_line.trim().parse().unwrap();
+    let scale = PACKED_SCALE as f64;
+
+    let mut weights = Vec::new();
+    for _i in 0..num_weight {
+        input_line.clear();
+        reader.read_line(&mut input_line).unwrap();
+        let weight = input_line.trim().parse::<f64>().unwrap();
+        let weight_scaled = (weight * scale).round() as i16;
+        weights.push(weight_scaled);
+    }
+
+    let orig_len = weights.len();
+    let mut compressed = compress(&weights);
+
+    writeln!(&mut writer, "{}", orig_len).unwrap();
+
+    while compressed.len() % 15 != 0 {
+        compressed.push(false);
+    }
+    for chunk in compressed.chunks(15) {
+        let mut bits = 0u32;
+        for (idx, &bit) in chunk.iter().enumerate() {
+            if bit {
+                bits |= 1 << idx;
+            }
+        }
+        let c = encode_utf16(bits).unwrap();
+        write!(&mut writer, "{}", c).unwrap();
+    }
+}

src/compression/test.rs

@@ -0,0 +1,70 @@
+extern crate test;
+use super::*;
+
+fn decompress_bits(bits: &[bool]) -> u16 {
+    let mut data = 0;
+    for &bit in bits {
+        data <<= 1;
+        if bit {
+            data |= 1;
+        }
+    }
+    data
+}
+
+fn decompress_word(data: &[bool]) -> (i32, usize) {
+    if data[0] {
+        let sign = data[3];
+        let (bits, length) = if data[1] {
+            if data[2] {
+                let bits = decompress_bits(&data[4..19]) as i32;
+                (bits + 249, 19)
+            } else {
+                let bits = decompress_bits(&data[4..11]) as i32;
+                (bits + 121, 11)
+            }
+        } else if data[2] {
+            let bits = decompress_bits(&data[4..10]) as i32;
+            (bits + 57, 10)
+        } else {
+            let bits = decompress_bits(&data[4..9]) as i32;
+            (bits + 25, 9)
+        };
+        let data = if sign { -bits } else { bits };
+        (data, length)
+    } else if data[1] {
+        let sign = data[3];
+        let (bits, length) = if data[2] {
+            let bits = decompress_bits(&data[4..8]) as i32;
+            (bits + 9, 8)
+        } else {
+            let bits = decompress_bits(&data[4..7]) as i32;
+            (bits + 1, 7)
+        };
+        let data = if sign { -bits } else { bits };
+        (data, length)
+    } else {
+        (0, 2)
+    }
+}
+
+fn decompress(data_bits: &[bool], length: usize) -> Vec<i16> {
+    let mut result = vec![0; length];
+    let mut offset = 0;
+    for r in result.iter_mut().take(length) {
+        let (word, consume) = decompress_word(&data_bits[offset..]);
+        *r = word as i16;
+        offset += consume;
+    }
+    result
+}
+
+#[test]
+fn test_compress_word() {
+    for val in i16::MIN..=i16::MAX {
+        let compressed = compress_word(val);
+        let (decompressed, consume) = decompress_word(&compressed);
+        assert_eq!(consume, compressed.len());
+        assert_eq!(val, decompressed as i16);
+    }
+}

src/main.rs

@@ -2,21 +2,23 @@
 #![feature(portable_simd)]
 #![feature(test)]
 mod book;
+mod compression;
 mod engine;
 mod play;
 mod record;
 mod remote;
-mod serialize;
+mod serializer;
 mod setup;
 mod sparse_mat;
 mod train;
 
 use crate::book::*;
+use crate::compression::*;
 use crate::engine::board::*;
 use crate::engine::search::*;
 use crate::play::*;
 use crate::remote::*;
-use crate::serialize::*;
+use crate::serializer::*;
 use crate::setup::*;
 use crate::train::*;
 use clap::{value_parser, Arg, ArgAction, ArgMatches, Command};

src/serialize.rs → src/serializer.rs

@@ -1,3 +1,5 @@
+#[cfg(test)]
+mod test;
 use crate::engine::bits::*;
 use crate::engine::board::*;
 use clap::ArgMatches;
@@ -35,9 +37,9 @@ pub fn encode_utf16(mut input: u32) -> Result<char, EncodeError> {
         return Err(EncodeError::OutOfRange);
     }
     input += 0x800;
-    if input >= 0x202A {
+    if input >= 0x2028 {
         // skip 0x202A - 0x202E
-        input += 5;
+        input += 7;
     }
     if input >= 0x2066 {
         // skip 0x2066 - 0x2069
@@ -47,7 +49,7 @@ pub fn encode_utf16(mut input: u32) -> Result<char, EncodeError> {
     Ok(char::from_u32(u32data).unwrap())
 }
 
-fn encode_bits(bits: i16, length: usize) -> Vec<bool> {
+pub fn encode_bits(bits: i32, length: usize) -> Vec<bool> {
     let mut result = Vec::new();
     for i in 0..length {
         let bit = (bits >> (length - 1 - i)) & 1;
@@ -56,54 +58,6 @@ fn encode_bits(bits: i16, length: usize) -> Vec<bool> {
     result
 }
 
-fn compress_word(data: i16) -> Vec<bool> {
-    if data == 0 {
-        vec![false, false]
-    } else {
-        let sign = data < 0;
-        let data_abs = data.abs();
-        if data_abs <= 8 {
-            let mut result = vec![false, true, false, sign];
-            let bits = data_abs - 1;
-            result.append(&mut encode_bits(bits, 3));
-            result
-        } else if data_abs <= 24 {
-            let mut result = vec![false, true, true, sign];
-            let bits = data_abs - 9;
-            result.append(&mut encode_bits(bits, 4));
-            result
-        } else if data_abs <= 56 {
-            let mut result = vec![true, false, false, sign];
-            let bits = data_abs - 25;
-            result.append(&mut encode_bits(bits, 5));
-            result
-        } else if data_abs <= 120 {
-            let mut result = vec![true, false, true, sign];
-            let bits = data_abs - 57;
-            result.append(&mut encode_bits(bits, 6));
-            result
-        } else if data_abs <= 248 {
-            let mut result = vec![true, true, false, sign];
-            let bits = data_abs - 121;
-            result.append(&mut encode_bits(bits, 7));
-            result
-        } else {
-            let mut result = vec![true, true, true, sign];
-            let bits = data_abs - 249;
-            result.append(&mut encode_bits(bits, 15));
-            result
-        }
-    }
-}
-
-pub fn compress(data: &[i16]) -> Vec<bool> {
-    let mut result_bits = Vec::new();
-    for &word in data {
-        result_bits.append(&mut compress_word(word));
-    }
-    result_bits
-}
-
 pub fn gen_last_table(matches: &ArgMatches) {
     let output_path = matches.get_one::<String>("OUTPUT").unwrap();
 

src/serializer/test.rs

@@ -0,0 +1,55 @@
+extern crate test;
+use super::*;
+
+pub fn decode_base64_impl(input: u8) -> Option<u8> {
+    if input == 43 {
+        // +
+        Some(62)
+    } else if input == 47 {
+        // /
+        Some(63)
+    } else if (48..=57).contains(&input) {
+        // 0-9
+        Some(input + 4)
+    } else if (65..=90).contains(&input) {
+        // A-Z
+        Some(input - 65)
+    } else if (97..=122).contains(&input) {
+        // a-z
+        Some(input - 71)
+    } else {
+        None
+    }
+}
+
+// Decode UTF16-encoded data
+
+pub fn decode_base_utf16(input: &[char]) -> Vec<bool> {
+    let mut output = Vec::with_capacity(input.len() * 15);
+    for &c in input {
+        let bits = c as u32;
+        let bits = if bits > 0x2069 { bits - 4 } else { bits };
+        let bits = if bits > 0x202e { bits - 7 } else { bits };
+        let bits = bits - 0x800;
+        for idx in 0..15 {
+            output.push((bits >> idx) & 1 == 1);
+        }
+    }
+    output
+}
+
+#[test]
+fn test_encode_utf16() {
+    const ENCODE_MAX: u32 = (1 << 15) - 1;
+    for val in 0..=ENCODE_MAX {
+        let encoded = encode_utf16(val).expect("Failed to encode");
+        let decoded = decode_base_utf16(&[encoded]);
+        let mut bits = 0u32;
+        for (idx, &bit) in decoded.iter().enumerate() {
+            if bit {
+                bits |= 1 << idx;
+            }
+        }
+        assert_eq!(val, bits);
+    }
+}