Add BigInt support and Unicode handling (#9)

Co-authored-by: Sindre Sorhus <sindresorhus@gmail.com>
Co-authored-by: Dustin Hagemeier <dustin@commit.international>

Author: 3 people

index.d.ts

@@ -9,15 +9,12 @@ declare const fnv1a: {
 	import fnv1a = require('@sindresorhus/fnv1a');
 
 	fnv1a('🦄🌈');
-	//=> 582881315
+	//=> 2868248295
 	```
 	*/
 	(string: string): number;
 
-	// TODO: remove this in the next major version, refactor the whole definition to:
-	// declare function fnv1a(string: string): number;
-	// export = fnv1a;
-	default: typeof fnv1a;
+	bigInt(string: string): BigInt;
 };
 
 export = fnv1a;

index.js

@@ -1,20 +1,81 @@
 'use strict';
-const OFFSET_BASIS_32 = 2166136261;
 
-const fnv1a = string => {
-	let hash = OFFSET_BASIS_32;
+// FNV_PRIMES and FNV_OFFSETS from
+// http://www.isthe.com/chongo/tech/comp/fnv/index.html#FNV-param
+//
+// Defining these as strings instead of BigInt literals avoids syntax errors on
+// legacy platforms.
 
-	for (let i = 0; i < string.length; i++) {
-		hash ^= string.charCodeAt(i);
+const FNV_PRIMES = {
+	32: '16777619',
+	64: '1099511628211',
+	128: '309485009821345068724781371',
+	256: '374144419156711147060143317175368453031918731002211',
+	512: '35835915874844867368919076489095108449946327955754392558399825615420669938882575126094039892345713852759',
+	1024: '5016456510113118655434598811035278955030765345404790744303017523831112055108147451509157692220295382716162651878526895249385292291816524375083746691371804094271873160484737966720260389217684476157468082573'
+};
+
+const FNV_OFFSETS = {
+	32: '2166136261',
+	64: '14695981039346656037',
+	128: '144066263297769815596495629667062367629',
+	256: '100029257958052580907070968620625704837092796014241193945225284501741471925557',
+	512: '9659303129496669498009435400716310466090418745672637896108374329434462657994582932197716438449813051892206539805784495328239340083876191928701583869517785',
+	1024: '14197795064947621068722070641403218320880622795441933960878474914617582723252296732303717722150864096521202355549365628174669108571814760471015076148029755969804077320157692458563003215304957150157403644460363550505412711285966361610267868082893823963790439336411086884584107735010676915'
+};
+
+// Legacy implementation for 32-bit + Number types, older systems that don't
+// support BigInt
+function fnv1a(str) {
+	// Handle unicode code points > 0x7f
+	let hash = Number(FNV_OFFSETS[32]);
+	let unicoded = false;
+	for (let i = 0; i < str.length; i++) {
+		let v = str.charCodeAt(i);
+		// Non-ASCII char triggers unicode escape logic
+		if (v > 0x7F && !unicoded) {
+			str = unescape(encodeURIComponent(str));
+			v = str.charCodeAt(i);
+			unicoded = true;
+		}
 
-		// 32-bit FNV prime: 2**24 + 2**8 + 0x93 = 16777619
-		// Using bitshift for accuracy and performance. Numbers in JS suck.
+		hash ^= v;
 		hash += (hash << 1) + (hash << 4) + (hash << 7) + (hash << 8) + (hash << 24);
 	}
 
 	return hash >>> 0;
-};
+}
+
+function bigInt(str, {size} = {size: 32}) {
+	if (typeof (BigInt) === 'undefined') {
+		throw new TypeError('BigInt is not supported');
+	}
+
+	if (!FNV_PRIMES[size]) {
+		throw new Error('`size` must be one of 32, 64, 128, 256, 512, or 1024');
+	}
+
+	let hash = BigInt(FNV_OFFSETS[size]);
+	const prime = BigInt(FNV_PRIMES[size]);
+
+	// Handle unicode code points > 0x7f
+	let unicoded = false;
+	for (let i = 0; i < str.length; i++) {
+		let v = str.charCodeAt(i);
+		// Non-ASCII char triggers unicode escape logic
+		if (v > 0x7F && !unicoded) {
+			str = unescape(encodeURIComponent(str));
+			v = str.charCodeAt(i);
+			unicoded = true;
+		}
+
+		hash ^= BigInt(v);
+		hash = BigInt.asUintN(size, hash * prime);
+	}
+
+	return hash;
+}
 
 module.exports = fnv1a;
-// TODO: remove this in the next major version, refactor the whole definition to:
+module.exports.bigInt = bigInt;
 module.exports.default = fnv1a;

package.json

@@ -36,8 +36,13 @@
 		"vo"
 	],
 	"devDependencies": {
-		"ava": "^1.4.1",
-		"tsd": "^0.7.1",
-		"xo": "^0.24.0"
+		"ava": "2.4.0",
+		"tsd": "0.11.0",
+		"xo": "0.25.3"
+	},
+	"xo": {
+		"globals": [
+			"BigInt"
+		]
 	}
 }

readme.md

@@ -13,18 +13,31 @@ FNV hashes are designed to be fast while maintaining a low collision rate. The F
 $ npm install @sindresorhus/fnv1a
 ```
 
-
 ## Usage
 
+### fnv1a(string)
+
 ```js
 const fnv1a = require('@sindresorhus/fnv1a');
 
 fnv1a('🦄🌈');
-//=> 582881315
+//=> 2868248295
 ```
 
-It returns the hash as a positive integer.
+It returns the hash as a 32-bit positive Number.
+
+### fnv1a.bigInt(string, [{size}])
+
+On systems that support BigInt, this method may be called to generate larger hashes. This method throws if `BigInt` is not available, however.
+
+```js
+const fnv1a = require('@sindresorhus/fnv1a');
+
+fnv1a.bigInt('hello world', {size: 128});
+//=> 143667438548887148232425432707801491127n
+```
 
+It returns the hash as a `size`-bit positive BigInt.
 
 ## Related
 

test.js

@@ -1,10 +1,9 @@
 import test from 'ava';
-import fnv1a from '.';
+import fnv1a, {bigInt} from '.';
 
-test('main', t => {
+test('default', t => {
+	// Test 32-bit for various strings
 	t.is(fnv1a(''), 2166136261);
-	t.is(fnv1a('🦄🌈'), 582881315);
-
 	t.is(fnv1a('h'), 3977000791);
 	t.is(fnv1a('he'), 1547363254);
 	t.is(fnv1a('hel'), 179613742);
@@ -17,5 +16,27 @@ test('main', t => {
 	t.is(fnv1a('hello worl'), 2767971961);
 	t.is(fnv1a('hello world'), 3582672807);
 
+	// Bigger test
 	t.is(fnv1a('Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium.'), 2964896417);
+
+	// Verify unicode handling against values from https://www.tools4noobs.com/online_tools/hash/
+	t.is(fnv1a('🦄🌈'), 0xAAF5FEE7);
+	t.is(fnv1a('\u{0000}\u{0080}\u{0100}\u{0180}\u{0250}\u{02b0}\u{0300}\u{0370}\u{0400}\u{0500}\u{0530}\u{0590}\u{0600}\u{0700}\u{0780}\u{0900}\u{0980}\u{0a00}\u{0a80}\u{0b00}\u{0b80}\u{0c00}\u{0c80}\u{0d00}\u{0d80}\u{0e00}\u{0e80}\u{0f00}\u{1000}\u{10a0}\u{1100}\u{1200}\u{13a0}\u{1400}\u{1680}\u{16a0}\u{1700}\u{1720}\u{1740}\u{1760}\u{1780}\u{1800}\u{1900}\u{1950}\u{19e0}\u{1d00}\u{1e00}\u{1f00}\u{2000}\u{2070}\u{20a0}\u{20d0}\u{2100}\u{2150}\u{2190}\u{2200}\u{2300}\u{2400}\u{2440}\u{2460}\u{2500}\u{2580}\u{25a0}\u{2600}\u{2700}\u{27c0}\u{27f0}\u{2800}\u{2900}\u{2980}\u{2a00}\u{2b00}\u{2e80}\u{2f00}\u{2ff0}\u{3000}\u{3040}\u{30a0}\u{3100}\u{3130}\u{3190}\u{31a0}\u{31f0}\u{3200}\u{3300}\u{3400}\u{4dc0}\u{4e00}\u{a000}\u{a490}\u{ac00}\u{d800}\u{dc00}\u{e000}\u{f900}\u{fb00}\u{fb50}\u{fe00}\u{fe20}\u{fe30}\u{fe50}\u{fe70}\u{ff00}\u{fff0}\u{10000}\u{10080}\u{10100}\u{10300}\u{10330}\u{10380}\u{10400}\u{10450}\u{10480}\u{10800}\u{1d000}\u{1d100}\u{1d300}\u{1d400}\u{20000}\u{2f800}\u{e0000}\u{e0100}'), 0x983FDF05);
 });
+
+// If BigInt support available ...
+if (typeof (BigInt) === 'undefined') {
+	console.warn('BigInt not supported - skipping fnv1a.bigInt() tests');
+} else {
+	test('bigInt()', t => {
+		// Sanity check larger hashes against values from
+		// https://fnvhash.github.io/fnv-calculator-online/
+
+		t.is(bigInt('hello world', {size: 32}), BigInt('0xd58b3fa7'));
+		t.is(bigInt('hello world', {size: 64}), BigInt('0x779a65e7023cd2e7'));
+		t.is(bigInt('hello world', {size: 128}), BigInt('0x6c155799fdc8eec4b91523808e7726b7'));
+		t.is(bigInt('hello world', {size: 256}), BigInt('0xecc3cf2e0edfccd3d87f21ec0883aad4db43eead66ce09eb4a97e04e1a184527'));
+		t.is(bigInt('hello world', {size: 512}), BigInt('0x2b9c19ec56ccf98da0f227cc82bfaacbd8350928bd2ceacae7bc8aa13e747f5c43ca4e2e98fc25e94e4e805675545ee95a3b968c0acfaecb90aea2fdbcd4de0f'));
+		t.is(bigInt('hello world', {size: 1024}), BigInt('0x3fa9d253e52ae80105b382c80a01e27a53d7bc1d201efb47b38f4d6e465489829d7d272127d20e1076129c00000000000000000000000000000000000000000000000000000000000000000000000000000253eb20f42a7228af9022d9f35ece5bb71e40fcd8717b80d164ab921709996e5c43aae801418e878cddf968d4616f'));
+	});
+}