Move to 64bit hash and custom hash functions

hashmap.go

@@ -1,7 +1,10 @@
 package immutable
 
 import (
+	"fmt"
 	"math"
+	"math/rand"
+	"time"
 
 	"github.com/object88/immutable/memory"
 )
@@ -26,6 +29,7 @@ type HashMap struct {
 	buckets []*bucket
 	lobMask uint32
 	lobSize uint8
+	seed    uint32
 }
 
 const (
@@ -56,17 +60,28 @@ func (h *HashMap) Get(key Key) Value {
 		return nil
 	}
 
-	hashkey := key.Hash()
+	hashkey := key.Hash(h.seed)
 
-	selectedBucket := hashkey & h.lobMask
+	selectedBucket := hashkey & uint64(h.lobMask)
 	b := h.buckets[selectedBucket]
 	maskedHash := hashkey >> h.lobSize
 
 	totalEntries := uint64(b.entryCount)
 
+	fmt.Printf("\nlobSize: %d; h.lobMask: 0x%016x\n", h.lobSize, h.lobMask)
+	fmt.Printf("hashKey: 0x%016x / selectedBucket: %d / mashedHash: 0x%016x\n", hashkey, selectedBucket, maskedHash)
+
 	for b != nil {
+		fmt.Printf("    entryCount: %d\n", b.entryCount)
+		fmt.Printf("    entries: [\n")
+		for i := uint64(0); i < uint64(b.entryCount); i++ {
+			fmt.Printf("      [0x%016x,%s] -> %s\n", b.hobs.Read(i), b.entries[i].key, b.entries[i].value)
+		}
+		fmt.Printf("    ]\n")
+
 		for index := uint64(0); index < totalEntries; index++ {
-			if uint32(b.hobs.Read(index)) == maskedHash && b.entries[index].key == key {
+			fmt.Printf("0x%016x <-> 0x%016x :: %s <-> %s\n", b.hobs.Read(index), maskedHash, b.entries[index].key, key)
+			if b.hobs.Read(index) == maskedHash && b.entries[index].key == key {
 				return b.entries[index].value
 			}
 		}
@@ -244,26 +259,25 @@ func (h *HashMap) instantiate(size int, contents []*keyValuePair) *BaseStruct {
 }
 
 func (h *HashMap) internalSet(key Key, value Value) {
-	// lobSize := h.lobSize // uint32(memory.PowerOf(h.size))
-	hobSize := uint32(32 - h.lobSize)
+	hobSize := uint32(64 - h.lobSize)
 
-	hashkey := key.Hash()
-	selectedBucket := hashkey & h.lobMask
+	hashkey := key.Hash(h.seed)
+	selectedBucket := hashkey & uint64(h.lobMask)
 	// fmt.Printf("At [%s,%s]; h:0x%08x, sb: %d, lob: 0x%08x\n", key, value, hashkey, selectedBucket, hashkey>>h.lobSize)
 	b := h.buckets[selectedBucket]
 	if b == nil {
 		// Create the bucket.
-		b = createEmptyBucket(memory.LargeBlock, hobSize)
+		b = createEmptyBucket(h.options.BucketStrategy, hobSize)
 		h.buckets[selectedBucket] = b
 	}
 	for b.entryCount == 8 {
 		if b.overflow == nil {
-			b.overflow = createEmptyBucket(memory.LargeBlock, hobSize)
+			b.overflow = createEmptyBucket(h.options.BucketStrategy, hobSize)
 		}
 		b = b.overflow
 	}
 	b.entries[b.entryCount] = entry{key, value}
-	b.hobs.Assign(uint64(b.entryCount), uint64(hashkey>>h.lobSize))
+	b.hobs.Assign(uint64(b.entryCount), hashkey>>h.lobSize)
 	b.entryCount++
 }
 
@@ -274,7 +288,11 @@ func createHashMap(size int, options *HashMapOptions) *HashMap {
 	lobMask := uint32(^(0xffffffff << lobSize))
 	buckets := make([]*bucket, initialSize)
 
-	return &HashMap{options, initialCount, int(initialSize), buckets, lobMask, lobSize}
+	src := rand.NewSource(time.Now().UnixNano())
+	random := rand.New(src)
+	seed := uint32(random.Int31())
+
+	return &HashMap{options, initialCount, int(initialSize), buckets, lobMask, lobSize, seed}
 }
 
 func createEmptyBucket(blockSize memory.BlockSize, hobSize uint32) *bucket {

hashmap_customkey_badhash_test.go

@@ -14,11 +14,11 @@ type MyBadKey struct {
 	value int
 }
 
-func (k MyBadKey) Hash() uint32 {
+func (k MyBadKey) Hash(seed uint32) uint64 {
 	if k.value%2 == 0 {
 		return 0x0
 	}
-	return 0xffffffff
+	return 0xffffffffffffffff
 }
 
 func (k MyBadKey) String() string {

hashmap_customkey_test.go

@@ -9,8 +9,8 @@ type MyKey struct {
 	value int
 }
 
-func (k MyKey) Hash() uint32 {
-	return uint32(k.value)
+func (k MyKey) Hash(seed uint32) uint64 {
+	return uint64(k.value)
 }
 
 func (k MyKey) String() string {

hashmap_gostringer.go

@@ -22,8 +22,8 @@ func (h *HashMap) GoString() string {
 		buffer.WriteString(fmt.Sprintf("    entryCount: %d\n", b.entryCount))
 		buffer.WriteString("    entries: [\n")
 		for b != nil {
-			for i := uint32(0); i < uint32(b.entryCount); i++ {
-				buffer.WriteString(fmt.Sprintf("      [0x%08x,%s] -> %s\n", b.hobs.Read(uint64(i)), b.entries[i].key, b.entries[i].value))
+			for i := uint64(0); i < uint64(b.entryCount); i++ {
+				buffer.WriteString(fmt.Sprintf("      [0x%016x,%s] -> %s\n", b.hobs.Read(i), b.entries[i].key, b.entries[i].value))
 			}
 
 			b = b.overflow

hashmap_test.go

@@ -303,8 +303,12 @@ func Test_Hashmap_ReadAndWriteLargeDataSet(t *testing.T) {
 	}
 
 	original := NewHashMap(contents, nil)
-	for k, v := range contents {
+	// fmt.Printf("%#v\n", original)
+	// for k, v := range contents {
+	for i := 0; i < max; i++ {
+		k := IntKey(i)
 		result := original.Get(k)
+		v := contents[k]
 		if result != v {
 			t.Fatalf("At %s; expected %d, got %d\n", k, v, result)
 		}

key.go

@@ -1,9 +1,9 @@
 package immutable
 
 import (
-	"encoding/binary"
 	"fmt"
-	"hash/fnv"
+	"math/rand"
+	"time"
 )
 
 // IntKey is an integer-based Key
@@ -12,13 +12,42 @@ type IntKey int
 // StringKey is a string-based Key
 type StringKey string
 
-// Hash calculates the 32-bit hash
-func (k IntKey) Hash() uint32 {
-	hasher := fnv.New32a()
+// // Hash calculates the 32-bit hash
+// func (k IntKey) Hash() uint32 {
+// 	hasher := fnv.New32a()
+//
+// 	binary.Write(hasher, binary.LittleEndian, uint32(k))
+// 	hash := hasher.Sum32()
+// 	return hash
+// }
+
+const m1 = 194865226553
+const m2 = 24574600569641
+
+var hashkey [4]uintptr
+
+func init() {
+	src := rand.NewSource(time.Now().UnixNano())
+	random := rand.New(src)
+	// getRandomData((*[len(hashkey) * sys.PtrSize]byte)(unsafe.Pointer(&hashkey))[:])
+	hashkey[0] = uintptr(random.Int63() | 1) // make sure these numbers are odd
+	hashkey[1] = uintptr(random.Int63() | 1)
+	hashkey[2] = uintptr(random.Int63() | 1)
+	hashkey[3] = uintptr(random.Int63() | 1)
+}
+
+// Hash does what Hash cannot.
+func (k IntKey) Hash(seed uint32) uint64 {
+	k1 := uint64(k)
+	h := uint64(uintptr(seed) + 8*hashkey[0])
+	h ^= (k1 & 0xffffffff)
+	h ^= (k1 & 0xffffffff00000000) << 32
+	h = rotl31(h*m1) * m2
+	return h
+}
 
-	binary.Write(hasher, binary.LittleEndian, uint32(k))
-	hash := hasher.Sum32()
-	return hash
+func rotl31(x uint64) uint64 {
+	return (x << 31) | (x >> (64 - 31))
 }
 
 func (k IntKey) String() string {

memory/memory32.go

@@ -1,5 +1,7 @@
 package memory
 
+import "fmt"
+
 const fullBlock = ^uint32(0)
 
 // Memories32 is all your memories.
@@ -15,8 +17,8 @@ func (m *Memories32) Assign(index uint64, value uint64) {
 	byteOffset := offset / bitsInLargeBlock
 	bitOffset := offset % bitsInLargeBlock
 
-	// fmt.Printf("\nAssigning %032b to index %d\n", value, index)
-	// fmt.Printf("byteOffset: %d, bitOffset: %d, bitsRemaining: %d\n", byteOffset, bitOffset, bitsRemaining)
+	fmt.Printf("\nAssigning %032b to index %d\n", value, index)
+	fmt.Printf("byteOffset: %d, bitOffset: %d, bitsRemaining: %d\n", byteOffset, bitOffset, bitsRemaining)
 
 	writeBitCount := bitsInLargeBlock - bitOffset
 	if writeBitCount > bitsRemaining {
@@ -27,7 +29,7 @@ func (m *Memories32) Assign(index uint64, value uint64) {
 	result := (initial & ^(^mask << bitOffset)) | ((value & ^mask) << bitOffset)
 	m.m[byteOffset] = uint32(result)
 
-	// fmt.Printf("result at %d: %032b\n", byteOffset, m.m[byteOffset])
+	fmt.Printf("result at %d: %032b\n", byteOffset, m.m[byteOffset])
 
 	bitsRemaining -= writeBitCount
 	byteOffset++
@@ -36,7 +38,7 @@ func (m *Memories32) Assign(index uint64, value uint64) {
 		o := (uint64(m.bitsPerEntry) - bitsRemaining)
 		result := ((value & (uint64(fullBlock) << o)) >> o)
 		m.m[byteOffset] = uint32(result)
-		// fmt.Printf("result at %d: %032b\n", byteOffset, m.m[byteOffset])
+		fmt.Printf("result at %d: %032b\n", byteOffset, m.m[byteOffset])
 
 		bitsRemaining -= 32
 		byteOffset++
@@ -48,7 +50,7 @@ func (m *Memories32) Assign(index uint64, value uint64) {
 		result := (initial & mask) | ((value & (^mask << writeBitCount)) >> writeBitCount)
 		m.m[byteOffset] = uint32(result)
 
-		// fmt.Printf("result at %d: %032b\n", byteOffset, m.m[byteOffset])
+		fmt.Printf("result at %d: %032b\n", byteOffset, m.m[byteOffset])
 	}
 }
 
@@ -58,8 +60,8 @@ func (m *Memories32) Read(index uint64) (result uint64) {
 	offset := bitsRemaining * index
 	bitOffset := offset % bitsInLargeBlock
 	byteOffset := offset / bitsInLargeBlock
-	// fmt.Printf("\nbitOffset: %d, byteOffset: %d\n", bitOffset, byteOffset)
-	// fmt.Printf("m.m: %x\n", m.m)
+	fmt.Printf("\nbitOffset: %d, byteOffset: %d\n", bitOffset, byteOffset)
+	fmt.Printf("m.m: %x\n", m.m)
 
 	readBitCount := bitsInLargeBlock - bitOffset
 	if readBitCount > bitsRemaining {
@@ -76,18 +78,18 @@ func (m *Memories32) Read(index uint64) (result uint64) {
 		if readBitCount > bitsInLargeBlock {
 			readBitCount = bitsInLargeBlock
 		}
-		// fmt.Printf("--> %064b; %d; %d\n", result, bitsRemaining, readBitCount)
+		fmt.Printf("--> %064b; %d; %d\n", result, bitsRemaining, readBitCount)
 		initial := uint64(m.m[byteOffset+1])
 		result |= ((initial & uint64(^(fullBlock << readBitCount))) << (uint64(m.bitsPerEntry) - bitsRemaining))
 		bitsRemaining -= readBitCount
 	}
 
 	if bitsRemaining > 0 {
 		initial := uint64(m.m[byteOffset+2])
-		// fmt.Printf("--> %064b; %d\n", result, bitsRemaining)
+		fmt.Printf("--> %064b; %d\n", result, bitsRemaining)
 		result |= ((initial & uint64(^(fullBlock << bitsRemaining))) << (uint64(m.bitsPerEntry) - bitsRemaining))
 	}
-	// fmt.Printf("--> %064b\n", result)
+	fmt.Printf("--> %064b\n", result)
 
 	return result
 }

memory/memory32_test.go

@@ -1,6 +1,7 @@
 package memory
 
 import (
+	"math"
 	"math/rand"
 	"testing"
 	"time"
@@ -110,45 +111,71 @@ func evaluateLargeAssign(t *testing.T, bitCount, count uint32, writeIndex, value
 	}
 }
 
-func Test_WriteAndRead2(t *testing.T) {
-	count := 4
-	set := make([]uint64, count)
-	for i := 0; i < count; i++ {
-		set[i] = 0x55555555
-	}
-
-	m := AllocateMemories(LargeBlock, 31, 4)
-
-	for k, v := range set {
-		m.Assign(uint64(k), v)
-	}
+func Test_Large_Random(t *testing.T) {
+	src := rand.NewSource(time.Now().UnixNano())
+	random := rand.New(src)
 
-	for k, v := range set {
-		result := m.Read(uint64(k))
-		if result != v {
-			t.Fatalf("At %d\nexpected %032b\nreceived %032b\n", k, v, result)
-		}
-	}
-}
+	width := uint32(64 - 11)
+	max := int64(math.Pow(2.0, float64(width)))
 
-func Test_WriteAndRead(t *testing.T) {
-	count := 4
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-	set := make([]uint64, count)
-	for i := 0; i < count; i++ {
-		set[i] = uint64(r.Int31()) & 0x7fffffff
+	count := uint64(8)
+	contents := make([]uint64, count)
+	for i := uint64(0); i < count; i++ {
+		contents[i] = uint64(random.Int63n(max))
 	}
 
-	m := AllocateMemories(LargeBlock, 31, 4)
-
-	for k, v := range set {
+	m := AllocateMemories(LargeBlock, width, 8)
+	for k, v := range contents {
 		m.Assign(uint64(k), v)
 	}
 
-	for k, v := range set {
+	for k, v := range contents {
 		result := m.Read(uint64(k))
 		if result != v {
-			t.Fatalf("At %d\nexpected %032b\nreceived %032b\n", k, v, result)
+			t.Fatalf("At %d\nexpected %064b\nreceived %064b\n", k, v, result)
 		}
 	}
 }
+
+// func Test_WriteAndRead2(t *testing.T) {
+// 	count := 4
+// 	set := make([]uint64, count)
+// 	for i := 0; i < count; i++ {
+// 		set[i] = 0x55555555
+// 	}
+//
+// 	m := AllocateMemories(LargeBlock, 31, 4)
+//
+// 	for k, v := range set {
+// 		m.Assign(uint64(k), v)
+// 	}
+//
+// 	for k, v := range set {
+// 		result := m.Read(uint64(k))
+// 		if result != v {
+// 			t.Fatalf("At %d\nexpected %032b\nreceived %032b\n", k, v, result)
+// 		}
+// 	}
+// }
+//
+// func Test_WriteAndRead(t *testing.T) {
+// 	count := 4
+// 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+// 	set := make([]uint64, count)
+// 	for i := 0; i < count; i++ {
+// 		set[i] = uint64(r.Int31()) & 0x7fffffff
+// 	}
+//
+// 	m := AllocateMemories(LargeBlock, 31, 4)
+//
+// 	for k, v := range set {
+// 		m.Assign(uint64(k), v)
+// 	}
+//
+// 	for k, v := range set {
+// 		result := m.Read(uint64(k))
+// 		if result != v {
+// 			t.Fatalf("At %d\nexpected %032b\nreceived %032b\n", k, v, result)
+// 		}
+// 	}
+// }