mirror of
https://github.com/roc-lang/roc.git
synced 2025-07-24 15:03:46 +00:00
Update to wyhash final 4
This update makes wyhash bad seed resist.
In old version of wyhash certain seeds would ruin the randomness.
Changes applied can be are based off of this diff:
a5995b9..77e50f2
This commit is contained in:
Brendan Hansknecht
parent
a616b61503
commit
2cf4f65d97
1 changed files with 58 additions and 48 deletions
|
|
@ -1005,12 +1005,12 @@ expect
|
|||
|
||||
# We have decided not to expose the standard roc hashing algorithm.
|
||||
# This is to avoid external dependence and the need for versioning.
|
||||
# The current implementation is a form of [Wyhash final3](https://github.com/wangyi-fudan/wyhash/blob/a5995b98ebfa7bd38bfadc0919326d2e7aabb805/wyhash.h).
|
||||
# The current implementation is a form of [Wyhash final4](https://github.com/wangyi-fudan/wyhash/blob/77e50f267fbc7b8e2d09f2d455219adb70ad4749/wyhash.h).
|
||||
# It is 64bit and little endian specific currently.
|
||||
# TODO: wyhash is slow for large keys, use something like cityhash if the keys are too long.
|
||||
# TODO: Add a builtin to distinguish big endian systems and change loading orders.
|
||||
# TODO: Switch out Wymum on systems with slow 128bit multiplication.
|
||||
LowLevelHasher := { originalSeed : U64, state : U64 } implements [
|
||||
LowLevelHasher := { initializedSeed : U64, state : U64 } implements [
|
||||
Hasher {
|
||||
addBytes,
|
||||
addU8,
|
||||
|
|
@ -1036,14 +1036,30 @@ createLowLevelHasher = \seedOpt ->
|
|||
when seedOpt is
|
||||
PseudoRandSeed -> pseudoSeed {}
|
||||
WithSeed s -> s
|
||||
@LowLevelHasher { originalSeed: seed, state: seed }
|
||||
@LowLevelHasher { initializedSeed: initSeed seed, state: seed }
|
||||
|
||||
combineState : LowLevelHasher, { a : U64, b : U64, seed : U64, length : U64 } -> LowLevelHasher
|
||||
combineState = \@LowLevelHasher { originalSeed, state }, { a, b, seed, length } ->
|
||||
tmp = wymix (Num.bitwiseXor wyp1 a) (Num.bitwiseXor seed b)
|
||||
hash = wymix (Num.bitwiseXor wyp1 length) tmp
|
||||
combineState = \@LowLevelHasher { initializedSeed, state }, { a, b, seed, length } ->
|
||||
mum =
|
||||
a
|
||||
|> Num.bitwiseXor wyp1
|
||||
|> wymum (Num.bitwiseXor b seed)
|
||||
nexta =
|
||||
mum.lower
|
||||
|> Num.bitwiseXor wyp0
|
||||
|> Num.bitwiseXor length
|
||||
nextb =
|
||||
mum.upper
|
||||
|> Num.bitwiseXor wyp1
|
||||
hash = wymix nexta nextb
|
||||
|
||||
@LowLevelHasher { originalSeed, state: wymix state hash }
|
||||
@LowLevelHasher { initializedSeed, state: wymix state hash }
|
||||
|
||||
initSeed = \seed ->
|
||||
seed
|
||||
|> Num.bitwiseXor wyp0
|
||||
|> wymix wyp1
|
||||
|> Num.bitwiseXor seed
|
||||
|
||||
complete = \@LowLevelHasher { state } -> state
|
||||
|
||||
|
|
@ -1052,8 +1068,7 @@ complete = \@LowLevelHasher { state } -> state
|
|||
# like using the output of the last hash as the seed to the current hash.
|
||||
# I am simply not sure the tradeoffs here. Theoretically this method is more sound.
|
||||
# Either way, the performance will be similar and we can change this later.
|
||||
addU8 = \@LowLevelHasher { originalSeed, state }, u8 ->
|
||||
seed = Num.bitwiseXor originalSeed wyp0
|
||||
addU8 = \@LowLevelHasher { initializedSeed, state }, u8 ->
|
||||
p0 = Num.toU64 u8
|
||||
a =
|
||||
Num.shiftLeftBy p0 16
|
||||
|
|
@ -1061,10 +1076,9 @@ addU8 = \@LowLevelHasher { originalSeed, state }, u8 ->
|
|||
|> Num.bitwiseOr p0
|
||||
b = 0
|
||||
|
||||
combineState (@LowLevelHasher { originalSeed, state }) { a, b, seed, length: 1 }
|
||||
combineState (@LowLevelHasher { initializedSeed, state }) { a, b, seed: initializedSeed, length: 1 }
|
||||
|
||||
addU16 = \@LowLevelHasher { originalSeed, state }, u16 ->
|
||||
seed = Num.bitwiseXor originalSeed wyp0
|
||||
addU16 = \@LowLevelHasher { initializedSeed, state }, u16 ->
|
||||
p0 = Num.bitwiseAnd u16 0xFF |> Num.toU64
|
||||
p1 = Num.shiftRightZfBy u16 8 |> Num.toU64
|
||||
a =
|
||||
|
|
@ -1073,26 +1087,23 @@ addU16 = \@LowLevelHasher { originalSeed, state }, u16 ->
|
|||
|> Num.bitwiseOr p1
|
||||
b = 0
|
||||
|
||||
combineState (@LowLevelHasher { originalSeed, state }) { a, b, seed, length: 2 }
|
||||
combineState (@LowLevelHasher { initializedSeed, state }) { a, b, seed: initializedSeed, length: 2 }
|
||||
|
||||
addU32 = \@LowLevelHasher { originalSeed, state }, u32 ->
|
||||
seed = Num.bitwiseXor originalSeed wyp0
|
||||
addU32 = \@LowLevelHasher { initializedSeed, state }, u32 ->
|
||||
p0 = Num.toU64 u32
|
||||
a = Num.shiftLeftBy p0 32 |> Num.bitwiseOr p0
|
||||
|
||||
combineState (@LowLevelHasher { originalSeed, state }) { a, b: a, seed, length: 4 }
|
||||
combineState (@LowLevelHasher { initializedSeed, state }) { a, b: a, seed: initializedSeed, length: 4 }
|
||||
|
||||
addU64 = \@LowLevelHasher { originalSeed, state }, u64 ->
|
||||
seed = Num.bitwiseXor originalSeed wyp0
|
||||
addU64 = \@LowLevelHasher { initializedSeed, state }, u64 ->
|
||||
p0 = Num.bitwiseAnd 0xFFFF_FFFF u64
|
||||
p1 = Num.shiftRightZfBy u64 32
|
||||
a = Num.shiftLeftBy p0 32 |> Num.bitwiseOr p1
|
||||
b = Num.shiftLeftBy p1 32 |> Num.bitwiseOr p0
|
||||
|
||||
combineState (@LowLevelHasher { originalSeed, state }) { a, b, seed, length: 8 }
|
||||
combineState (@LowLevelHasher { initializedSeed, state }) { a, b, seed: initializedSeed, length: 8 }
|
||||
|
||||
addU128 = \@LowLevelHasher { originalSeed, state }, u128 ->
|
||||
seed = Num.bitwiseXor originalSeed wyp0
|
||||
addU128 = \@LowLevelHasher { initializedSeed, state }, u128 ->
|
||||
lower = u128 |> Num.toU64
|
||||
upper = Num.shiftRightZfBy u128 64 |> Num.toU64
|
||||
p0 = Num.bitwiseAnd 0xFFFF_FFFF lower
|
||||
|
|
@ -1102,12 +1113,11 @@ addU128 = \@LowLevelHasher { originalSeed, state }, u128 ->
|
|||
a = Num.shiftLeftBy p0 32 |> Num.bitwiseOr p2
|
||||
b = Num.shiftLeftBy p3 32 |> Num.bitwiseOr p1
|
||||
|
||||
combineState (@LowLevelHasher { originalSeed, state }) { a, b, seed, length: 16 }
|
||||
combineState (@LowLevelHasher { initializedSeed, state }) { a, b, seed: initializedSeed, length: 16 }
|
||||
|
||||
addBytes : LowLevelHasher, List U8 -> LowLevelHasher
|
||||
addBytes = \@LowLevelHasher { originalSeed, state }, list ->
|
||||
addBytes = \@LowLevelHasher { initializedSeed, state }, list ->
|
||||
length = List.len list
|
||||
seed = Num.bitwiseXor originalSeed wyp0
|
||||
abs =
|
||||
if length <= 16 then
|
||||
if length >= 4 then
|
||||
|
|
@ -1117,17 +1127,17 @@ addBytes = \@LowLevelHasher { originalSeed, state }, list ->
|
|||
(wyr4 list (Num.subWrap length 4) |> Num.shiftLeftBy 32)
|
||||
|> Num.bitwiseOr (wyr4 list (Num.subWrap length 4 |> Num.subWrap x))
|
||||
|
||||
{ a, b, seed }
|
||||
{ a, b, seed: initializedSeed }
|
||||
else if length > 0 then
|
||||
{ a: wyr3 list 0 length, b: 0, seed }
|
||||
{ a: wyr3 list 0 length, b: 0, seed: initializedSeed }
|
||||
else
|
||||
{ a: 0, b: 0, seed }
|
||||
{ a: 0, b: 0, seed: initializedSeed }
|
||||
else if length <= 48 then
|
||||
hashBytesHelper16 seed list 0 length
|
||||
hashBytesHelper16 initializedSeed list 0 length
|
||||
else
|
||||
hashBytesHelper48 seed seed seed list 0 length
|
||||
hashBytesHelper48 initializedSeed initializedSeed initializedSeed list 0 length
|
||||
|
||||
combineState (@LowLevelHasher { originalSeed, state }) { a: abs.a, b: abs.b, seed: abs.seed, length: Num.toU64 length }
|
||||
combineState (@LowLevelHasher { initializedSeed, state }) { a: abs.a, b: abs.b, seed: abs.seed, length: Num.toU64 length }
|
||||
|
||||
hashBytesHelper48 : U64, U64, U64, List U8, Nat, Nat -> { a : U64, b : U64, seed : U64 }
|
||||
hashBytesHelper48 = \seed, see1, see2, list, index, remaining ->
|
||||
|
|
@ -1240,7 +1250,7 @@ expect
|
|||
|> addBytes []
|
||||
|> complete
|
||||
|
||||
hash == 0x1C3F_F8BF_07F9_B0B3
|
||||
hash == 0xD59C59757DBBE6B3
|
||||
|
||||
expect
|
||||
hash =
|
||||
|
|
@ -1248,7 +1258,7 @@ expect
|
|||
|> addBytes [0x42]
|
||||
|> complete
|
||||
|
||||
hash == 0x8F9F_0A1E_E06F_0D52
|
||||
hash == 0x38CE03D0E61AF963
|
||||
|
||||
expect
|
||||
hash =
|
||||
|
|
@ -1256,7 +1266,7 @@ expect
|
|||
|> addU8 0x42
|
||||
|> complete
|
||||
|
||||
hash == 0x8F9F_0A1E_E06F_0D52
|
||||
hash == 0x38CE03D0E61AF963
|
||||
|
||||
expect
|
||||
hash =
|
||||
|
|
@ -1264,7 +1274,7 @@ expect
|
|||
|> addBytes [0xFF, 0xFF]
|
||||
|> complete
|
||||
|
||||
hash == 0x86CC_8B71_563F_F084
|
||||
hash == 0xE1CB2FA0D6A64113
|
||||
|
||||
expect
|
||||
hash =
|
||||
|
|
@ -1272,7 +1282,7 @@ expect
|
|||
|> addU16 0xFFFF
|
||||
|> complete
|
||||
|
||||
hash == 0x86CC_8B71_563F_F084
|
||||
hash == 0xE1CB2FA0D6A64113
|
||||
|
||||
expect
|
||||
hash =
|
||||
|
|
@ -1280,7 +1290,7 @@ expect
|
|||
|> addBytes [0x36, 0xA7]
|
||||
|> complete
|
||||
|
||||
hash == 0xD1A5_0F24_2536_84F8
|
||||
hash == 0x26B8319EDAF81B15
|
||||
|
||||
expect
|
||||
hash =
|
||||
|
|
@ -1288,7 +1298,7 @@ expect
|
|||
|> addU16 0xA736
|
||||
|> complete
|
||||
|
||||
hash == 0xD1A5_0F24_2536_84F8
|
||||
hash == 0x26B8319EDAF81B15
|
||||
|
||||
expect
|
||||
hash =
|
||||
|
|
@ -1296,7 +1306,7 @@ expect
|
|||
|> addBytes [0x00, 0x00, 0x00, 0x00]
|
||||
|> complete
|
||||
|
||||
hash == 0x3762_ACB1_7604_B541
|
||||
hash == 0xA187D7CA074F9EE7
|
||||
|
||||
expect
|
||||
hash =
|
||||
|
|
@ -1304,7 +1314,7 @@ expect
|
|||
|> addU32 0x0000_0000
|
||||
|> complete
|
||||
|
||||
hash == 0x3762_ACB1_7604_B541
|
||||
hash == 0xA187D7CA074F9EE7
|
||||
|
||||
expect
|
||||
hash =
|
||||
|
|
@ -1312,7 +1322,7 @@ expect
|
|||
|> addBytes [0xA9, 0x2F, 0xEE, 0x21]
|
||||
|> complete
|
||||
|
||||
hash == 0x20F3_3FD7_D32E_C7A9
|
||||
hash == 0xA499EFE4C1454D09
|
||||
|
||||
expect
|
||||
hash =
|
||||
|
|
@ -1320,7 +1330,7 @@ expect
|
|||
|> addU32 0x21EE_2FA9
|
||||
|> complete
|
||||
|
||||
hash == 0x20F3_3FD7_D32E_C7A9
|
||||
hash == 0xA499EFE4C1454D09
|
||||
|
||||
expect
|
||||
hash =
|
||||
|
|
@ -1328,7 +1338,7 @@ expect
|
|||
|> addBytes [0x5D, 0x66, 0xB1, 0x8F, 0x68, 0x44, 0xC7, 0x03, 0xE1, 0xDD, 0x23, 0x34, 0xBB, 0x9A, 0x42, 0xA7]
|
||||
|> complete
|
||||
|
||||
hash == 0xA16F_DDAA_C167_74C7
|
||||
hash == 0xDD39A206AED64C73
|
||||
|
||||
expect
|
||||
hash =
|
||||
|
|
@ -1336,7 +1346,7 @@ expect
|
|||
|> addU128 0xA742_9ABB_3423_DDE1_03C7_4468_8FB1_665D
|
||||
|> complete
|
||||
|
||||
hash == 0xA16F_DDAA_C167_74C7
|
||||
hash == 0xDD39A206AED64C73
|
||||
|
||||
expect
|
||||
hash =
|
||||
|
|
@ -1344,7 +1354,7 @@ expect
|
|||
|> Hash.hashStrBytes "abcdefghijklmnopqrstuvwxyz"
|
||||
|> complete
|
||||
|
||||
hash == 0xBEE0_A8FD_E990_D285
|
||||
hash == 0x51C59DF5B1D15F40
|
||||
|
||||
expect
|
||||
hash =
|
||||
|
|
@ -1352,7 +1362,7 @@ expect
|
|||
|> Hash.hashStrBytes "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
|
||||
|> complete
|
||||
|
||||
hash == 0xB3C5_8528_9D82_A6EF
|
||||
hash == 0xD8D0A129D97A4E95
|
||||
|
||||
expect
|
||||
hash =
|
||||
|
|
@ -1360,7 +1370,7 @@ expect
|
|||
|> Hash.hashStrBytes "1234567890123456789012345678901234567890123456789012345678901234567890"
|
||||
|> complete
|
||||
|
||||
hash == 0xDB6B_7997_7A55_BA03
|
||||
hash == 0x8188065B44FB4AAA
|
||||
|
||||
expect
|
||||
hash =
|
||||
|
|
@ -1368,7 +1378,7 @@ expect
|
|||
|> addBytes (List.repeat 0x77 100)
|
||||
|> complete
|
||||
|
||||
hash == 0x171F_EEE2_B764_8E5E
|
||||
hash == 0x47A2A606EADF3378
|
||||
|
||||
# Note, had to specify u8 in the lists below to avoid ability type resolution error.
|
||||
# Apparently it won't pick the default integer.
|
||||
|
|
@ -1378,7 +1388,7 @@ expect
|
|||
|> Hash.hashUnordered [8u8, 82u8, 3u8, 8u8, 24u8] List.walk
|
||||
|> complete
|
||||
|
||||
hash == 0x999F_B530_3529_F17D
|
||||
hash == 0xB2E8254C08F16B20
|
||||
|
||||
expect
|
||||
hash1 =
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue