language-servers/roc
1
0
Fork 0
mirror of https://github.com/roc-lang/roc.git synced 2025-08-03 03:42:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Update to wyhash final 4

Browse source 
This update makes wyhash bad seed resist.
In old version of wyhash certain seeds would ruin the randomness.
Changes applied can be are based off of this diff:
a5995b9..77e50f2
This commit is contained in:
Brendan Hansknecht 2023-09-08 16:02:38 -07:00
parent a616b61503
commit 2cf4f65d97
No known key found for this signature in database
GPG key ID: 0EA784685083E75B
1 changed files with 58 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

106
crates/compiler/builtins/roc/Dict.roc
View file

@ -1005,12 +1005,12 @@ expect
# We have decided not to expose the standard roc hashing algorithm. # We have decided not to expose the standard roc hashing algorithm.
# This is to avoid external dependence and the need for versioning. # This is to avoid external dependence and the need for versioning.
# The current implementation is a form of [Wyhash final3](https://github.com/wangyi-fudan/wyhash/blob/a5995b98ebfa7bd38bfadc0919326d2e7aabb805/wyhash.h). # The current implementation is a form of [Wyhash final4](https://github.com/wangyi-fudan/wyhash/blob/77e50f267fbc7b8e2d09f2d455219adb70ad4749/wyhash.h).
# It is 64bit and little endian specific currently. # It is 64bit and little endian specific currently.
# TODO: wyhash is slow for large keys, use something like cityhash if the keys are too long. # TODO: wyhash is slow for large keys, use something like cityhash if the keys are too long.
# TODO: Add a builtin to distinguish big endian systems and change loading orders. # TODO: Add a builtin to distinguish big endian systems and change loading orders.
# TODO: Switch out Wymum on systems with slow 128bit multiplication. # TODO: Switch out Wymum on systems with slow 128bit multiplication.
LowLevelHasher := { originalSeed : U64, state : U64 } implements [ LowLevelHasher := { initializedSeed : U64, state : U64 } implements [
Hasher { Hasher {
addBytes, addBytes,
addU8, addU8,
@ -1036,14 +1036,30 @@ createLowLevelHasher = \seedOpt ->
when seedOpt is when seedOpt is
PseudoRandSeed -> pseudoSeed {} PseudoRandSeed -> pseudoSeed {}
WithSeed s -> s WithSeed s -> s
@LowLevelHasher { originalSeed: seed, state: seed } @LowLevelHasher { initializedSeed: initSeed seed, state: seed }
combineState : LowLevelHasher, { a : U64, b : U64, seed : U64, length : U64 } -> LowLevelHasher combineState : LowLevelHasher, { a : U64, b : U64, seed : U64, length : U64 } -> LowLevelHasher
combineState = \@LowLevelHasher { originalSeed, state }, { a, b, seed, length } -> combineState = \@LowLevelHasher { initializedSeed, state }, { a, b, seed, length } ->
tmp = wymix (Num.bitwiseXor wyp1 a) (Num.bitwiseXor seed b) mum =
hash = wymix (Num.bitwiseXor wyp1 length) tmp a
|> Num.bitwiseXor wyp1
|> wymum (Num.bitwiseXor b seed)
nexta =
mum.lower
|> Num.bitwiseXor wyp0
|> Num.bitwiseXor length
nextb =
mum.upper
|> Num.bitwiseXor wyp1
hash = wymix nexta nextb
@LowLevelHasher { originalSeed, state: wymix state hash } @LowLevelHasher { initializedSeed, state: wymix state hash }
initSeed = \seed ->
seed
|> Num.bitwiseXor wyp0
|> wymix wyp1
|> Num.bitwiseXor seed
complete = \@LowLevelHasher { state } -> state complete = \@LowLevelHasher { state } -> state
@ -1052,8 +1068,7 @@ complete = \@LowLevelHasher { state } -> state
# like using the output of the last hash as the seed to the current hash. # like using the output of the last hash as the seed to the current hash.
# I am simply not sure the tradeoffs here. Theoretically this method is more sound. # I am simply not sure the tradeoffs here. Theoretically this method is more sound.
# Either way, the performance will be similar and we can change this later. # Either way, the performance will be similar and we can change this later.
addU8 = \@LowLevelHasher { originalSeed, state }, u8 -> addU8 = \@LowLevelHasher { initializedSeed, state }, u8 ->
seed = Num.bitwiseXor originalSeed wyp0
p0 = Num.toU64 u8 p0 = Num.toU64 u8
a = a =
Num.shiftLeftBy p0 16 Num.shiftLeftBy p0 16
@ -1061,10 +1076,9 @@ addU8 = \@LowLevelHasher { originalSeed, state }, u8 ->
|> Num.bitwiseOr p0 |> Num.bitwiseOr p0
b = 0 b = 0
combineState (@LowLevelHasher { originalSeed, state }) { a, b, seed, length: 1 } combineState (@LowLevelHasher { initializedSeed, state }) { a, b, seed: initializedSeed, length: 1 }
addU16 = \@LowLevelHasher { originalSeed, state }, u16 -> addU16 = \@LowLevelHasher { initializedSeed, state }, u16 ->
seed = Num.bitwiseXor originalSeed wyp0
p0 = Num.bitwiseAnd u16 0xFF |> Num.toU64 p0 = Num.bitwiseAnd u16 0xFF |> Num.toU64
p1 = Num.shiftRightZfBy u16 8 |> Num.toU64 p1 = Num.shiftRightZfBy u16 8 |> Num.toU64
a = a =
@ -1073,26 +1087,23 @@ addU16 = \@LowLevelHasher { originalSeed, state }, u16 ->
|> Num.bitwiseOr p1 |> Num.bitwiseOr p1
b = 0 b = 0
combineState (@LowLevelHasher { originalSeed, state }) { a, b, seed, length: 2 } combineState (@LowLevelHasher { initializedSeed, state }) { a, b, seed: initializedSeed, length: 2 }
addU32 = \@LowLevelHasher { originalSeed, state }, u32 -> addU32 = \@LowLevelHasher { initializedSeed, state }, u32 ->
seed = Num.bitwiseXor originalSeed wyp0
p0 = Num.toU64 u32 p0 = Num.toU64 u32
a = Num.shiftLeftBy p0 32 |> Num.bitwiseOr p0 a = Num.shiftLeftBy p0 32 |> Num.bitwiseOr p0
combineState (@LowLevelHasher { originalSeed, state }) { a, b: a, seed, length: 4 } combineState (@LowLevelHasher { initializedSeed, state }) { a, b: a, seed: initializedSeed, length: 4 }
addU64 = \@LowLevelHasher { originalSeed, state }, u64 -> addU64 = \@LowLevelHasher { initializedSeed, state }, u64 ->
seed = Num.bitwiseXor originalSeed wyp0
p0 = Num.bitwiseAnd 0xFFFF_FFFF u64 p0 = Num.bitwiseAnd 0xFFFF_FFFF u64
p1 = Num.shiftRightZfBy u64 32 p1 = Num.shiftRightZfBy u64 32
a = Num.shiftLeftBy p0 32 |> Num.bitwiseOr p1 a = Num.shiftLeftBy p0 32 |> Num.bitwiseOr p1
b = Num.shiftLeftBy p1 32 |> Num.bitwiseOr p0 b = Num.shiftLeftBy p1 32 |> Num.bitwiseOr p0
combineState (@LowLevelHasher { originalSeed, state }) { a, b, seed, length: 8 } combineState (@LowLevelHasher { initializedSeed, state }) { a, b, seed: initializedSeed, length: 8 }
addU128 = \@LowLevelHasher { originalSeed, state }, u128 -> addU128 = \@LowLevelHasher { initializedSeed, state }, u128 ->
seed = Num.bitwiseXor originalSeed wyp0
lower = u128 |> Num.toU64 lower = u128 |> Num.toU64
upper = Num.shiftRightZfBy u128 64 |> Num.toU64 upper = Num.shiftRightZfBy u128 64 |> Num.toU64
p0 = Num.bitwiseAnd 0xFFFF_FFFF lower p0 = Num.bitwiseAnd 0xFFFF_FFFF lower
@ -1102,12 +1113,11 @@ addU128 = \@LowLevelHasher { originalSeed, state }, u128 ->
a = Num.shiftLeftBy p0 32 |> Num.bitwiseOr p2 a = Num.shiftLeftBy p0 32 |> Num.bitwiseOr p2
b = Num.shiftLeftBy p3 32 |> Num.bitwiseOr p1 b = Num.shiftLeftBy p3 32 |> Num.bitwiseOr p1
combineState (@LowLevelHasher { originalSeed, state }) { a, b, seed, length: 16 } combineState (@LowLevelHasher { initializedSeed, state }) { a, b, seed: initializedSeed, length: 16 }
addBytes : LowLevelHasher, List U8 -> LowLevelHasher addBytes : LowLevelHasher, List U8 -> LowLevelHasher
addBytes = \@LowLevelHasher { originalSeed, state }, list -> addBytes = \@LowLevelHasher { initializedSeed, state }, list ->
length = List.len list length = List.len list
seed = Num.bitwiseXor originalSeed wyp0
abs = abs =
if length <= 16 then if length <= 16 then
if length >= 4 then if length >= 4 then
@ -1117,17 +1127,17 @@ addBytes = \@LowLevelHasher { originalSeed, state }, list ->
(wyr4 list (Num.subWrap length 4) |> Num.shiftLeftBy 32) (wyr4 list (Num.subWrap length 4) |> Num.shiftLeftBy 32)
|> Num.bitwiseOr (wyr4 list (Num.subWrap length 4 |> Num.subWrap x)) |> Num.bitwiseOr (wyr4 list (Num.subWrap length 4 |> Num.subWrap x))
{ a, b, seed } { a, b, seed: initializedSeed }
else if length > 0 then else if length > 0 then
{ a: wyr3 list 0 length, b: 0, seed } { a: wyr3 list 0 length, b: 0, seed: initializedSeed }
else else
{ a: 0, b: 0, seed } { a: 0, b: 0, seed: initializedSeed }
else if length <= 48 then else if length <= 48 then
hashBytesHelper16 seed list 0 length hashBytesHelper16 initializedSeed list 0 length
else else
hashBytesHelper48 seed seed seed list 0 length hashBytesHelper48 initializedSeed initializedSeed initializedSeed list 0 length
combineState (@LowLevelHasher { originalSeed, state }) { a: abs.a, b: abs.b, seed: abs.seed, length: Num.toU64 length } combineState (@LowLevelHasher { initializedSeed, state }) { a: abs.a, b: abs.b, seed: abs.seed, length: Num.toU64 length }
hashBytesHelper48 : U64, U64, U64, List U8, Nat, Nat -> { a : U64, b : U64, seed : U64 } hashBytesHelper48 : U64, U64, U64, List U8, Nat, Nat -> { a : U64, b : U64, seed : U64 }
hashBytesHelper48 = \seed, see1, see2, list, index, remaining -> hashBytesHelper48 = \seed, see1, see2, list, index, remaining ->
@ -1240,7 +1250,7 @@ expect
|> addBytes [] |> addBytes []
|> complete |> complete
hash == 0x1C3F_F8BF_07F9_B0B3 hash == 0xD59C59757DBBE6B3
expect expect
hash = hash =
@ -1248,7 +1258,7 @@ expect
|> addBytes [0x42] |> addBytes [0x42]
|> complete |> complete
hash == 0x8F9F_0A1E_E06F_0D52 hash == 0x38CE03D0E61AF963
expect expect
hash = hash =
@ -1256,7 +1266,7 @@ expect
|> addU8 0x42 |> addU8 0x42
|> complete |> complete
hash == 0x8F9F_0A1E_E06F_0D52 hash == 0x38CE03D0E61AF963
expect expect
hash = hash =
@ -1264,7 +1274,7 @@ expect
|> addBytes [0xFF, 0xFF] |> addBytes [0xFF, 0xFF]
|> complete |> complete
hash == 0x86CC_8B71_563F_F084 hash == 0xE1CB2FA0D6A64113
expect expect
hash = hash =
@ -1272,7 +1282,7 @@ expect
|> addU16 0xFFFF |> addU16 0xFFFF
|> complete |> complete
hash == 0x86CC_8B71_563F_F084 hash == 0xE1CB2FA0D6A64113
expect expect
hash = hash =
@ -1280,7 +1290,7 @@ expect
|> addBytes [0x36, 0xA7] |> addBytes [0x36, 0xA7]
|> complete |> complete
hash == 0xD1A5_0F24_2536_84F8 hash == 0x26B8319EDAF81B15
expect expect
hash = hash =
@ -1288,7 +1298,7 @@ expect
|> addU16 0xA736 |> addU16 0xA736
|> complete |> complete
hash == 0xD1A5_0F24_2536_84F8 hash == 0x26B8319EDAF81B15
expect expect
hash = hash =
@ -1296,7 +1306,7 @@ expect
|> addBytes [0x00, 0x00, 0x00, 0x00] |> addBytes [0x00, 0x00, 0x00, 0x00]
|> complete |> complete
hash == 0x3762_ACB1_7604_B541 hash == 0xA187D7CA074F9EE7
expect expect
hash = hash =
@ -1304,7 +1314,7 @@ expect
|> addU32 0x0000_0000 |> addU32 0x0000_0000
|> complete |> complete
hash == 0x3762_ACB1_7604_B541 hash == 0xA187D7CA074F9EE7
expect expect
hash = hash =
@ -1312,7 +1322,7 @@ expect
|> addBytes [0xA9, 0x2F, 0xEE, 0x21] |> addBytes [0xA9, 0x2F, 0xEE, 0x21]
|> complete |> complete
hash == 0x20F3_3FD7_D32E_C7A9 hash == 0xA499EFE4C1454D09
expect expect
hash = hash =
@ -1320,7 +1330,7 @@ expect
|> addU32 0x21EE_2FA9 |> addU32 0x21EE_2FA9
|> complete |> complete
hash == 0x20F3_3FD7_D32E_C7A9 hash == 0xA499EFE4C1454D09
expect expect
hash = hash =
@ -1328,7 +1338,7 @@ expect
|> addBytes [0x5D, 0x66, 0xB1, 0x8F, 0x68, 0x44, 0xC7, 0x03, 0xE1, 0xDD, 0x23, 0x34, 0xBB, 0x9A, 0x42, 0xA7] |> addBytes [0x5D, 0x66, 0xB1, 0x8F, 0x68, 0x44, 0xC7, 0x03, 0xE1, 0xDD, 0x23, 0x34, 0xBB, 0x9A, 0x42, 0xA7]
|> complete |> complete
hash == 0xA16F_DDAA_C167_74C7 hash == 0xDD39A206AED64C73
expect expect
hash = hash =
@ -1336,7 +1346,7 @@ expect
|> addU128 0xA742_9ABB_3423_DDE1_03C7_4468_8FB1_665D |> addU128 0xA742_9ABB_3423_DDE1_03C7_4468_8FB1_665D
|> complete |> complete
hash == 0xA16F_DDAA_C167_74C7 hash == 0xDD39A206AED64C73
expect expect
hash = hash =
@ -1344,7 +1354,7 @@ expect
|> Hash.hashStrBytes "abcdefghijklmnopqrstuvwxyz" |> Hash.hashStrBytes "abcdefghijklmnopqrstuvwxyz"
|> complete |> complete
hash == 0xBEE0_A8FD_E990_D285 hash == 0x51C59DF5B1D15F40
expect expect
hash = hash =
@ -1352,7 +1362,7 @@ expect
|> Hash.hashStrBytes "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" |> Hash.hashStrBytes "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
|> complete |> complete
hash == 0xB3C5_8528_9D82_A6EF hash == 0xD8D0A129D97A4E95
expect expect
hash = hash =
@ -1360,7 +1370,7 @@ expect
|> Hash.hashStrBytes "1234567890123456789012345678901234567890123456789012345678901234567890" |> Hash.hashStrBytes "1234567890123456789012345678901234567890123456789012345678901234567890"
|> complete |> complete
hash == 0xDB6B_7997_7A55_BA03 hash == 0x8188065B44FB4AAA
expect expect
hash = hash =
@ -1368,7 +1378,7 @@ expect
|> addBytes (List.repeat 0x77 100) |> addBytes (List.repeat 0x77 100)
|> complete |> complete
hash == 0x171F_EEE2_B764_8E5E hash == 0x47A2A606EADF3378
# Note, had to specify u8 in the lists below to avoid ability type resolution error. # Note, had to specify u8 in the lists below to avoid ability type resolution error.
# Apparently it won't pick the default integer. # Apparently it won't pick the default integer.
@ -1378,7 +1388,7 @@ expect
|> Hash.hashUnordered [8u8, 82u8, 3u8, 8u8, 24u8] List.walk |> Hash.hashUnordered [8u8, 82u8, 3u8, 8u8, 24u8] List.walk
|> complete |> complete
hash == 0x999F_B530_3529_F17D hash == 0xB2E8254C08F16B20
expect expect
hash1 = hash1 =