1 /** 2 * Computes xxHash hashes of arbitrary data. xxHash hashes are either uint32_t, 3 * uint64_t or uint128_t quantities that are like a 4 * checksum or CRC, but are more robust and very performant. 5 * 6 $(SCRIPT inhibitQuickIndex = 1;) 7 8 $(DIVC quickindex, 9 $(BOOKTABLE , 10 $(TR $(TH Category) $(TH Functions) 11 ) 12 $(TR $(TDNW Template API) $(TD $(MYREF XXHTemplate) 13 ) 14 ) 15 $(TR $(TDNW OOP API) $(TD $(MYREF XXH32Digest)) 16 ) 17 $(TR $(TDNW Helpers) $(TD $(MYREF xxh32Of)) 18 ) 19 ) 20 ) 21 22 * This module conforms to the APIs defined in `std.digest`. To understand the 23 * differences between the template and the OOP API, see $(MREF std, digest). 24 * 25 * This module publicly imports $(MREF std, digest) and can be used as a stand-alone 26 * module. 27 * 28 * License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0). 29 * 30 * CTFE: 31 * Digests do not work in CTFE 32 * 33 * Authors: 34 * Carsten Schlote, Piotr Szturmaj, Kai Nacke, Johannes Pfau $(BR) 35 * The routines and algorithms are provided by the xxhash.[ch] source 36 * provided at $(I git@github.com:Cyan4973/xxHash.git). 37 * 38 * References: 39 * $(LINK2 https://github.com/Cyan4973/xxHash, GitHub website of project) 40 * 41 * Source: $(PHOBOSSRC std/digest/xxh.d) 42 * 43 */ 44 45 /* xxh.d - A wrapper for the original C implementation */ 46 //module std.digest.xxh; 47 module local_dev.xxh; 48 49 version (X86) 50 version = HaveUnalignedLoads; 51 else version (X86_64) 52 version = HaveUnalignedLoads; 53 54 //TODO: Properly detect this requirement. 55 //version = CheckACCAlignment; 56 57 //TODO: Check, if this code is an advantage over XXH provided code 58 // The code from core.int128 doesn't inline. 59 //version = Have128BitInteger; 60 61 version (Have128BitInteger) 62 { 63 import core.int128; 64 } 65 66 /// 67 @safe unittest 68 { 69 //Template API 70 import std.digest.md; 71 72 //Feeding data 73 ubyte[1024] data; 74 XXH_32 xxh; 75 xxh.start(); 76 xxh.put(data[]); 77 xxh.start(); //Start again 78 xxh.put(data[]); 79 auto hash = xxh.finish(); 80 } 81 82 /// 83 @safe unittest 84 { 85 //OOP API 86 import std.digest.md; 87 88 auto xxh = new XXH32Digest(); 89 ubyte[] hash = xxh.digest("abc"); 90 assert(toHexString(hash) == "32D153FF", "Got " ~ toHexString(hash)); 91 92 //Feeding data 93 ubyte[1024] data; 94 xxh.put(data[]); 95 xxh.reset(); //Start again 96 xxh.put(data[]); 97 hash = xxh.finish(); 98 } 99 100 public import std.digest; 101 102 /* --- Port of C sources (release 0.8.1) to D language below ---------------- */ 103 104 enum XXH_NO_STREAM = false; 105 enum XXH_SIZE_OPT = 0; 106 enum XXH_FORCE_ALIGN_CHECK = true; 107 enum XXH32_ENDJMP = false; 108 109 private import core.bitop : rol, bswap; 110 private import std.exception : enforce; 111 private import object : Exception; 112 113 /** Thrown on XXH errors. */ 114 class XXHException : Exception 115 { 116 import std.exception : basicExceptionCtors; 117 /// 118 mixin basicExceptionCtors; 119 } 120 /// 121 @safe unittest 122 { 123 import std.exception : enforce, assertThrown; 124 assertThrown(enforce!XXHException(false, "Throw me...")); 125 } 126 127 /* ************************************* 128 * Misc 129 ***************************************/ 130 131 alias XXH32_hash_t = uint; 132 alias XXH64_hash_t = ulong; 133 align(16) struct XXH128_hash_t 134 { 135 XXH64_hash_t low64; /** `value & 0xFFFFFFFFFFFFFFFF` */ 136 XXH64_hash_t high64; /** `value >> 64` */ 137 } 138 139 alias XXH32_canonical_t = ubyte[XXH32_hash_t.sizeof]; 140 static assert(XXH32_canonical_t.sizeof == 4, "32bit integers should be 4 bytes?"); 141 alias XXH64_canonical_t = ubyte[XXH64_hash_t.sizeof]; 142 static assert(XXH64_hash_t.sizeof == 8, "64bit integers should be 8 bytes?"); 143 alias XXH128_canonical_t = ubyte[XXH128_hash_t.sizeof]; 144 static assert(XXH128_hash_t.sizeof == 16, "128bit integers should be 16 bytes?"); 145 146 147 enum XXH_VERSION_MAJOR = 0; 148 enum XXH_VERSION_MINOR = 8; 149 enum XXH_VERSION_RELEASE = 1; 150 151 /** Version number, encoded as two digits each */ 152 enum XXH_VERSION_NUMBER = 153 (XXH_VERSION_MAJOR * 100 * 100 + 154 XXH_VERSION_MINOR * 100 + 155 XXH_VERSION_RELEASE); 156 157 /** Get version number */ 158 uint xxh_versionNumber() @safe pure nothrow @nogc 159 { 160 return XXH_VERSION_NUMBER; 161 } 162 /// 163 @safe unittest 164 { 165 assert(XXH_VERSION_NUMBER == xxh_versionNumber(), "Version mismatch"); 166 } 167 168 enum XXH_errorcode 169 { 170 XXH_OK = 0, /** OK */ 171 XXH_ERROR /** Error */ 172 } 173 174 /* Structure for XXH32 streaming API. 175 * 176 * See: XXH64_state_s, XXH3_state_s 177 */ 178 struct XXH32_state_t 179 { 180 XXH32_hash_t total_len_32; /** Total length hashed, modulo 2^32 */ 181 XXH32_hash_t large_len; /** Whether the hash is >= 16 (handles total_len_32 overflow) */ 182 XXH32_hash_t[4] v; /** Accumulator lanes */ 183 XXH32_hash_t[4] mem32; /** Internal buffer for partial reads. Treated as unsigned char[16]. */ 184 XXH32_hash_t memsize; /** Amount of data in mem32 */ 185 XXH32_hash_t reserved; /** Reserved field. Do not read nor write to it. */ 186 } 187 188 /* Structure for XXH64 streaming API. 189 * 190 * See: XXH32_state_s, XXH3_state_s 191 */ 192 struct XXH64_state_t 193 { 194 XXH64_hash_t total_len; /** Total length hashed. This is always 64-bit. */ 195 XXH64_hash_t[4] v; /** Accumulator lanes */ 196 XXH64_hash_t[4] mem64; /** Internal buffer for partial reads. Treated as unsigned char[32]. */ 197 XXH32_hash_t memsize; /** Amount of data in mem64 */ 198 XXH32_hash_t reserved32; /** Reserved field, needed for padding anyways*/ 199 XXH64_hash_t reserved64; /** Reserved field. Do not read or write to it. */ 200 } /* typedef'd to XXH64_state_t */ 201 202 203 /* *************************** 204 * Memory reads 205 *****************************/ 206 207 /** Enum to indicate whether a pointer is aligned. 208 */ 209 private enum XXH_alignment 210 { 211 XXH_aligned, /** Aligned */ 212 XXH_unaligned /** Possibly unaligned */ 213 } 214 215 private uint xxh_read32(const void* ptr) @trusted pure nothrow @nogc 216 { 217 uint val; 218 version (HaveUnalignedLoads) 219 val = *(cast(uint*) ptr); 220 else 221 (cast(ubyte*)&val)[0 .. uint.sizeof] = (cast(ubyte*) ptr)[0 .. uint.sizeof]; 222 return val; 223 } 224 225 private uint xxh_readLE32(const void* ptr) @safe pure nothrow @nogc 226 { 227 version (LittleEndian) 228 return xxh_read32(ptr); 229 else 230 return bswap(xxh_read32(ptr)); 231 } 232 233 private uint xxh_readBE32(const void* ptr) @safe pure nothrow @nogc 234 { 235 version (LittleEndian) 236 return bswap(xxh_read32(ptr)); 237 else 238 return xxh_read32(ptr); 239 } 240 241 private uint xxh_readLE32_align(const void* ptr, XXH_alignment align_) @trusted pure nothrow @nogc 242 { 243 if (align_ == XXH_alignment.XXH_unaligned) 244 { 245 return xxh_readLE32(ptr); 246 } 247 else 248 { 249 version (LittleEndian) 250 return *cast(const uint*) ptr; 251 else 252 return bswap(*cast(const uint*) ptr); 253 } 254 } 255 256 /* ******************************************************************* 257 * 32-bit hash functions 258 *********************************************************************/ 259 260 enum XXH_PRIME32_1 = 0x9E3779B1U; /** 0b10011110001101110111100110110001 */ 261 enum XXH_PRIME32_2 = 0x85EBCA77U; /** 0b10000101111010111100101001110111 */ 262 enum XXH_PRIME32_3 = 0xC2B2AE3DU; /** 0b11000010101100101010111000111101 */ 263 enum XXH_PRIME32_4 = 0x27D4EB2FU; /** 0b00100111110101001110101100101111 */ 264 enum XXH_PRIME32_5 = 0x165667B1U; /** 0b00010110010101100110011110110001 */ 265 266 /** Normal stripe processing routine. 267 * 268 * This shuffles the bits so that any bit from @p input impacts several bits in 269 * acc. 270 * 271 * Param: acc = The accumulator lane. 272 * Param: input = The stripe of input to mix. 273 * Return: The mixed accumulator lane. 274 */ 275 private uint xxh32_round(uint acc, uint input) @safe pure nothrow @nogc 276 { 277 acc += input * XXH_PRIME32_2; 278 acc = rol(acc, 13); 279 acc *= XXH_PRIME32_1; 280 return acc; 281 } 282 283 /** Mixes all bits to finalize the hash. 284 * 285 * The final mix ensures that all input bits have a chance to impact any bit in 286 * the output digest, resulting in an unbiased distribution. 287 * 288 * Param: hash = The hash to avalanche. 289 * Return The avalanched hash. 290 */ 291 private uint xxh32_avalanche(uint hash) @safe pure nothrow @nogc 292 { 293 hash ^= hash >> 15; 294 hash *= XXH_PRIME32_2; 295 hash ^= hash >> 13; 296 hash *= XXH_PRIME32_3; 297 hash ^= hash >> 16; 298 return hash; 299 } 300 301 /* Alias wrapper for xxh_readLE32_align() */ 302 private uint xxh_get32bits(const void* p, XXH_alignment align_) @safe pure nothrow @nogc 303 { 304 return xxh_readLE32_align(p, align_); 305 } 306 307 /** Processes the last 0-15 bytes of @p ptr. 308 * 309 * There may be up to 15 bytes remaining to consume from the input. 310 * This final stage will digest them to ensure that all input bytes are present 311 * in the final mix. 312 * 313 * Param: hash = The hash to finalize. 314 * Param: ptr = The pointer to the remaining input. 315 * Param: len = The remaining length, modulo 16. 316 * Param: align = Whether @p ptr is aligned. 317 * Return: The finalized hash. 318 * See: xxh64_finalize(). 319 */ 320 private uint xxh32_finalize(uint hash, const(ubyte)* ptr, size_t len, XXH_alignment align_) 321 @trusted pure nothrow @nogc 322 { 323 static void XXH_PROCESS1(ref uint hash, ref const(ubyte)* ptr) 324 { 325 hash += (*ptr++) * XXH_PRIME32_5; 326 hash = rol(hash, 11) * XXH_PRIME32_1; 327 } 328 329 void XXH_PROCESS4(ref uint hash, ref const(ubyte)* ptr) 330 { 331 hash += xxh_get32bits(ptr, align_) * XXH_PRIME32_3; 332 ptr += 4; 333 hash = rol(hash, 17) * XXH_PRIME32_4; 334 } 335 336 /* Compact rerolled version; generally faster */ 337 if (!XXH32_ENDJMP) 338 { 339 len &= 15; 340 while (len >= 4) 341 { 342 XXH_PROCESS4(hash, ptr); 343 len -= 4; 344 } 345 while (len > 0) 346 { 347 XXH_PROCESS1(hash, ptr); 348 --len; 349 } 350 return xxh32_avalanche(hash); 351 } 352 else 353 { 354 switch (len & 15) /* or switch (bEnd - p) */ 355 { 356 case 12: 357 XXH_PROCESS4(hash, ptr); 358 goto case; 359 case 8: 360 XXH_PROCESS4(hash, ptr); 361 goto case; 362 case 4: 363 XXH_PROCESS4(hash, ptr); 364 return xxh32_avalanche(hash); 365 366 case 13: 367 XXH_PROCESS4(hash, ptr); 368 goto case; 369 case 9: 370 XXH_PROCESS4(hash, ptr); 371 goto case; 372 case 5: 373 XXH_PROCESS4(hash, ptr); 374 XXH_PROCESS1(hash, ptr); 375 return xxh32_avalanche(hash); 376 377 case 14: 378 XXH_PROCESS4(hash, ptr); 379 goto case; 380 case 10: 381 XXH_PROCESS4(hash, ptr); 382 goto case; 383 case 6: 384 XXH_PROCESS4(hash, ptr); 385 XXH_PROCESS1(hash, ptr); 386 XXH_PROCESS1(hash, ptr); 387 return xxh32_avalanche(hash); 388 389 case 15: 390 XXH_PROCESS4(hash, ptr); 391 goto case; 392 case 11: 393 XXH_PROCESS4(hash, ptr); 394 goto case; 395 case 7: 396 XXH_PROCESS4(hash, ptr); 397 goto case; 398 case 3: 399 XXH_PROCESS1(hash, ptr); 400 goto case; 401 case 2: 402 XXH_PROCESS1(hash, ptr); 403 goto case; 404 case 1: 405 XXH_PROCESS1(hash, ptr); 406 goto case; 407 case 0: 408 return xxh32_avalanche(hash); 409 default: 410 assert(0, "Internal error"); 411 } 412 return hash; /* reaching this point is deemed impossible */ 413 } 414 } 415 416 /** The implementation for XXH32(). 417 * 418 * Params: 419 * input = Directly passed from XXH32(). 420 * len = Ditto 421 * seed = Ditto 422 * align_ = Whether input is aligned. 423 * Return: The calculated hash. 424 */ 425 private uint xxh32_endian_align( 426 const(ubyte)* input, size_t len, uint seed, XXH_alignment align_) 427 @trusted pure nothrow @nogc 428 { 429 uint h32; 430 431 if (len >= 16) 432 { 433 const ubyte* bEnd = input + len; 434 const ubyte* limit = bEnd - 15; 435 uint v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; 436 uint v2 = seed + XXH_PRIME32_2; 437 uint v3 = seed + 0; 438 uint v4 = seed - XXH_PRIME32_1; 439 440 do 441 { 442 v1 = xxh32_round(v1, xxh_get32bits(input, align_)); 443 input += 4; 444 v2 = xxh32_round(v2, xxh_get32bits(input, align_)); 445 input += 4; 446 v3 = xxh32_round(v3, xxh_get32bits(input, align_)); 447 input += 4; 448 v4 = xxh32_round(v4, xxh_get32bits(input, align_)); 449 input += 4; 450 } 451 while (input < limit); 452 453 h32 = rol(v1, 1) + rol(v2, 7) + rol(v3, 12) + rol(v4, 18); 454 } 455 else 456 { 457 h32 = seed + XXH_PRIME32_5; 458 } 459 460 h32 += cast(uint) len; 461 462 return xxh32_finalize(h32, input, len & 15, align_); 463 } 464 465 /* XXH PUBLIC API - hidden in D module ! */ 466 /** Calculate a XXH32 digest on provided data 467 * 468 * Params: 469 * input = Pointer to data 470 * len = length of datablock 471 * seed = seed value 472 * Returns: a XXH32_hash_t 473 */ 474 private XXH32_hash_t XXH32(const void* input, size_t len, XXH32_hash_t seed) 475 @safe pure nothrow @nogc 476 { 477 static if (!XXH_NO_STREAM && XXH_SIZE_OPT >= 2) 478 { 479 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ 480 XXH32_state_t state; 481 auto rc = xxh32_reset(&state, seed); 482 if (rc == XXH_errorcode.XXH_OK) 483 { 484 rc = xxh32_update(&state, cast(const(ubyte)*) input, len); 485 if (rc == XXH_errorcode.XXH_OK) 486 { 487 return xxh32_digest(&state); 488 } 489 } 490 return XXH_errorcode.XXH_ERROR; 491 } 492 else 493 { 494 if (XXH_FORCE_ALIGN_CHECK) 495 { 496 if (((cast(size_t) input) & 3) == 0) 497 { /* Input is 4-bytes aligned, leverage the speed benefit */ 498 return xxh32_endian_align(cast(const(ubyte)*) input, len, 499 seed, XXH_alignment.XXH_aligned); 500 } 501 } 502 503 return xxh32_endian_align(cast(const(ubyte)*) input, len, seed, 504 XXH_alignment.XXH_unaligned); 505 } 506 } 507 508 /* XXH PUBLIC API - hidden in D module */ 509 /** Reset state with seed 510 * 511 * Params: 512 * state = Pointer to state structure 513 * seed = A seed value 514 * Returns: XXH_errorcode.OK or XXH_errorcode.FAIL 515 */ 516 private XXH_errorcode xxh32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) 517 @safe pure nothrow @nogc 518 in (statePtr != null, "statePtr is null") 519 { 520 *statePtr = XXH32_state_t.init; 521 statePtr.v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; 522 statePtr.v[1] = seed + XXH_PRIME32_2; 523 statePtr.v[2] = seed + 0; 524 statePtr.v[3] = seed - XXH_PRIME32_1; 525 return XXH_errorcode.XXH_OK; 526 } 527 528 /* XXH PUBLIC API - hidden in D module */ 529 /** Update the state with some more data 530 * 531 * Params: 532 * state = Pointer to state structure 533 * input = A pointer to data 534 * len = length of input data 535 * Returns: XXH_errorcode.OK or XXH_errorcode.FAIL 536 */ 537 private XXH_errorcode xxh32_update(XXH32_state_t* state, const void* input, size_t len) 538 @trusted pure nothrow @nogc 539 in 540 { 541 if (input == null) assert(len == 0, "input null ptr only allowed with len == 0"); 542 } 543 do 544 { 545 if (input == null && len == 0) 546 return XXH_errorcode.XXH_OK; 547 else if (input == null && len != 0) 548 return XXH_errorcode.XXH_ERROR; 549 else 550 { 551 const(ubyte)* p = cast(const(ubyte)*) input; 552 const ubyte* bEnd = p + len; 553 554 state.total_len_32 += cast(XXH32_hash_t) len; 555 state.large_len |= cast(XXH32_hash_t)((len >= 16) | (state.total_len_32 >= 16)); 556 557 if (state.memsize + len < 16) 558 { 559 /* fill in tmp buffer */ 560 (cast(ubyte*) state.mem32)[state.memsize .. state.memsize + len] = (cast(ubyte*) input)[0 .. len]; 561 state.memsize += cast(XXH32_hash_t) len; 562 return XXH_errorcode.XXH_OK; 563 } 564 565 if (state.memsize) 566 { 567 /* some data left from previous update */ 568 (cast(ubyte*) state.mem32)[state.memsize .. state.memsize + (16 - state.memsize)] = 569 (cast(ubyte*) input)[0 .. (16 - state.memsize)]; 570 { 571 const(uint)* p32 = cast(const(uint)*)&state.mem32[0]; 572 state.v[0] = xxh32_round(state.v[0], xxh_readLE32(p32)); 573 p32++; 574 state.v[1] = xxh32_round(state.v[1], xxh_readLE32(p32)); 575 p32++; 576 state.v[2] = xxh32_round(state.v[2], xxh_readLE32(p32)); 577 p32++; 578 state.v[3] = xxh32_round(state.v[3], xxh_readLE32(p32)); 579 } 580 p += 16 - state.memsize; 581 state.memsize = 0; 582 } 583 584 if (p <= bEnd - 16) 585 { 586 const ubyte* limit = bEnd - 16; 587 588 do 589 { 590 state.v[0] = xxh32_round(state.v[0], xxh_readLE32(p)); 591 p += 4; 592 state.v[1] = xxh32_round(state.v[1], xxh_readLE32(p)); 593 p += 4; 594 state.v[2] = xxh32_round(state.v[2], xxh_readLE32(p)); 595 p += 4; 596 state.v[3] = xxh32_round(state.v[3], xxh_readLE32(p)); 597 p += 4; 598 } 599 while (p <= limit); 600 601 } 602 603 if (p < bEnd) 604 { 605 (cast(ubyte*) state.mem32)[0 .. cast(size_t)(bEnd - p) ] = 606 (cast(ubyte*) p)[0 .. cast(size_t)(bEnd - p) ]; 607 state.memsize = cast(XXH32_hash_t)(bEnd - p); 608 } 609 } 610 611 return XXH_errorcode.XXH_OK; 612 } 613 614 /* XXH PUBLIC API - hidden in D module */ 615 /** Finalize state and return the final XXH32 digest 616 * 617 * Params: 618 * state = Pointer to state structure 619 * Returns: the final XXH32 digest 620 */ 621 private XXH32_hash_t xxh32_digest(const XXH32_state_t* state) 622 @trusted pure nothrow @nogc 623 { 624 uint h32; 625 626 if (state.large_len) 627 { 628 h32 = rol(state.v[0], 1) + rol(state.v[1], 7) + 629 rol(state.v[2], 12) + rol(state.v[3], 18); 630 } 631 else 632 { 633 h32 = state.v[2] /* == seed */ + XXH_PRIME32_5; 634 } 635 636 h32 += state.total_len_32; 637 638 return xxh32_finalize(h32, cast(const ubyte*) state.mem32, state.memsize, 639 XXH_alignment.XXH_aligned); 640 } 641 642 /* XXH PUBLIC API - hidden in D module */ 643 /** Covert the XXH32_hash_t to a byte array of same size 644 * 645 * Params: 646 * dst = Pointer to target storage 647 * hash = a XXH32_hash_t value 648 * Returns: nothing 649 */ 650 private void xxh32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) 651 @trusted pure nothrow @nogc 652 { 653 static assert((XXH32_canonical_t).sizeof == (XXH32_hash_t).sizeof, 654 "(XXH32_canonical_t).sizeof != (XXH32_hash_t).sizeof"); 655 version (LittleEndian) 656 hash = bswap(hash); 657 (cast(ubyte*) dst) [0 .. dst.sizeof] = (cast(ubyte*) &hash) [0 .. dst.sizeof]; 658 } 659 660 /* XXH PUBLIC API - hidden in D module */ 661 /** Covert the XXH32_hash_t to a byte array of same size 662 * 663 * Params: 664 * src = Pointer to source storage 665 * Returns: the converted value as XXH32_hash_t 666 */ 667 private XXH32_hash_t xxh32_hashFromCanonical(const XXH32_canonical_t* src) @safe pure nothrow @nogc 668 { 669 return xxh_readBE32(src); 670 } 671 672 /* ----------------------------------------------------------------------------------------*/ 673 674 /* Helper functions to read 64bit data quantities from memory follow */ 675 676 private ulong xxh_read64(const void* ptr) 677 @trusted pure nothrow @nogc 678 { 679 ulong val; 680 version (HaveUnalignedLoads) 681 val = *(cast(ulong*) ptr); 682 else 683 (cast(ubyte*)&val)[0 .. ulong.sizeof] = (cast(ubyte*) ptr)[0 .. ulong.sizeof]; 684 return val; 685 } 686 687 private ulong xxh_readLE64(const void* ptr) 688 @safe pure nothrow @nogc 689 { 690 version (LittleEndian) 691 return xxh_read64(ptr); 692 else 693 return bswap(xxh_read64(ptr)); 694 } 695 696 private ulong xxh_readBE64(const void* ptr) 697 @safe pure nothrow @nogc 698 { 699 version (LittleEndian) 700 return bswap(xxh_read64(ptr)); 701 else 702 return xxh_read64(ptr); 703 } 704 705 private ulong xxh_readLE64_align(const void* ptr, XXH_alignment align_) 706 @trusted pure nothrow @nogc 707 { 708 if (align_ == XXH_alignment.XXH_unaligned) 709 { 710 return xxh_readLE64(ptr); 711 } 712 else 713 { 714 version (LittleEndian) 715 return *cast(const ulong*) ptr; 716 else 717 return bswap(*cast(const ulong*) ptr); 718 } 719 } 720 721 enum XXH_PRIME64_1 = 0x9E3779B185EBCA87; /** 0b1001111000110111011110011011000110000101111010111100101010000111 */ 722 enum XXH_PRIME64_2 = 0xC2B2AE3D27D4EB4F; /** 0b1100001010110010101011100011110100100111110101001110101101001111 */ 723 enum XXH_PRIME64_3 = 0x165667B19E3779F9; /** 0b0001011001010110011001111011000110011110001101110111100111111001 */ 724 enum XXH_PRIME64_4 = 0x85EBCA77C2B2AE63; /** 0b1000010111101011110010100111011111000010101100101010111001100011 */ 725 enum XXH_PRIME64_5 = 0x27D4EB2F165667C5; /** 0b0010011111010100111010110010111100010110010101100110011111000101 */ 726 727 private ulong xxh64_round(ulong acc, ulong input) 728 @safe pure nothrow @nogc 729 { 730 acc += input * XXH_PRIME64_2; 731 acc = rol(acc, 31); 732 acc *= XXH_PRIME64_1; 733 return acc; 734 } 735 736 private ulong xxh64_mergeRound(ulong acc, ulong val) 737 @safe pure nothrow @nogc 738 { 739 val = xxh64_round(0, val); 740 acc ^= val; 741 acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; 742 return acc; 743 } 744 745 private ulong xxh64_avalanche(ulong hash) 746 @safe pure nothrow @nogc 747 { 748 hash ^= hash >> 33; 749 hash *= XXH_PRIME64_2; 750 hash ^= hash >> 29; 751 hash *= XXH_PRIME64_3; 752 hash ^= hash >> 32; 753 return hash; 754 } 755 756 ulong xxh_get64bits(const void* p, XXH_alignment align_) 757 @safe pure nothrow @nogc 758 { 759 return xxh_readLE64_align(p, align_); 760 } 761 762 /** Processes the last 0-31 bytes of data at ptr addr. 763 * 764 * There may be up to 31 bytes remaining to consume from the input. 765 * This final stage will digest them to ensure that all input bytes are present 766 * in the final mix. 767 * 768 * Param: hash = The hash to finalize. 769 * Param: ptr = The pointer to the remaining input. 770 * Param: len = The remaining length, modulo 32. 771 * Param: align = Whether @p ptr is aligned. 772 * Return: The finalized hash 773 * See: xxh32_finalize(). 774 */ 775 private ulong xxh64_finalize(ulong hash, const(ubyte)* ptr, size_t len, XXH_alignment align_) 776 @trusted pure nothrow @nogc 777 in 778 { 779 if (ptr == null) assert(len == 0, "input null ptr only allowed with len == 0"); 780 } 781 do 782 { 783 len &= 31; 784 while (len >= 8) 785 { 786 ulong k1 = xxh64_round(0, xxh_get64bits(ptr, align_)); 787 ptr += 8; 788 hash ^= k1; 789 hash = rol(hash, 27) * XXH_PRIME64_1 + XXH_PRIME64_4; 790 len -= 8; 791 } 792 if (len >= 4) 793 { 794 hash ^= cast(ulong)(xxh_get32bits(ptr, align_)) * XXH_PRIME64_1; 795 ptr += 4; 796 hash = rol(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; 797 len -= 4; 798 } 799 while (len > 0) 800 { 801 hash ^= (*ptr++) * XXH_PRIME64_5; 802 hash = rol(hash, 11) * XXH_PRIME64_1; 803 --len; 804 } 805 return xxh64_avalanche(hash); 806 } 807 808 /** The implementation for XXH64(). 809 * 810 * Params: 811 * input = pointer to input data, directly passed from XXH64() 812 * len = length of input data, directly passed from XXH64() 813 * seed = Seed value, directly passed from XXH64() 814 * align = Whether input pointer is aligned. 815 * Return: The calculated hash. 816 */ 817 private ulong xxh64_endian_align( 818 const(ubyte)* input, size_t len, 819 ulong seed, XXH_alignment align_) 820 @trusted pure nothrow @nogc 821 in 822 { 823 if (input == null) assert(len == 0, "input null ptr only allowed with len == 0"); 824 } 825 do 826 { 827 ulong h64; 828 829 if (len >= 32) 830 { 831 const ubyte* bEnd = input + len; 832 const ubyte* limit = bEnd - 31; 833 ulong v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; 834 ulong v2 = seed + XXH_PRIME64_2; 835 ulong v3 = seed + 0; 836 ulong v4 = seed - XXH_PRIME64_1; 837 838 do 839 { 840 v1 = xxh64_round(v1, xxh_get64bits(input, align_)); 841 input += 8; 842 v2 = xxh64_round(v2, xxh_get64bits(input, align_)); 843 input += 8; 844 v3 = xxh64_round(v3, xxh_get64bits(input, align_)); 845 input += 8; 846 v4 = xxh64_round(v4, xxh_get64bits(input, align_)); 847 input += 8; 848 } 849 while (input < limit); 850 851 h64 = rol(v1, 1) + rol(v2, 7) + rol(v3, 12) + rol(v4, 18); 852 h64 = xxh64_mergeRound(h64, v1); 853 h64 = xxh64_mergeRound(h64, v2); 854 h64 = xxh64_mergeRound(h64, v3); 855 h64 = xxh64_mergeRound(h64, v4); 856 857 } 858 else 859 { 860 h64 = seed + XXH_PRIME64_5; 861 } 862 863 h64 += cast(ulong) len; 864 865 return xxh64_finalize(h64, input, len, align_); 866 } 867 868 /* XXH PUBLIC API - hidden in D module */ 869 870 private XXH64_hash_t XXH64(const void* input, size_t len, XXH64_hash_t seed) 871 @safe pure nothrow @nogc 872 { 873 static if (!XXH_NO_STREAM && XXH_SIZE_OPT >= 2) 874 { 875 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ 876 XXH64_state_t state; 877 auto rc = xxh64_reset(&state, seed); 878 if (rc == XXH_errorcode.XXH_OK) 879 { 880 rc = xxh64_update(&state, cast(const(ubyte)*) input, len); 881 if (rc == XXH_errorcode.XXH_OK) 882 { 883 return xxh64_digest(&state); 884 } 885 } 886 return XXH_errorcode.XXH_ERROR; 887 } 888 else 889 { 890 if (XXH_FORCE_ALIGN_CHECK) 891 { 892 if (((cast(size_t) input) & 7) == 0) 893 { /* Input is aligned, let's leverage the speed advantage */ 894 return xxh64_endian_align(cast(const(ubyte)*) input, len, 895 seed, XXH_alignment.XXH_aligned); 896 } 897 } 898 899 return xxh64_endian_align(cast(const(ubyte)*) input, len, seed, 900 XXH_alignment.XXH_unaligned); 901 } 902 } 903 904 /* XXH PUBLIC API - hidden in D module */ 905 private XXH_errorcode xxh64_reset(XXH64_state_t* statePtr, XXH64_hash_t seed) 906 @safe pure nothrow @nogc 907 { 908 assert(statePtr != null, "statePtr == null"); 909 *statePtr = XXH64_state_t.init; 910 statePtr.v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; 911 statePtr.v[1] = seed + XXH_PRIME64_2; 912 statePtr.v[2] = seed + 0; 913 statePtr.v[3] = seed - XXH_PRIME64_1; 914 return XXH_errorcode.XXH_OK; 915 } 916 917 /* XXH PUBLIC API - hidden in D module */ 918 private XXH_errorcode xxh64_update(XXH64_state_t* state, const void* input, size_t len) 919 @trusted pure nothrow @nogc 920 in 921 { 922 if (input == null) assert(len == 0, "input null ptr only allowed with len == 0"); 923 } 924 do 925 { 926 if (input == null && len == 0) 927 return XXH_errorcode.XXH_OK; 928 else if (input == null && len != 0) 929 return XXH_errorcode.XXH_ERROR; 930 else 931 { 932 const(ubyte)* p = cast(const(ubyte)*) input; 933 const ubyte* bEnd = p + len; 934 935 state.total_len += len; 936 937 if (state.memsize + len < 32) 938 { 939 /* fill in tmp buffer */ 940 (cast(ubyte*) state.mem64) [state.memsize .. state.memsize + len] = 941 (cast(ubyte*) input) [0 .. len]; 942 state.memsize += cast(uint) len; 943 return XXH_errorcode.XXH_OK; 944 } 945 946 if (state.memsize) 947 { 948 /* tmp buffer is full */ 949 (cast(ubyte*) state.mem64) [state.memsize .. state.memsize + (32 - state.memsize)] = 950 (cast(ubyte*) input) [0 .. (32 - state.memsize)]; 951 state.v[0] = xxh64_round(state.v[0], xxh_readLE64(&state.mem64[0])); 952 state.v[1] = xxh64_round(state.v[1], xxh_readLE64(&state.mem64[1])); 953 state.v[2] = xxh64_round(state.v[2], xxh_readLE64(&state.mem64[2])); 954 state.v[3] = xxh64_round(state.v[3], xxh_readLE64(&state.mem64[3])); 955 p += 32 - state.memsize; 956 state.memsize = 0; 957 } 958 959 if (p + 32 <= bEnd) 960 { 961 const ubyte* limit = bEnd - 32; 962 963 do 964 { 965 state.v[0] = xxh64_round(state.v[0], xxh_readLE64(p)); 966 p += 8; 967 state.v[1] = xxh64_round(state.v[1], xxh_readLE64(p)); 968 p += 8; 969 state.v[2] = xxh64_round(state.v[2], xxh_readLE64(p)); 970 p += 8; 971 state.v[3] = xxh64_round(state.v[3], xxh_readLE64(p)); 972 p += 8; 973 } 974 while (p <= limit); 975 976 } 977 978 if (p < bEnd) 979 { 980 (cast(void*) &state.mem64[0]) [0 .. cast(size_t) (bEnd - p)] = 981 (cast(void*) p) [0 .. cast(size_t) (bEnd - p)]; 982 state.memsize = cast(XXH32_hash_t)(bEnd - p); 983 } 984 } 985 986 return XXH_errorcode.XXH_OK; 987 } 988 989 /* XXH PUBLIC API - hidden in D module */ 990 private XXH64_hash_t xxh64_digest(const XXH64_state_t* state) 991 @trusted pure nothrow @nogc 992 { 993 ulong h64; 994 995 if (state.total_len >= 32) 996 { 997 h64 = rol(state.v[0], 1) + rol(state.v[1], 998 7) + rol(state.v[2], 12) + rol(state.v[3], 18); 999 h64 = xxh64_mergeRound(h64, state.v[0]); 1000 h64 = xxh64_mergeRound(h64, state.v[1]); 1001 h64 = xxh64_mergeRound(h64, state.v[2]); 1002 h64 = xxh64_mergeRound(h64, state.v[3]); 1003 } 1004 else 1005 { 1006 h64 = state.v[2] /*seed*/ + XXH_PRIME64_5; 1007 } 1008 1009 h64 += cast(ulong) state.total_len; 1010 1011 return xxh64_finalize(h64, cast(const ubyte*) state.mem64, 1012 cast(size_t) state.total_len, XXH_alignment.XXH_aligned); 1013 } 1014 1015 1016 /* XXH PUBLIC API - hidden in D module */ 1017 private void xxh64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) 1018 @trusted pure nothrow @nogc 1019 { 1020 static assert((XXH64_canonical_t).sizeof == (XXH64_hash_t).sizeof, 1021 "(XXH64_canonical_t).sizeof != (XXH64_hash_t).sizeof"); 1022 version (LittleEndian) 1023 hash = bswap(hash); 1024 (cast(ubyte*) dst) [0 .. dst.sizeof] = (cast(ubyte*) &hash) [0 .. dst.sizeof]; 1025 } 1026 1027 /* XXH PUBLIC API - hidden in D module */ 1028 private XXH64_hash_t xxh64_hashFromCanonical(const XXH64_canonical_t* src) 1029 @safe pure nothrow @nogc 1030 { 1031 return xxh_readBE64(src); 1032 } 1033 1034 /* ********************************************************************* 1035 * XXH3 1036 * New generation hash designed for speed on small keys and vectorization 1037 ************************************************************************ */ 1038 1039 enum XXH3_SECRET_SIZE_MIN = 136; /// The bare minimum size for a custom secret. 1040 enum XXH3_SECRET_DEFAULT_SIZE = 192; /* minimum XXH3_SECRET_SIZE_MIN */ 1041 enum XXH_SECRET_DEFAULT_SIZE = 192; /* minimum XXH3_SECRET_SIZE_MIN */ 1042 enum XXH3_INTERNALBUFFER_SIZE = 256; ///The size of the internal XXH3 buffer. 1043 1044 /* Structure for XXH3 streaming API. 1045 * 1046 * Note: ** This structure has a strict alignment requirement of 64 bytes!! ** 1047 * Do not allocate this with `malloc()` or `new`, it will not be sufficiently aligned. 1048 * 1049 * Do never access the members of this struct directly. 1050 * 1051 * See: XXH3_INITSTATE() for stack initialization. 1052 * See: XXH32_state_s, XXH64_state_s 1053 */ 1054 private align(64) struct XXH3_state_t 1055 { 1056 align(64) XXH64_hash_t[8] acc; 1057 /** The 8 accumulators. See XXH32_state_s::v and XXH64_state_s::v */ 1058 align(64) ubyte[XXH3_SECRET_DEFAULT_SIZE] customSecret; 1059 /** Used to store a custom secret generated from a seed. */ 1060 align(64) ubyte[XXH3_INTERNALBUFFER_SIZE] buffer; 1061 /** The internal buffer. See: XXH32_state_s::mem32 */ 1062 XXH32_hash_t bufferedSize; 1063 /** The amount of memory in buffer, See: XXH32_state_s::memsize */ 1064 XXH32_hash_t useSeed; 1065 /** Reserved field. Needed for padding on 64-bit. */ 1066 size_t nbStripesSoFar; 1067 /** Number or stripes processed. */ 1068 XXH64_hash_t totalLen; 1069 /** Total length hashed. 64-bit even on 32-bit targets. */ 1070 size_t nbStripesPerBlock; 1071 /** Number of stripes per block. */ 1072 size_t secretLimit; 1073 /** Size of customSecret or extSecret */ 1074 XXH64_hash_t seed; 1075 /** Seed for _withSeed variants. Must be zero otherwise, See: XXH3_INITSTATE() */ 1076 XXH64_hash_t reserved64; 1077 /** Reserved field. */ 1078 const(ubyte)* extSecret; 1079 /** Reference to an external secret for the _withSecret variants, null 1080 * for other variants. */ 1081 /* note: there may be some padding at the end due to alignment on 64 bytes */ 1082 } /* typedef'd to XXH3_state_t */ 1083 1084 static assert(XXH_SECRET_DEFAULT_SIZE >= XXH3_SECRET_SIZE_MIN, "default keyset is not large enough"); 1085 1086 /** Pseudorandom secret taken directly from FARSH. */ 1087 private align(64) immutable ubyte[XXH3_SECRET_DEFAULT_SIZE] xxh3_kSecret = [ 1088 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 1089 0xf7, 0x21, 0xad, 0x1c, 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 1090 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, 0xcb, 0x79, 0xe6, 0x4e, 1091 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, 1092 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 1093 0x81, 0x3a, 0x26, 0x4c, 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 1094 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, 0x71, 0x64, 0x48, 0x97, 1095 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, 1096 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 1097 0xc7, 0x0b, 0x4f, 0x1d, 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 1098 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, 0xea, 0xc5, 0xac, 0x83, 1099 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, 1100 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 1101 0x29, 0xd4, 0x68, 0x9e, 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 1102 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, 0x45, 0xcb, 0x3a, 0x8f, 1103 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, 1104 ]; 1105 1106 /* This performs a 32x32 -> 64 bit multiplikation */ 1107 private ulong xxh_mult32to64(uint x, uint y) @safe pure nothrow @nogc 1108 { 1109 return (cast(ulong)(x) * cast(ulong)(y)); 1110 } 1111 1112 /** Calculates a 64 to 128-bit long multiply. 1113 * 1114 * Param: lhs , rhs The 64-bit integers to be multiplied 1115 * Return: The 128-bit result represented in an XXH128_hash_t structure. 1116 */ 1117 private XXH128_hash_t xxh_mult64to128(ulong lhs, ulong rhs) @safe pure nothrow @nogc 1118 { 1119 version (Have128BitInteger) 1120 { 1121 Cent cent_lhs; cent_lhs.lo = lhs; 1122 Cent cent_rhs; cent_rhs.lo = rhs; 1123 const Cent product = mul(cent_lhs, cent_rhs); 1124 XXH128_hash_t r128; 1125 r128.low64 = product.lo; 1126 r128.high64 = product.hi; 1127 } 1128 else 1129 { 1130 /* First calculate all of the cross products. */ 1131 const ulong lo_lo = xxh_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); 1132 const ulong hi_lo = xxh_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); 1133 const ulong lo_hi = xxh_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); 1134 const ulong hi_hi = xxh_mult32to64(lhs >> 32, rhs >> 32); 1135 1136 /* Now add the products together. These will never overflow. */ 1137 const ulong cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; 1138 const ulong upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; 1139 const ulong lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); 1140 1141 XXH128_hash_t r128; 1142 r128.low64 = lower; 1143 r128.high64 = upper; 1144 } 1145 return r128; 1146 } 1147 1148 /** Calculates a 64-bit to 128-bit multiply, then XOR folds it. 1149 * 1150 * The reason for the separate function is to prevent passing too many structs 1151 * around by value. This will hopefully inline the multiply, but we don't force it. 1152 * 1153 * Param: lhs , rhs The 64-bit integers to multiply 1154 * Return: The low 64 bits of the product XOR'd by the high 64 bits. 1155 * See: xxh_mult64to128() 1156 */ 1157 private ulong xxh3_mul128_fold64(ulong lhs, ulong rhs) @safe pure nothrow @nogc 1158 { 1159 XXH128_hash_t product = xxh_mult64to128(lhs, rhs); 1160 return product.low64 ^ product.high64; 1161 } 1162 1163 /* Seems to produce slightly better code on GCC for some reason. */ 1164 static private ulong xxh_xorshift64(ulong v64, int shift) @safe pure nothrow @nogc 1165 in(0 <= shift && shift < 64, "shift out of range") 1166 { 1167 return v64 ^ (v64 >> shift); 1168 } 1169 1170 /* 1171 * This is a fast avalanche stage, 1172 * suitable when input bits are already partially mixed 1173 */ 1174 static private XXH64_hash_t xxh3_avalanche(ulong h64) @safe pure nothrow @nogc 1175 { 1176 h64 = xxh_xorshift64(h64, 37); 1177 h64 *= 0x165667919E3779F9; 1178 h64 = xxh_xorshift64(h64, 32); 1179 return h64; 1180 } 1181 1182 /* 1183 * This is a stronger avalanche, 1184 * inspired by Pelle Evensen's rrmxmx 1185 * preferable when input has not been previously mixed 1186 */ 1187 static private XXH64_hash_t xxh3_rrmxmx(ulong h64, ulong len) @safe pure nothrow @nogc 1188 { 1189 /* this mix is inspired by Pelle Evensen's rrmxmx */ 1190 h64 ^= rol(h64, 49) ^ rol(h64, 24); 1191 h64 *= 0x9FB21C651E98DF25; 1192 h64 ^= (h64 >> 35) + len; 1193 h64 *= 0x9FB21C651E98DF25; 1194 return xxh_xorshift64(h64, 28); 1195 } 1196 1197 /* ========================================== 1198 * Short keys 1199 * ========================================== 1200 * One of the shortcomings of XXH32 and XXH64 was that their performance was 1201 * sub-optimal on short lengths. It used an iterative algorithm which strongly 1202 * favored lengths that were a multiple of 4 or 8. 1203 * 1204 * Instead of iterating over individual inputs, we use a set of single shot 1205 * functions which piece together a range of lengths and operate in constant time. 1206 * 1207 * Additionally, the number of multiplies has been significantly reduced. This 1208 * reduces latency, especially when emulating 64-bit multiplies on 32-bit. 1209 * 1210 * Depending on the platform, this may or may not be faster than XXH32, but it 1211 * is almost guaranteed to be faster than XXH64. 1212 */ 1213 1214 /* 1215 * At very short lengths, there isn't enough input to fully hide secrets, or use 1216 * the entire secret. 1217 * 1218 * There is also only a limited amount of mixing we can do before significantly 1219 * impacting performance. 1220 * 1221 * Therefore, we use different sections of the secret and always mix two secret 1222 * samples with an XOR. This should have no effect on performance on the 1223 * seedless or withSeed variants because everything _should_ be constant folded 1224 * by modern compilers. 1225 * 1226 * The XOR mixing hides individual parts of the secret and increases entropy. 1227 * 1228 * This adds an extra layer of strength for custom secrets. 1229 */ 1230 private XXH64_hash_t xxh3_len_1to3_64b( 1231 const ubyte* input, size_t len, const ubyte* secret, XXH64_hash_t seed) 1232 @trusted pure nothrow @nogc 1233 in(input != null, "input == null") 1234 in(1 <= len && len <= 3, "len out of range") 1235 in(secret != null, "secret == null") 1236 { 1237 /* 1238 * len = 1: combined = { input[0], 0x01, input[0], input[0] } 1239 * len = 2: combined = { input[1], 0x02, input[0], input[1] } 1240 * len = 3: combined = { input[2], 0x03, input[0], input[1] } 1241 */ 1242 { 1243 const ubyte c1 = input[0]; 1244 const ubyte c2 = input[len >> 1]; 1245 const ubyte c3 = input[len - 1]; 1246 const uint combined = (cast(uint) c1 << 16) | ( 1247 cast(uint) c2 << 24) | (cast(uint) c3 << 0) | (cast(uint) len << 8); 1248 const ulong bitflip = (xxh_readLE32(secret) ^ xxh_readLE32(secret + 4)) + seed; 1249 const ulong keyed = cast(ulong) combined ^ bitflip; 1250 return xxh64_avalanche(keyed); 1251 } 1252 } 1253 1254 private XXH64_hash_t xxh3_len_4to8_64b( 1255 const ubyte* input, size_t len, const ubyte* secret, XXH64_hash_t seed) 1256 @trusted pure nothrow @nogc 1257 in(input != null, "input == null") 1258 in(secret != null, "secret == null") 1259 in(4 <= len && len <= 8, "len out of range") 1260 { 1261 seed ^= cast(ulong) bswap(cast(uint) seed) << 32; 1262 { 1263 const uint input1 = xxh_readLE32(input); 1264 const uint input2 = xxh_readLE32(input + len - 4); 1265 const ulong bitflip = (xxh_readLE64(secret + 8) ^ xxh_readLE64(secret + 16)) - seed; 1266 const ulong input64 = input2 + ((cast(ulong) input1) << 32); 1267 const ulong keyed = input64 ^ bitflip; 1268 return xxh3_rrmxmx(keyed, len); 1269 } 1270 } 1271 1272 private XXH64_hash_t xxh3_len_9to16_64b( 1273 const ubyte* input, size_t len, const ubyte* secret, XXH64_hash_t seed) 1274 @trusted pure nothrow @nogc 1275 in(input != null, "input == null") 1276 in(secret != null, "secret == null") 1277 in(9 <= len && len <= 16, "len out of range") 1278 { 1279 { 1280 const ulong bitflip1 = (xxh_readLE64(secret + 24) ^ xxh_readLE64(secret + 32)) + seed; 1281 const ulong bitflip2 = (xxh_readLE64(secret + 40) ^ xxh_readLE64(secret + 48)) - seed; 1282 const ulong input_lo = xxh_readLE64(input) ^ bitflip1; 1283 const ulong input_hi = xxh_readLE64(input + len - 8) ^ bitflip2; 1284 const ulong acc = len + bswap(input_lo) + input_hi + xxh3_mul128_fold64(input_lo, 1285 input_hi); 1286 return xxh3_avalanche(acc); 1287 } 1288 } 1289 1290 private bool xxh_likely(bool exp) 1291 @safe pure nothrow @nogc 1292 { 1293 return exp; 1294 } 1295 1296 private bool xxh_unlikely(bool exp) 1297 @safe pure nothrow @nogc 1298 { 1299 return exp; 1300 } 1301 1302 private XXH64_hash_t xxh3_len_0to16_64b( 1303 const ubyte* input, size_t len, const ubyte* secret, XXH64_hash_t seed) 1304 @trusted pure nothrow @nogc 1305 in(len <= 16, "len > 16") 1306 { 1307 { 1308 if (xxh_likely(len > 8)) 1309 return xxh3_len_9to16_64b(input, len, secret, seed); 1310 if (xxh_likely(len >= 4)) 1311 return xxh3_len_4to8_64b(input, len, secret, seed); 1312 if (len) 1313 return xxh3_len_1to3_64b(input, len, secret, seed); 1314 return xxh64_avalanche(seed ^ (xxh_readLE64(secret + 56) ^ xxh_readLE64(secret + 64))); 1315 } 1316 } 1317 1318 /* 1319 * DISCLAIMER: There are known *seed-dependent* multicollisions here due to 1320 * multiplication by zero, affecting hashes of lengths 17 to 240. 1321 * 1322 * However, they are very unlikely. 1323 * 1324 * Keep this in mind when using the unseeded xxh3_64bits() variant: As with all 1325 * unseeded non-cryptographic hashes, it does not attempt to defend itself 1326 * against specially crafted inputs, only random inputs. 1327 * 1328 * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes 1329 * cancelling out the secret is taken an arbitrary number of times (addressed 1330 * in xxh3_accumulate_512), this collision is very unlikely with random inputs 1331 * and/or proper seeding: 1332 * 1333 * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a 1334 * function that is only called up to 16 times per hash with up to 240 bytes of 1335 * input. 1336 * 1337 * This is not too bad for a non-cryptographic hash function, especially with 1338 * only 64 bit outputs. 1339 * 1340 * The 128-bit variant (which trades some speed for strength) is NOT affected 1341 * by this, although it is always a good idea to use a proper seed if you care 1342 * about strength. 1343 */ 1344 private ulong xxh3_mix16B(const(ubyte)* input, const(ubyte)* secret, ulong seed64) 1345 @trusted pure nothrow @nogc 1346 { 1347 { 1348 const ulong input_lo = xxh_readLE64(input); 1349 const ulong input_hi = xxh_readLE64(input + 8); 1350 return xxh3_mul128_fold64( 1351 input_lo ^ (xxh_readLE64(secret) + seed64), 1352 input_hi ^ (xxh_readLE64(secret + 8) - seed64)); 1353 } 1354 } 1355 1356 /* For mid range keys, XXH3 uses a Mum-hash variant. */ 1357 private XXH64_hash_t xxh3_len_17to128_64b( 1358 const(ubyte)* input, size_t len, const(ubyte)* secret, size_t secretSize, XXH64_hash_t seed) 1359 @trusted pure nothrow @nogc 1360 in(secretSize >= XXH3_SECRET_SIZE_MIN, "secretSize < XXH3_SECRET_SIZE_MIN") 1361 in(16 < len && len <= 128, "len out of range") 1362 { 1363 ulong acc = len * XXH_PRIME64_1; 1364 if (len > 32) 1365 { 1366 if (len > 64) 1367 { 1368 if (len > 96) 1369 { 1370 acc += xxh3_mix16B(input + 48, secret + 96, seed); 1371 acc += xxh3_mix16B(input + len - 64, secret + 112, seed); 1372 } 1373 acc += xxh3_mix16B(input + 32, secret + 64, seed); 1374 acc += xxh3_mix16B(input + len - 48, secret + 80, seed); 1375 } 1376 acc += xxh3_mix16B(input + 16, secret + 32, seed); 1377 acc += xxh3_mix16B(input + len - 32, secret + 48, seed); 1378 } 1379 acc += xxh3_mix16B(input + 0, secret + 0, seed); 1380 acc += xxh3_mix16B(input + len - 16, secret + 16, seed); 1381 1382 return xxh3_avalanche(acc); 1383 } 1384 1385 enum XXH3_MIDSIZE_MAX = 240; 1386 enum XXH3_MIDSIZE_STARTOFFSET = 3; 1387 enum XXH3_MIDSIZE_LASTOFFSET = 17; 1388 1389 private XXH64_hash_t xxh3_len_129to240_64b( 1390 const(ubyte)* input, size_t len, const(ubyte)* secret, size_t secretSize, XXH64_hash_t seed) 1391 @trusted pure nothrow @nogc 1392 in(secretSize >= XXH3_SECRET_SIZE_MIN, "secretSize < XXH3_SECRET_SIZE_MIN") 1393 in { const int nbRounds = cast(int) len / 16; assert(nbRounds >= 8, "nbRounds < 8"); } 1394 in(128 < len && len <= XXH3_MIDSIZE_MAX, "128 >= len || len > XXH3_MIDSIZE_MAX") 1395 { 1396 ulong acc = len * XXH_PRIME64_1; 1397 const int nbRounds = cast(int) len / 16; 1398 int i; 1399 for (i = 0; i < 8; i++) 1400 { 1401 acc += xxh3_mix16B(input + (16 * i), secret + (16 * i), seed); 1402 } 1403 acc = xxh3_avalanche(acc); 1404 for (i = 8; i < nbRounds; i++) 1405 { 1406 acc += xxh3_mix16B(input + (16 * i), 1407 secret + (16 * (i - 8)) + XXH3_MIDSIZE_STARTOFFSET, seed); 1408 } 1409 /* last bytes */ 1410 acc += xxh3_mix16B(input + len - 16, 1411 secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); 1412 return xxh3_avalanche(acc); 1413 } 1414 1415 /* ======= Long Keys ======= */ 1416 1417 enum XXH_STRIPE_LEN = 64; 1418 enum XXH_SECRET_CONSUME_RATE = 8; /* nb of secret bytes consumed at each accumulation */ 1419 enum XXH_ACC_NB = (XXH_STRIPE_LEN / (ulong).sizeof); 1420 1421 private void xxh_writeLE64(void* dst, ulong v64) 1422 @trusted pure nothrow @nogc 1423 { 1424 version (LittleEndian) {} 1425 else 1426 v64 = bswap(v64); 1427 (cast(ubyte*) dst) [0 .. v64.sizeof] = (cast(ubyte*) &v64) [0 .. v64.sizeof]; 1428 } 1429 1430 /* scalar variants - universal */ 1431 1432 enum XXH_ACC_ALIGN = 8; 1433 1434 /* Scalar round for xxh3_accumulate_512_scalar(). */ 1435 private void xxh3_scalarRound(void* acc, const(void)* input, const(void)* secret, size_t lane) 1436 @trusted pure nothrow @nogc 1437 in(lane < XXH_ACC_NB, "lane >= XXH_ACC_NB") 1438 { 1439 version (CheckACCAlignment) 1440 assert((cast(size_t) acc & (XXH_ACC_ALIGN - 1)) == 0, "(cast(size_t) acc & (XXH_ACC_ALIGN - 1)) != 0"); 1441 ulong* xacc = cast(ulong*) acc; 1442 ubyte* xinput = cast(ubyte*) input; 1443 ubyte* xsecret = cast(ubyte*) secret; 1444 { 1445 const ulong data_val = xxh_readLE64(xinput + lane * 8); 1446 const ulong data_key = data_val ^ xxh_readLE64(xsecret + lane * 8); 1447 xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ 1448 xacc[lane] += xxh_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32); 1449 } 1450 } 1451 1452 /* Processes a 64 byte block of data using the scalar path. */ 1453 private void xxh3_accumulate_512_scalar(void* acc, const(void)* input, const(void)* secret) 1454 @safe pure nothrow @nogc 1455 { 1456 size_t i; 1457 for (i = 0; i < XXH_ACC_NB; i++) 1458 { 1459 xxh3_scalarRound(acc, input, secret, i); 1460 } 1461 } 1462 1463 /* Scalar scramble step for xxh3_scrambleAcc_scalar(). 1464 * 1465 * This is extracted to its own function because the NEON path uses a combination 1466 * of NEON and scalar. 1467 */ 1468 private void xxh3_scalarScrambleRound(void* acc, const(void)* secret, size_t lane) 1469 @trusted pure nothrow @nogc 1470 in(lane < XXH_ACC_NB, "lane >= XXH_ACC_NB") 1471 { 1472 version (CheckACCAlignment) 1473 assert(((cast(size_t) acc) & (XXH_ACC_ALIGN - 1)) == 0, "((cast(size_t) acc) & (XXH_ACC_ALIGN - 1)) != 0"); 1474 ulong* xacc = cast(ulong*) acc; /* presumed aligned */ 1475 const ubyte* xsecret = cast(const ubyte*) secret; /* no alignment restriction */ 1476 { 1477 const ulong key64 = xxh_readLE64(xsecret + lane * 8); 1478 ulong acc64 = xacc[lane]; 1479 acc64 = xxh_xorshift64(acc64, 47); 1480 acc64 ^= key64; 1481 acc64 *= XXH_PRIME32_1; 1482 xacc[lane] = acc64; 1483 } 1484 } 1485 1486 /* Scrambles the accumulators after a large chunk has been read */ 1487 private void xxh3_scrambleAcc_scalar(void* acc, const(void)* secret) 1488 @safe pure nothrow @nogc 1489 { 1490 size_t i; 1491 for (i = 0; i < XXH_ACC_NB; i++) 1492 { 1493 xxh3_scalarScrambleRound(acc, secret, i); 1494 } 1495 } 1496 1497 private void xxh3_initCustomSecret_scalar(void* customSecret, ulong seed64) 1498 @trusted pure nothrow @nogc 1499 { 1500 /* 1501 * We need a separate pointer for the hack below, 1502 * which requires a non-const pointer. 1503 * Any decent compiler will optimize this out otherwise. 1504 */ 1505 const ubyte* kSecretPtr = cast(ubyte*) xxh3_kSecret; 1506 static assert((XXH_SECRET_DEFAULT_SIZE & 15) == 0, "(XXH_SECRET_DEFAULT_SIZE & 15) != 0"); 1507 1508 /* 1509 * Note: in debug mode, this overrides the asm optimization 1510 * and Clang will emit MOVK chains again. 1511 */ 1512 //assert(kSecretPtr == xxh3_kSecret); 1513 1514 { 1515 const int nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; 1516 int i; 1517 for (i = 0; i < nbRounds; i++) 1518 { 1519 /* 1520 * The asm hack causes Clang to assume that kSecretPtr aliases with 1521 * customSecret, and on aarch64, this prevented LDP from merging two 1522 * loads together for free. Putting the loads together before the stores 1523 * properly generates LDP. 1524 */ 1525 ulong lo = xxh_readLE64(kSecretPtr + 16 * i) + seed64; 1526 ulong hi = xxh_readLE64(kSecretPtr + 16 * i + 8) - seed64; 1527 xxh_writeLE64(cast(ubyte*) customSecret + 16 * i, lo); 1528 xxh_writeLE64(cast(ubyte*) customSecret + 16 * i + 8, hi); 1529 } 1530 } 1531 } 1532 1533 alias XXH3_f_accumulate_512 = void function(void*, const(void)*, const(void)*) @safe pure nothrow @nogc; 1534 alias XXH3_f_scrambleAcc = void function(void*, const void*) @safe pure nothrow @nogc; 1535 alias XXH3_f_initCustomSecret = void function(void*, ulong) @safe pure nothrow @nogc; 1536 1537 immutable XXH3_f_accumulate_512 xxh3_accumulate_512 = &xxh3_accumulate_512_scalar; 1538 immutable XXH3_f_scrambleAcc xxh3_scrambleAcc = &xxh3_scrambleAcc_scalar; 1539 immutable XXH3_f_initCustomSecret xxh3_initCustomSecret = &xxh3_initCustomSecret_scalar; 1540 1541 enum XXH_PREFETCH_DIST = 384; 1542 /* TODO: Determine how to implement prefetching in D! Disabled for now */ 1543 private void XXH_PREFETCH(const ubyte* ptr) @safe pure nothrow @nogc 1544 { 1545 // cast(void)(ptr); /* DISABLED prefetch and do nothing here */ 1546 1547 // In C it is done with the following code lines: 1548 // if XXH_SIZE_OPT >= 1 1549 // define XXH_PREFETCH(ptr) (void)(ptr) 1550 // elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ 1551 // include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ 1552 // define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) 1553 // elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) 1554 // define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw == read */, 3 /* locality */) 1555 // else 1556 // define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ 1557 // endif 1558 } 1559 1560 /* 1561 * xxh3_accumulate() 1562 * Loops over xxh3_accumulate_512(). 1563 * Assumption: nbStripes will not overflow the secret size 1564 */ 1565 private void xxh3_accumulate( 1566 ulong* acc, const ubyte* input, 1567 const ubyte* secret, size_t nbStripes, XXH3_f_accumulate_512 f_acc512) 1568 @trusted pure nothrow @nogc 1569 { 1570 size_t n; 1571 for (n = 0; n < nbStripes; n++) 1572 { 1573 const ubyte* in_ = input + n * XXH_STRIPE_LEN; 1574 XXH_PREFETCH(in_ + XXH_PREFETCH_DIST); 1575 f_acc512(acc, in_, secret + n * XXH_SECRET_CONSUME_RATE); 1576 } 1577 } 1578 1579 private void xxh3_hashLong_internal_loop( 1580 ulong* acc, const ubyte* input, size_t len, const ubyte* secret, 1581 size_t secretSize, XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble) 1582 @trusted pure nothrow @nogc 1583 in(secretSize >= XXH3_SECRET_SIZE_MIN, "secretSize < XXH3_SECRET_SIZE_MIN") 1584 in(len > XXH_STRIPE_LEN, "len <= XXH_STRIPE_LEN") 1585 { 1586 const size_t nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; 1587 const size_t block_len = XXH_STRIPE_LEN * nbStripesPerBlock; 1588 const size_t nb_blocks = (len - 1) / block_len; 1589 1590 size_t n; 1591 1592 for (n = 0; n < nb_blocks; n++) 1593 { 1594 xxh3_accumulate(acc, input + n * block_len, secret, nbStripesPerBlock, f_acc512); 1595 f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); 1596 } 1597 1598 /* last partial block */ 1599 { 1600 const size_t nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; 1601 assert(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE), 1602 "nbStripes > (secretSize / XXH_SECRET_CONSUME_RATE)"); 1603 xxh3_accumulate(acc, input + nb_blocks * block_len, secret, nbStripes, f_acc512); 1604 1605 /* last stripe */ 1606 { 1607 const ubyte* p = input + len - XXH_STRIPE_LEN; 1608 f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); 1609 } 1610 } 1611 } 1612 1613 enum XXH_SECRET_LASTACC_START = 7; /* not aligned on 8, last secret is different from acc & scrambler */ 1614 1615 private ulong xxh3_mix2Accs(const(ulong)* acc, const(ubyte)* secret) 1616 @trusted pure nothrow @nogc 1617 { 1618 return xxh3_mul128_fold64(acc[0] ^ xxh_readLE64(secret), acc[1] ^ xxh_readLE64(secret + 8)); 1619 } 1620 1621 private XXH64_hash_t xxh3_mergeAccs(const(ulong)* acc, const(ubyte)* secret, ulong start) 1622 @trusted pure nothrow @nogc 1623 { 1624 ulong result64 = start; 1625 size_t i = 0; 1626 1627 for (i = 0; i < 4; i++) 1628 { 1629 result64 += xxh3_mix2Accs(acc + 2 * i, secret + 16 * i); 1630 } 1631 1632 return xxh3_avalanche(result64); 1633 } 1634 1635 static immutable XXH3_INIT_ACC = [ 1636 XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, 1637 XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 1638 ]; 1639 1640 private XXH64_hash_t xxh3_hashLong_64b_internal( 1641 const(void)* input, size_t len, const(void)* secret, size_t secretSize, 1642 XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble) 1643 @trusted pure nothrow @nogc 1644 { 1645 align(XXH_ACC_ALIGN) ulong[XXH_ACC_NB] acc = XXH3_INIT_ACC; /* NOTE: This doesn't work in D, fails on 32bit NetBSD */ 1646 1647 xxh3_hashLong_internal_loop(&acc[0], cast(const(ubyte)*) input, len, 1648 cast(const(ubyte)*) secret, secretSize, f_acc512, f_scramble); 1649 1650 /* converge into final hash */ 1651 static assert(acc.sizeof == 64, "acc.sizeof != 64"); 1652 /* do not align on 8, so that the secret is different from the accumulator */ 1653 assert(secretSize >= acc.sizeof + XXH_SECRET_MERGEACCS_START, 1654 "secretSize < acc.sizeof + XXH_SECRET_MERGEACCS_START"); 1655 return xxh3_mergeAccs(&acc[0], cast(const(ubyte)*) secret + XXH_SECRET_MERGEACCS_START, 1656 cast(ulong) len * XXH_PRIME64_1); 1657 } 1658 1659 enum XXH_SECRET_MERGEACCS_START = 11; 1660 1661 /* 1662 * It's important for performance to transmit secret's size (when it's static) 1663 * so that the compiler can properly optimize the vectorized loop. 1664 * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. 1665 */ 1666 private XXH64_hash_t xxh3_hashLong_64b_withSecret( 1667 const(void)* input, size_t len, XXH64_hash_t seed64, const(ubyte)* secret, size_t secretLen) 1668 @safe pure nothrow @nogc 1669 { 1670 return xxh3_hashLong_64b_internal(input, len, secret, secretLen, 1671 xxh3_accumulate_512, xxh3_scrambleAcc); 1672 } 1673 1674 /* 1675 * It's preferable for performance that XXH3_hashLong is not inlined, 1676 * as it results in a smaller function for small data, easier to the instruction cache. 1677 * Note that inside this no_inline function, we do inline the internal loop, 1678 * and provide a statically defined secret size to allow optimization of vector loop. 1679 */ 1680 private XXH64_hash_t xxh3_hashLong_64b_default( 1681 const(void)* input, size_t len, XXH64_hash_t seed64, const(ubyte)* secret, size_t secretLen) 1682 @safe pure nothrow @nogc 1683 { 1684 return xxh3_hashLong_64b_internal(input, len, &xxh3_kSecret[0], 1685 (xxh3_kSecret).sizeof, xxh3_accumulate_512, xxh3_scrambleAcc); 1686 } 1687 1688 enum XXH_SEC_ALIGN = 8; 1689 1690 /* 1691 * xxh3_hashLong_64b_withSeed(): 1692 * Generate a custom key based on alteration of default xxh3_kSecret with the seed, 1693 * and then use this key for long mode hashing. 1694 * 1695 * This operation is decently fast but nonetheless costs a little bit of time. 1696 * Try to avoid it whenever possible (typically when seed == 0). 1697 * 1698 * It's important for performance that XXH3_hashLong is not inlined. Not sure 1699 * why (uop cache maybe?), but the difference is large and easily measurable. 1700 */ 1701 private XXH64_hash_t xxh3_hashLong_64b_withSeed_internal( 1702 const(void)* input, size_t len, XXH64_hash_t seed, 1703 XXH3_f_accumulate_512 f_acc512, 1704 XXH3_f_scrambleAcc f_scramble, 1705 XXH3_f_initCustomSecret f_initSec) 1706 @trusted pure nothrow @nogc 1707 { 1708 //#if XXH_SIZE_OPT <= 0 1709 if (seed == 0) 1710 return xxh3_hashLong_64b_internal(input, len, &xxh3_kSecret[0], 1711 (xxh3_kSecret).sizeof, f_acc512, f_scramble); 1712 //#endif 1713 else 1714 { 1715 align(XXH_SEC_ALIGN) ubyte[XXH_SECRET_DEFAULT_SIZE] secret; 1716 f_initSec(&secret[0], seed); 1717 return xxh3_hashLong_64b_internal(input, len, &secret[0], 1718 (secret).sizeof, f_acc512, f_scramble); 1719 } 1720 } 1721 1722 /* 1723 * It's important for performance that XXH3_hashLong is not inlined. 1724 */ 1725 private XXH64_hash_t xxh3_hashLong_64b_withSeed(const(void)* input, size_t len, 1726 XXH64_hash_t seed, const(ubyte)* secret, size_t secretLen) 1727 @safe pure nothrow @nogc 1728 { 1729 return xxh3_hashLong_64b_withSeed_internal(input, len, seed, 1730 xxh3_accumulate_512, xxh3_scrambleAcc, xxh3_initCustomSecret); 1731 } 1732 1733 alias XXH3_hashLong64_f = XXH64_hash_t function( 1734 const(void)*, size_t, XXH64_hash_t, const(ubyte)*, size_t) 1735 @safe pure nothrow @nogc; 1736 1737 private XXH64_hash_t xxh3_64bits_internal(const(void)* input, size_t len, 1738 XXH64_hash_t seed64, const(void)* secret, size_t secretLen, XXH3_hashLong64_f f_hashLong) 1739 @safe pure nothrow @nogc 1740 in(secretLen >= XXH3_SECRET_SIZE_MIN, "secretLen < XXH3_SECRET_SIZE_MIN") 1741 { 1742 /* 1743 * If an action is to be taken if `secretLen` condition is not respected, 1744 * it should be done here. 1745 * For now, it's a contract pre-condition. 1746 * Adding a check and a branch here would cost performance at every hash. 1747 * Also, note that function signature doesn't offer room to return an error. 1748 */ 1749 if (len <= 16) 1750 return xxh3_len_0to16_64b(cast(const(ubyte)*) input, len, 1751 cast(const(ubyte)*) secret, seed64); 1752 if (len <= 128) 1753 return xxh3_len_17to128_64b(cast(const(ubyte)*) input, len, 1754 cast(const(ubyte)*) secret, secretLen, seed64); 1755 if (len <= XXH3_MIDSIZE_MAX) 1756 return xxh3_len_129to240_64b(cast(const(ubyte)*) input, len, 1757 cast(const(ubyte)*) secret, secretLen, seed64); 1758 return f_hashLong(input, len, seed64, cast(const(ubyte)*) secret, secretLen); 1759 } 1760 1761 /* === Public entry point === */ 1762 1763 /* XXH PUBLIC API - hidden in D module */ 1764 private XXH64_hash_t xxh3_64bits(const(void)* input, size_t length) 1765 @safe pure nothrow @nogc 1766 { 1767 return xxh3_64bits_internal(input, length, 0, &xxh3_kSecret[0], 1768 (xxh3_kSecret).sizeof, &xxh3_hashLong_64b_default); 1769 } 1770 1771 /* XXH PUBLIC API - hidden in D module */ 1772 private XXH64_hash_t xxh3_64bits_withSecret( 1773 const(void)* input, size_t length, const(void)* secret, size_t secretSize) 1774 @safe pure nothrow @nogc 1775 { 1776 return xxh3_64bits_internal(input, length, 0, secret, secretSize, 1777 &xxh3_hashLong_64b_withSecret); 1778 } 1779 1780 /* XXH PUBLIC API - hidden in D module */ private 1781 XXH64_hash_t xxh3_64bits_withSeed(const(void)* input, size_t length, XXH64_hash_t seed) 1782 @safe pure nothrow @nogc 1783 { 1784 return xxh3_64bits_internal(input, length, seed, &xxh3_kSecret[0], 1785 (xxh3_kSecret).sizeof, &xxh3_hashLong_64b_withSeed); 1786 } 1787 1788 /* XXH PUBLIC API - hidden in D module */ private 1789 XXH64_hash_t xxh3_64bits_withSecretandSeed( 1790 const(void)* input, size_t length, const(void)* secret, size_t secretSize, XXH64_hash_t seed) 1791 @safe pure nothrow @nogc 1792 { 1793 if (length <= XXH3_MIDSIZE_MAX) 1794 return xxh3_64bits_internal(input, length, seed, &xxh3_kSecret[0], 1795 (xxh3_kSecret).sizeof, null); 1796 return xxh3_hashLong_64b_withSecret(input, length, seed, 1797 cast(const(ubyte)*) secret, secretSize); 1798 } 1799 1800 /* === XXH3 streaming === */ 1801 1802 private void XXH3_INITSTATE(XXH3_state_t* XXH3_state_ptr) 1803 @safe nothrow @nogc 1804 { 1805 (XXH3_state_ptr).seed = 0; 1806 } 1807 1808 private void xxh3_reset_internal( 1809 XXH3_state_t* statePtr, XXH64_hash_t seed, const void* secret, size_t secretSize) 1810 @trusted pure nothrow @nogc 1811 in 1812 { 1813 const size_t initStart = XXH3_state_t.bufferedSize.offsetof; 1814 assert(XXH3_state_t.nbStripesPerBlock.offsetof > initStart, 1815 "(XXH3_state_t.nbStripesPerBlock.offsetof <= initStart"); 1816 } 1817 in(statePtr != null, "statePtr == null") 1818 in(secretSize >= XXH3_SECRET_SIZE_MIN, "secretSize < XXH3_SECRET_SIZE_MIN") 1819 { 1820 /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ 1821 *statePtr = XXH3_state_t.init; 1822 statePtr.acc[0] = XXH_PRIME32_3; 1823 statePtr.acc[1] = XXH_PRIME64_1; 1824 statePtr.acc[2] = XXH_PRIME64_2; 1825 statePtr.acc[3] = XXH_PRIME64_3; 1826 statePtr.acc[4] = XXH_PRIME64_4; 1827 statePtr.acc[5] = XXH_PRIME32_2; 1828 statePtr.acc[6] = XXH_PRIME64_5; 1829 statePtr.acc[7] = XXH_PRIME32_1; 1830 statePtr.seed = seed; 1831 statePtr.useSeed = (seed != 0); 1832 statePtr.extSecret = cast(const(ubyte)*) secret; 1833 statePtr.secretLimit = secretSize - XXH_STRIPE_LEN; 1834 statePtr.nbStripesPerBlock = statePtr.secretLimit / XXH_SECRET_CONSUME_RATE; 1835 } 1836 1837 /* XXH PUBLIC API - hidden in D module */ private 1838 XXH_errorcode xxh3_64bits_reset(XXH3_state_t* statePtr) 1839 @safe pure nothrow @nogc 1840 { 1841 if (statePtr == null) 1842 return XXH_errorcode.XXH_ERROR; 1843 xxh3_reset_internal(statePtr, 0, &xxh3_kSecret[0], XXH_SECRET_DEFAULT_SIZE); 1844 return XXH_errorcode.XXH_OK; 1845 } 1846 1847 /* XXH PUBLIC API - hidden in D module */ 1848 private XXH_errorcode xxh3_64bits_reset_withSecret( 1849 XXH3_state_t* statePtr, const void* secret, size_t secretSize) 1850 @safe pure nothrow @nogc 1851 { 1852 if (statePtr == null) 1853 return XXH_errorcode.XXH_ERROR; 1854 xxh3_reset_internal(statePtr, 0, secret, secretSize); 1855 if (secret == null) 1856 return XXH_errorcode.XXH_ERROR; 1857 if (secretSize < XXH3_SECRET_SIZE_MIN) 1858 return XXH_errorcode.XXH_ERROR; 1859 return XXH_errorcode.XXH_OK; 1860 } 1861 1862 /* XXH PUBLIC API - hidden in D module */ 1863 private XXH_errorcode xxh3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) 1864 @safe pure nothrow @nogc 1865 { 1866 if (statePtr == null) 1867 return XXH_errorcode.XXH_ERROR; 1868 if (seed == 0) 1869 return xxh3_64bits_reset(statePtr); 1870 if ((seed != statePtr.seed) || (statePtr.extSecret != null)) 1871 xxh3_initCustomSecret(&statePtr.customSecret[0], seed); 1872 xxh3_reset_internal(statePtr, seed, null, XXH_SECRET_DEFAULT_SIZE); 1873 return XXH_errorcode.XXH_OK; 1874 } 1875 1876 /* XXH PUBLIC API - hidden in D module */ 1877 private XXH_errorcode xxh3_64bits_reset_withSecretandSeed( 1878 XXH3_state_t* statePtr, const(void)* secret, size_t secretSize, XXH64_hash_t seed64) 1879 @safe pure nothrow @nogc 1880 { 1881 if (statePtr == null) 1882 return XXH_errorcode.XXH_ERROR; 1883 if (secret == null) 1884 return XXH_errorcode.XXH_ERROR; 1885 if (secretSize < XXH3_SECRET_SIZE_MIN) 1886 return XXH_errorcode.XXH_ERROR; 1887 xxh3_reset_internal(statePtr, seed64, secret, secretSize); 1888 statePtr.useSeed = 1; /* always, even if seed64 == 0 */ 1889 return XXH_errorcode.XXH_OK; 1890 } 1891 1892 /* Note : when xxh3_consumeStripes() is invoked, 1893 * there must be a guarantee that at least one more byte must be consumed from input 1894 * so that the function can blindly consume all stripes using the "normal" secret segment */ 1895 private void xxh3_consumeStripes( 1896 ulong* acc, size_t* nbStripesSoFarPtr, size_t nbStripesPerBlock, const ubyte* input, size_t nbStripes, 1897 const ubyte* secret, size_t secretLimit, 1898 XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble) 1899 @trusted pure nothrow @nogc 1900 in(nbStripes <= nbStripesPerBlock, "nbStripes > nbStripesPerBlock") /* can handle max 1 scramble per invocation */ 1901 in(*nbStripesSoFarPtr < nbStripesPerBlock, "*nbStripesSoFarPtr >= nbStripesPerBlock") 1902 { 1903 if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes) 1904 { 1905 /* need a scrambling operation */ 1906 const size_t nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr; 1907 const size_t nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock; 1908 xxh3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, 1909 nbStripesToEndofBlock, f_acc512); 1910 f_scramble(acc, secret + secretLimit); 1911 xxh3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN, 1912 secret, nbStripesAfterBlock, f_acc512); 1913 *nbStripesSoFarPtr = nbStripesAfterBlock; 1914 } 1915 else 1916 { 1917 xxh3_accumulate(acc, input, 1918 secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, f_acc512); 1919 *nbStripesSoFarPtr += nbStripes; 1920 } 1921 } 1922 1923 enum XXH3_STREAM_USE_STACK = 1; 1924 /* 1925 * Both xxh3_64bits_update and xxh3_128bits_update use this routine. 1926 */ 1927 private XXH_errorcode xxh3_update( 1928 XXH3_state_t* state, scope const(ubyte)* input, size_t len, 1929 XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble) 1930 @trusted pure nothrow @nogc 1931 in(state != null, "state == null") 1932 in 1933 { 1934 if (input == null) assert(len == 0, "input null ptr only allowed with len == 0"); 1935 } 1936 do 1937 { 1938 if (input == null && len == 0) 1939 return XXH_errorcode.XXH_OK; 1940 else if (input == null && len != 0) 1941 return XXH_errorcode.XXH_ERROR; 1942 else 1943 { 1944 const ubyte* bEnd = input + len; 1945 const(ubyte)* secret = (state.extSecret == null) ? &state.customSecret[0] : &state.extSecret[0]; 1946 static if (XXH3_STREAM_USE_STACK >= 1) 1947 { 1948 /* For some reason, gcc and MSVC seem to suffer greatly 1949 * when operating accumulators directly into state. 1950 * Operating into stack space seems to enable proper optimization. 1951 * clang, on the other hand, doesn't seem to need this trick */ 1952 align(XXH_ACC_ALIGN) ulong[8] acc; 1953 (cast(ubyte*) &acc[0]) [0 .. acc.sizeof] = (cast(ubyte*) &state.acc[0]) [0 .. acc.sizeof]; 1954 } 1955 else 1956 { 1957 ulong* acc = state.acc; 1958 } 1959 state.totalLen += len; 1960 assert(state.bufferedSize <= XXH3_INTERNALBUFFER_SIZE, "state.bufferedSize > XXH3_INTERNALBUFFER_SIZE"); 1961 1962 /* small input : just fill in tmp buffer */ 1963 if (state.bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) 1964 { 1965 (cast(ubyte*) &state.buffer[0]) [state.bufferedSize .. state.bufferedSize + len] = 1966 (cast(ubyte*) input) [0 .. len]; 1967 state.bufferedSize += cast(XXH32_hash_t) len; 1968 return XXH_errorcode.XXH_OK; 1969 } 1970 1971 /* total input is now > XXH3_INTERNALBUFFER_SIZE */ 1972 enum XXH3_INTERNALBUFFER_STRIPES = (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN); 1973 static assert(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0, "XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN != 0"); /* clean multiple */ 1974 1975 /* 1976 * Internal buffer is partially filled (always, except at beginning) 1977 * Complete it, then consume it. 1978 */ 1979 if (state.bufferedSize) 1980 { 1981 const size_t loadSize = XXH3_INTERNALBUFFER_SIZE - state.bufferedSize; 1982 (cast(ubyte*)&state.buffer[0]) [state.bufferedSize .. state.bufferedSize + loadSize] = 1983 (cast(ubyte*) input) [0 .. loadSize]; 1984 input += loadSize; 1985 xxh3_consumeStripes(&acc[0], &state.nbStripesSoFar, state.nbStripesPerBlock, 1986 &state.buffer[0], XXH3_INTERNALBUFFER_STRIPES, secret, 1987 state.secretLimit, f_acc512, f_scramble); 1988 state.bufferedSize = 0; 1989 } 1990 assert(input < bEnd, "input >= bEnd"); 1991 1992 /* large input to consume : ingest per full block */ 1993 if (cast(size_t)(bEnd - input) > state.nbStripesPerBlock * XXH_STRIPE_LEN) 1994 { 1995 size_t nbStripes = cast(size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; 1996 assert(state.nbStripesPerBlock >= state.nbStripesSoFar, "state.nbStripesPerBlock < state.nbStripesSoFar"); 1997 /* join to current block's end */ 1998 { 1999 const size_t nbStripesToEnd = state.nbStripesPerBlock - state.nbStripesSoFar; 2000 assert(nbStripesToEnd <= nbStripes, "nbStripesToEnd > nbStripes"); 2001 xxh3_accumulate(&acc[0], input, 2002 secret + state.nbStripesSoFar * XXH_SECRET_CONSUME_RATE, 2003 nbStripesToEnd, f_acc512); 2004 f_scramble(&acc[0], secret + state.secretLimit); 2005 state.nbStripesSoFar = 0; 2006 input += nbStripesToEnd * XXH_STRIPE_LEN; 2007 nbStripes -= nbStripesToEnd; 2008 } 2009 /* consume per entire blocks */ 2010 while (nbStripes >= state.nbStripesPerBlock) 2011 { 2012 xxh3_accumulate(&acc[0], input, secret, state.nbStripesPerBlock, f_acc512); 2013 f_scramble(&acc[0], secret + state.secretLimit); 2014 input += state.nbStripesPerBlock * XXH_STRIPE_LEN; 2015 nbStripes -= state.nbStripesPerBlock; 2016 } 2017 /* consume last partial block */ 2018 xxh3_accumulate(&acc[0], input, secret, nbStripes, f_acc512); 2019 input += nbStripes * XXH_STRIPE_LEN; 2020 assert(input < bEnd, "input exceeds buffer, no bytes left"); /* at least some bytes left */ 2021 state.nbStripesSoFar = nbStripes; 2022 /* buffer predecessor of last partial stripe */ 2023 (cast(ubyte*) &state.buffer[0]) 2024 [state.buffer.sizeof - XXH_STRIPE_LEN .. state.buffer.sizeof - XXH_STRIPE_LEN + XXH_STRIPE_LEN] = 2025 (cast(ubyte*) input - XXH_STRIPE_LEN) [0 .. XXH_STRIPE_LEN]; 2026 assert(bEnd - input <= XXH_STRIPE_LEN, "input exceed strip length"); 2027 } 2028 else 2029 { 2030 /* content to consume <= block size */ 2031 /* Consume input by a multiple of internal buffer size */ 2032 if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) 2033 { 2034 const ubyte* limit = bEnd - XXH3_INTERNALBUFFER_SIZE; 2035 do 2036 { 2037 xxh3_consumeStripes(&acc[0], &state.nbStripesSoFar, state.nbStripesPerBlock, input, 2038 XXH3_INTERNALBUFFER_STRIPES, secret, 2039 state.secretLimit, f_acc512, f_scramble); 2040 input += XXH3_INTERNALBUFFER_SIZE; 2041 } 2042 while (input < limit); 2043 /* buffer predecessor of last partial stripe */ 2044 (cast(ubyte*) &state.buffer[0]) 2045 [state.buffer.sizeof - XXH_STRIPE_LEN .. state.buffer.sizeof - XXH_STRIPE_LEN + XXH_STRIPE_LEN] = 2046 (cast(ubyte*) input - XXH_STRIPE_LEN) [0 .. XXH_STRIPE_LEN]; 2047 } 2048 } 2049 2050 /* Some remaining input (always) : buffer it */ 2051 assert(input < bEnd, "input exceeds buffer"); 2052 assert(bEnd - input <= XXH3_INTERNALBUFFER_SIZE, "input outside buffer"); 2053 assert(state.bufferedSize == 0, "bufferedSize != 0"); 2054 (cast(ubyte*) &state.buffer[0]) [0 .. cast(size_t)(bEnd - input)] = 2055 (cast(ubyte*) input) [0 .. cast(size_t)(bEnd - input)]; 2056 state.bufferedSize = cast(XXH32_hash_t)(bEnd - input); 2057 static if (XXH3_STREAM_USE_STACK >= 1) 2058 { 2059 /* save stack accumulators into state */ 2060 (cast(ubyte*) &state.acc[0]) [0 .. acc.sizeof] = (cast(ubyte*) &acc[0]) [0 .. acc.sizeof]; 2061 } 2062 } 2063 2064 return XXH_errorcode.XXH_OK; 2065 } 2066 2067 /* XXH PUBLIC API - hidden in D module */ 2068 private XXH_errorcode xxh3_64bits_update(XXH3_state_t* state, scope const(void)* input, size_t len) 2069 @safe pure nothrow @nogc 2070 { 2071 return xxh3_update(state, cast(const(ubyte)*) input, len, 2072 xxh3_accumulate_512, xxh3_scrambleAcc); 2073 } 2074 2075 private void xxh3_digest_long(XXH64_hash_t* acc, const XXH3_state_t* state, const ubyte* secret) 2076 @trusted pure nothrow @nogc 2077 { 2078 /* 2079 * Digest on a local copy. This way, the state remains unaltered, and it can 2080 * continue ingesting more input afterwards. 2081 */ 2082 (cast(ubyte*) &acc[0]) [0 .. state.acc.sizeof] = (cast(ubyte*) &state.acc[0]) [0 .. state.acc.sizeof]; 2083 if (state.bufferedSize >= XXH_STRIPE_LEN) 2084 { 2085 const size_t nbStripes = (state.bufferedSize - 1) / XXH_STRIPE_LEN; 2086 size_t nbStripesSoFar = state.nbStripesSoFar; 2087 xxh3_consumeStripes(acc, &nbStripesSoFar, state.nbStripesPerBlock, &state.buffer[0], 2088 nbStripes, secret, state.secretLimit, xxh3_accumulate_512, xxh3_scrambleAcc); 2089 /* last stripe */ 2090 xxh3_accumulate_512(acc, &state.buffer[0] + state.bufferedSize - XXH_STRIPE_LEN, 2091 secret + state.secretLimit - XXH_SECRET_LASTACC_START); 2092 } 2093 else 2094 { /* bufferedSize < XXH_STRIPE_LEN */ 2095 ubyte[XXH_STRIPE_LEN] lastStripe; 2096 const size_t catchupSize = XXH_STRIPE_LEN - state.bufferedSize; 2097 assert(state.bufferedSize > 0, "bufferedSize <= 0"); /* there is always some input buffered */ 2098 (cast(ubyte*) &lastStripe[0]) [0 .. catchupSize] = 2099 (cast(ubyte*) &state.buffer[0]) [state.buffer.sizeof - catchupSize .. state.buffer.sizeof]; 2100 (cast(ubyte*) &lastStripe[0]) [catchupSize .. catchupSize + state.bufferedSize] = 2101 (cast(ubyte*) &state.buffer[0]) [0 .. state.buffer.sizeof]; 2102 xxh3_accumulate_512(&acc[0], &lastStripe[0], &secret[0] + state.secretLimit - XXH_SECRET_LASTACC_START); 2103 } 2104 } 2105 2106 /* XXH PUBLIC API - hidden in D module */ 2107 private XXH64_hash_t xxh3_64bits_digest(const XXH3_state_t* state) 2108 @trusted pure nothrow @nogc 2109 { 2110 const ubyte* secret = (state.extSecret == null) ? &state.customSecret[0] : &state.extSecret[0]; 2111 if (state.totalLen > XXH3_MIDSIZE_MAX) 2112 { 2113 align(XXH_ACC_ALIGN) XXH64_hash_t[XXH_ACC_NB] acc; 2114 xxh3_digest_long(&acc[0], state, secret); 2115 return xxh3_mergeAccs(&acc[0], secret + XXH_SECRET_MERGEACCS_START, 2116 cast(ulong) state.totalLen * XXH_PRIME64_1); 2117 } 2118 /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ 2119 if (state.useSeed) 2120 return xxh3_64bits_withSeed(&state.buffer[0], cast(size_t) state.totalLen, state.seed); 2121 return xxh3_64bits_withSecret(&state.buffer[0], 2122 cast(size_t)(state.totalLen), secret, state.secretLimit + XXH_STRIPE_LEN); 2123 } 2124 2125 /* ========================================== 2126 * XXH3 128 bits (a.k.a XXH128) 2127 * ========================================== 2128 * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, 2129 * even without counting the significantly larger output size. 2130 * 2131 * For example, extra steps are taken to avoid the seed-dependent collisions 2132 * in 17-240 byte inputs (See xxh3_mix16B and xxh128_mix32B). 2133 * 2134 * This strength naturally comes at the cost of some speed, especially on short 2135 * lengths. Note that longer hashes are about as fast as the 64-bit version 2136 * due to it using only a slight modification of the 64-bit loop. 2137 * 2138 * XXH128 is also more oriented towards 64-bit machines. It is still extremely 2139 * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). 2140 */ 2141 private XXH128_hash_t xxh3_len_1to3_128b( 2142 const ubyte* input, size_t len, const ubyte* secret, XXH64_hash_t seed) 2143 @trusted pure nothrow @nogc 2144 { 2145 /* A doubled version of 1to3_64b with different constants. */ 2146 assert(input != null, "input is null"); 2147 assert(1 <= len && len <= 3, "len is out of range"); 2148 assert(secret != null, "secret is null"); 2149 /* 2150 * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } 2151 * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } 2152 * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } 2153 */ 2154 { 2155 const ubyte c1 = input[0]; 2156 const ubyte c2 = input[len >> 1]; 2157 const ubyte c3 = input[len - 1]; 2158 const uint combinedl = (cast(uint) c1 << 16) | ( 2159 cast(uint) c2 << 24) | (cast(uint) c3 << 0) | (cast(uint) len << 8); 2160 const uint combinedh = rol(bswap(combinedl), 13); 2161 const ulong bitflipl = (xxh_readLE32(secret) ^ xxh_readLE32(secret + 4)) + seed; 2162 const ulong bitfliph = (xxh_readLE32(secret + 8) ^ xxh_readLE32(secret + 12)) - seed; 2163 const ulong keyed_lo = cast(ulong) combinedl ^ bitflipl; 2164 const ulong keyed_hi = cast(ulong) combinedh ^ bitfliph; 2165 XXH128_hash_t h128; 2166 h128.low64 = xxh64_avalanche(keyed_lo); 2167 h128.high64 = xxh64_avalanche(keyed_hi); 2168 return h128; 2169 } 2170 } 2171 2172 private XXH128_hash_t xxh3_len_4to8_128b( 2173 const ubyte* input, size_t len, const ubyte* secret, XXH64_hash_t seed) 2174 @trusted pure nothrow @nogc 2175 { 2176 assert(input != null, "input is null"); 2177 assert(secret != null, "secret is null"); 2178 assert(4 <= len && len <= 8, "len is out of range"); 2179 seed ^= cast(ulong) bswap(cast(uint) seed) << 32; 2180 { 2181 const uint input_lo = xxh_readLE32(input); 2182 const uint input_hi = xxh_readLE32(input + len - 4); 2183 const ulong input_64 = input_lo + (cast(ulong) input_hi << 32); 2184 const ulong bitflip = (xxh_readLE64(secret + 16) ^ xxh_readLE64(secret + 24)) + seed; 2185 const ulong keyed = input_64 ^ bitflip; 2186 2187 /* Shift len to the left to ensure it is even, this avoids even multiplies. */ 2188 XXH128_hash_t m128 = xxh_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); 2189 2190 m128.high64 += (m128.low64 << 1); 2191 m128.low64 ^= (m128.high64 >> 3); 2192 2193 m128.low64 = xxh_xorshift64(m128.low64, 35); 2194 m128.low64 *= 0x9FB21C651E98DF25; 2195 m128.low64 = xxh_xorshift64(m128.low64, 28); 2196 m128.high64 = xxh3_avalanche(m128.high64); 2197 return m128; 2198 } 2199 } 2200 2201 private XXH128_hash_t xxh3_len_9to16_128b( 2202 const ubyte* input, size_t len, const ubyte* secret, XXH64_hash_t seed) 2203 @trusted pure nothrow @nogc 2204 { 2205 assert(input != null, "input is null"); 2206 assert(secret != null, "secret is null"); 2207 assert(9 <= len && len <= 16, "len out of range"); 2208 { 2209 const ulong bitflipl = (xxh_readLE64(secret + 32) ^ xxh_readLE64(secret + 40)) - seed; 2210 const ulong bitfliph = (xxh_readLE64(secret + 48) ^ xxh_readLE64(secret + 56)) + seed; 2211 const ulong input_lo = xxh_readLE64(input); 2212 ulong input_hi = xxh_readLE64(input + len - 8); 2213 XXH128_hash_t m128 = xxh_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); 2214 /* 2215 * Put len in the middle of m128 to ensure that the length gets mixed to 2216 * both the low and high bits in the 128x64 multiply below. 2217 */ 2218 m128.low64 += cast(ulong)(len - 1) << 54; 2219 input_hi ^= bitfliph; 2220 /* 2221 * Add the high 32 bits of input_hi to the high 32 bits of m128, then 2222 * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to 2223 * the high 64 bits of m128. 2224 * 2225 * The best approach to this operation is different on 32-bit and 64-bit. 2226 */ 2227 if ((void*).sizeof < (ulong).sizeof) 2228 { /* 32-bit */ 2229 /* 2230 * 32-bit optimized version, which is more readable. 2231 * 2232 * On 32-bit, it removes an ADC and delays a dependency between the two 2233 * halves of m128.high64, but it generates an extra mask on 64-bit. 2234 */ 2235 m128.high64 += (input_hi & 0xFFFFFFFF00000000) + xxh_mult32to64(cast(uint) input_hi, 2236 XXH_PRIME32_2); 2237 } 2238 else 2239 { 2240 /* 2241 * 64-bit optimized (albeit more confusing) version. 2242 * 2243 * Uses some properties of addition and multiplication to remove the mask: 2244 * 2245 * Let: 2246 * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) 2247 * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) 2248 * c = XXH_PRIME32_2 2249 * 2250 * a + (b * c) 2251 * Inverse Property: x + y - x == y 2252 * a + (b * (1 + c - 1)) 2253 * Distributive Property: x * (y + z) == (x * y) + (x * z) 2254 * a + (b * 1) + (b * (c - 1)) 2255 * Identity Property: x * 1 == x 2256 * a + b + (b * (c - 1)) 2257 * 2258 * Substitute a, b, and c: 2259 * input_hi.hi + input_hi.lo + ((ulong)input_hi.lo * (XXH_PRIME32_2 - 1)) 2260 * 2261 * Since input_hi.hi + input_hi.lo == input_hi, we get this: 2262 * input_hi + ((ulong)input_hi.lo * (XXH_PRIME32_2 - 1)) 2263 */ 2264 m128.high64 += input_hi + xxh_mult32to64(cast(uint) input_hi, XXH_PRIME32_2 - 1); 2265 } 2266 /* m128 ^= bswap(m128 >> 64); */ 2267 m128.low64 ^= bswap(m128.high64); 2268 2269 { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ 2270 XXH128_hash_t h128 = xxh_mult64to128(m128.low64, XXH_PRIME64_2); 2271 h128.high64 += m128.high64 * XXH_PRIME64_2; 2272 2273 h128.low64 = xxh3_avalanche(h128.low64); 2274 h128.high64 = xxh3_avalanche(h128.high64); 2275 return h128; 2276 } 2277 } 2278 } 2279 2280 private XXH128_hash_t xxh3_len_0to16_128b( 2281 const ubyte* input, size_t len, const ubyte* secret, XXH64_hash_t seed) 2282 @trusted pure nothrow @nogc 2283 { 2284 assert(len <= 16, "len > 16"); 2285 { 2286 if (len > 8) 2287 return xxh3_len_9to16_128b(input, len, secret, seed); 2288 if (len >= 4) 2289 return xxh3_len_4to8_128b(input, len, secret, seed); 2290 if (len) 2291 return xxh3_len_1to3_128b(input, len, secret, seed); 2292 { 2293 XXH128_hash_t h128; 2294 const ulong bitflipl = xxh_readLE64(secret + 64) ^ xxh_readLE64(secret + 72); 2295 const ulong bitfliph = xxh_readLE64(secret + 80) ^ xxh_readLE64(secret + 88); 2296 h128.low64 = xxh64_avalanche(seed ^ bitflipl); 2297 h128.high64 = xxh64_avalanche(seed ^ bitfliph); 2298 return h128; 2299 } 2300 } 2301 } 2302 2303 private XXH128_hash_t xxh128_mix32B( 2304 XXH128_hash_t acc, const ubyte* input_1, const ubyte* input_2, const ubyte* secret, XXH64_hash_t seed) 2305 @trusted pure nothrow @nogc 2306 { 2307 acc.low64 += xxh3_mix16B(input_1, secret + 0, seed); 2308 acc.low64 ^= xxh_readLE64(input_2) + xxh_readLE64(input_2 + 8); 2309 acc.high64 += xxh3_mix16B(input_2, secret + 16, seed); 2310 acc.high64 ^= xxh_readLE64(input_1) + xxh_readLE64(input_1 + 8); 2311 return acc; 2312 } 2313 2314 private XXH128_hash_t xxh3_len_17to128_128b( 2315 const ubyte* input, size_t len, const ubyte* secret, size_t secretSize, XXH64_hash_t seed) 2316 @trusted pure nothrow @nogc 2317 in(secretSize >= XXH3_SECRET_SIZE_MIN, "secretSie < XXH3_SECRET_SIZE_MIN") 2318 in(16 < len && len <= 128, "len out of range") 2319 { 2320 XXH128_hash_t acc; 2321 acc.low64 = len * XXH_PRIME64_1; 2322 acc.high64 = 0; 2323 2324 static if (XXH_SIZE_OPT >= 1) 2325 { 2326 /* Smaller, but slightly slower. */ 2327 size_t i = (len - 1) / 32; 2328 do 2329 { 2330 acc = xxh128_mix32B(acc, input + 16 * i, 2331 input + len - 16 * (i + 1), secret + 32 * i, seed); 2332 } 2333 while (i-- != 0); 2334 } 2335 else 2336 { 2337 if (len > 32) 2338 { 2339 if (len > 64) 2340 { 2341 if (len > 96) 2342 { 2343 acc = xxh128_mix32B(acc, input + 48, input + len - 64, secret + 96, seed); 2344 } 2345 acc = xxh128_mix32B(acc, input + 32, input + len - 48, secret + 64, seed); 2346 } 2347 acc = xxh128_mix32B(acc, input + 16, input + len - 32, secret + 32, seed); 2348 } 2349 acc = xxh128_mix32B(acc, input, input + len - 16, secret, seed); 2350 } 2351 { 2352 XXH128_hash_t h128; 2353 h128.low64 = acc.low64 + acc.high64; 2354 h128.high64 = (acc.low64 * XXH_PRIME64_1) + ( 2355 acc.high64 * XXH_PRIME64_4) + ((len - seed) * XXH_PRIME64_2); 2356 h128.low64 = xxh3_avalanche(h128.low64); 2357 h128.high64 = cast(XXH64_hash_t) 0 - xxh3_avalanche(h128.high64); 2358 return h128; 2359 } 2360 } 2361 2362 private XXH128_hash_t xxh3_len_129to240_128b( 2363 const ubyte* input, size_t len, const ubyte* secret, size_t secretSize, XXH64_hash_t seed) 2364 @trusted pure nothrow @nogc 2365 in(secretSize >= XXH3_SECRET_SIZE_MIN, "secretSize < XXH3_SECRET_SIZE_MIN") 2366 in(128 < len && len <= XXH3_MIDSIZE_MAX, "len > 128 or len > XXH3_MIDSIZE_MAX") 2367 { 2368 XXH128_hash_t acc; 2369 const int nbRounds = cast(int) len / 32; 2370 int i; 2371 acc.low64 = len * XXH_PRIME64_1; 2372 acc.high64 = 0; 2373 for (i = 0; i < 4; i++) 2374 { 2375 acc = xxh128_mix32B(acc, input + (32 * i), input + (32 * i) + 16, secret + (32 * i), 2376 seed); 2377 } 2378 acc.low64 = xxh3_avalanche(acc.low64); 2379 acc.high64 = xxh3_avalanche(acc.high64); 2380 assert(nbRounds >= 4, "nbRounds < 4"); 2381 for (i = 4; i < nbRounds; i++) 2382 { 2383 acc = xxh128_mix32B(acc, input + (32 * i), input + (32 * i) + 16, 2384 secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)), seed); 2385 } 2386 /* last bytes */ 2387 acc = xxh128_mix32B(acc, input + len - 16, input + len - 32, 2388 secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, 0 - seed); 2389 2390 { 2391 XXH128_hash_t h128; 2392 h128.low64 = acc.low64 + acc.high64; 2393 h128.high64 = (acc.low64 * XXH_PRIME64_1) + ( 2394 acc.high64 * XXH_PRIME64_4) + ((len - seed) * XXH_PRIME64_2); 2395 h128.low64 = xxh3_avalanche(h128.low64); 2396 h128.high64 = cast(XXH64_hash_t) 0 - xxh3_avalanche(h128.high64); 2397 return h128; 2398 } 2399 } 2400 2401 private XXH128_hash_t xxh3_hashLong_128b_internal( 2402 const void* input, size_t len, const ubyte* secret, size_t secretSize, 2403 XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble) 2404 @trusted pure nothrow @nogc 2405 { 2406 align(XXH_ACC_ALIGN) ulong[XXH_ACC_NB] acc = XXH3_INIT_ACC; 2407 2408 xxh3_hashLong_internal_loop(&acc[0], cast(const ubyte*) input, len, 2409 secret, secretSize, f_acc512, f_scramble); 2410 2411 /* converge into final hash */ 2412 static assert(acc.sizeof == 64, "acc isn't 64 bytes long"); 2413 assert(secretSize >= acc.sizeof + XXH_SECRET_MERGEACCS_START, "secretSze < allowed limit."); 2414 { 2415 XXH128_hash_t h128; 2416 h128.low64 = xxh3_mergeAccs(&acc[0], 2417 secret + XXH_SECRET_MERGEACCS_START, cast(ulong) len * XXH_PRIME64_1); 2418 h128.high64 = xxh3_mergeAccs(&acc[0], secret + secretSize - (acc) 2419 .sizeof - XXH_SECRET_MERGEACCS_START, ~(cast(ulong) len * XXH_PRIME64_2)); 2420 return h128; 2421 } 2422 } 2423 2424 private XXH128_hash_t xxh3_hashLong_128b_default( 2425 const void* input, size_t len, XXH64_hash_t seed64, const void* secret, size_t secretLen) 2426 @safe pure nothrow @nogc 2427 { 2428 return xxh3_hashLong_128b_internal(input, len, &xxh3_kSecret[0], 2429 (xxh3_kSecret).sizeof, xxh3_accumulate_512, xxh3_scrambleAcc); 2430 } 2431 2432 /* 2433 * It's important for performance to pass @p secretLen (when it's static) 2434 * to the compiler, so that it can properly optimize the vectorized loop. 2435 */ 2436 private XXH128_hash_t xxh3_hashLong_128b_withSecret( 2437 const void* input, size_t len, XXH64_hash_t seed64, const void* secret, size_t secretLen) 2438 @safe pure nothrow @nogc 2439 { 2440 return xxh3_hashLong_128b_internal(input, len, cast(const ubyte*) secret, 2441 secretLen, xxh3_accumulate_512, xxh3_scrambleAcc); 2442 } 2443 2444 private XXH128_hash_t xxh3_hashLong_128b_withSeed_internal( 2445 const void* input, size_t len, XXH64_hash_t seed64, 2446 XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble, 2447 XXH3_f_initCustomSecret f_initSec) 2448 @trusted pure nothrow @nogc 2449 { 2450 if (seed64 == 0) 2451 return xxh3_hashLong_128b_internal(input, len, &xxh3_kSecret[0], 2452 (xxh3_kSecret).sizeof, f_acc512, f_scramble); 2453 { 2454 align(XXH_SEC_ALIGN) ubyte[XXH_SECRET_DEFAULT_SIZE] secret; 2455 f_initSec(&secret[0], seed64); 2456 return xxh3_hashLong_128b_internal(input, len, 2457 cast(const ubyte*)&secret[0], (secret).sizeof, f_acc512, f_scramble); 2458 } 2459 } 2460 /* 2461 * It's important for performance that XXH3_hashLong is not inlined. 2462 */ 2463 private XXH128_hash_t xxh3_hashLong_128b_withSeed( 2464 const void* input, size_t len, XXH64_hash_t seed64, const void* secret, size_t secretLen) 2465 @safe pure nothrow @nogc 2466 { 2467 return xxh3_hashLong_128b_withSeed_internal(input, len, seed64, 2468 xxh3_accumulate_512, xxh3_scrambleAcc, xxh3_initCustomSecret); 2469 } 2470 2471 alias XXH3_hashLong128_f = XXH128_hash_t function(const void*, size_t, 2472 XXH64_hash_t, const void*, size_t) @safe pure nothrow @nogc; 2473 2474 private XXH128_hash_t xxh3_128bits_internal( 2475 const void* input, size_t len, XXH64_hash_t seed64, const void* secret, size_t secretLen, 2476 XXH3_hashLong128_f f_hl128) 2477 @safe pure nothrow @nogc 2478 in(secretLen >= XXH3_SECRET_SIZE_MIN, "Secret length is < XXH3_SECRET_SIZE_MIN") 2479 { 2480 /* 2481 * If an action is to be taken if `secret` conditions are not respected, 2482 * it should be done here. 2483 * For now, it's a contract pre-condition. 2484 * Adding a check and a branch here would cost performance at every hash. 2485 */ 2486 if (len <= 16) 2487 return xxh3_len_0to16_128b(cast(const ubyte*) input, len, 2488 cast(const ubyte*) secret, seed64); 2489 if (len <= 128) 2490 return xxh3_len_17to128_128b(cast(const ubyte*) input, len, 2491 cast(const ubyte*) secret, secretLen, seed64); 2492 if (len <= XXH3_MIDSIZE_MAX) 2493 return xxh3_len_129to240_128b(cast(const ubyte*) input, len, 2494 cast(const ubyte*) secret, secretLen, seed64); 2495 return f_hl128(input, len, seed64, secret, secretLen); 2496 } 2497 2498 /* === Public XXH128 API === */ 2499 2500 /* XXH PUBLIC API - hidden in D module */ 2501 private XXH128_hash_t xxh3_128bits(const void* input, size_t len) 2502 @safe pure nothrow @nogc 2503 { 2504 return xxh3_128bits_internal(input, len, 0, &xxh3_kSecret[0], 2505 (xxh3_kSecret).sizeof, &xxh3_hashLong_128b_default); 2506 } 2507 2508 /* XXH PUBLIC API - hidden in D module */ 2509 private XXH128_hash_t xxh3_128bits_withSecret( 2510 const void* input, size_t len, const void* secret, size_t secretSize) 2511 @safe pure nothrow @nogc 2512 { 2513 return xxh3_128bits_internal(input, len, 0, cast(const ubyte*) secret, 2514 secretSize, &xxh3_hashLong_128b_withSecret); 2515 } 2516 2517 /* XXH PUBLIC API - hidden in D module */ 2518 private XXH128_hash_t xxh3_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) 2519 @safe pure nothrow @nogc 2520 { 2521 return xxh3_128bits_internal(input, len, seed, &xxh3_kSecret[0], 2522 (xxh3_kSecret).sizeof, &xxh3_hashLong_128b_withSeed); 2523 } 2524 2525 /* XXH PUBLIC API - hidden in D module */ 2526 private XXH128_hash_t xxh3_128bits_withSecretandSeed( 2527 const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed) 2528 @safe pure nothrow @nogc 2529 { 2530 if (len <= XXH3_MIDSIZE_MAX) 2531 return xxh3_128bits_internal(input, len, seed, &xxh3_kSecret[0], 2532 (xxh3_kSecret).sizeof, null); 2533 return xxh3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); 2534 } 2535 2536 /* XXH PUBLIC API - hidden in D module */ 2537 private XXH128_hash_t XXH128(const void* input, size_t len, XXH64_hash_t seed) 2538 @safe pure nothrow @nogc 2539 { 2540 return xxh3_128bits_withSeed(input, len, seed); 2541 } 2542 2543 /* XXH PUBLIC API - hidden in D module */ 2544 private XXH_errorcode xxh3_128bits_reset(XXH3_state_t* statePtr) 2545 @safe pure nothrow @nogc 2546 { 2547 return xxh3_64bits_reset(statePtr); 2548 } 2549 2550 /* XXH PUBLIC API - hidden in D module */ 2551 private XXH_errorcode xxh3_128bits_reset_withSecret( 2552 XXH3_state_t* statePtr, const void* secret, size_t secretSize) 2553 @safe pure nothrow @nogc 2554 { 2555 return xxh3_64bits_reset_withSecret(statePtr, secret, secretSize); 2556 } 2557 2558 /* XXH PUBLIC API - hidden in D module */ private 2559 XXH_errorcode xxh3_128bits_reset_withSeed( 2560 XXH3_state_t* statePtr, XXH64_hash_t seed) 2561 @safe pure nothrow @nogc 2562 { 2563 return xxh3_64bits_reset_withSeed(statePtr, seed); 2564 } 2565 2566 /* XXH PUBLIC API - hidden in D module */ private 2567 XXH_errorcode xxh3_128bits_reset_withSecretandSeed( 2568 XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed) 2569 @safe pure nothrow @nogc 2570 { 2571 return xxh3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); 2572 } 2573 2574 /* XXH PUBLIC API - hidden in D module */ private 2575 XXH_errorcode xxh3_128bits_update( 2576 XXH3_state_t* state, scope const void* input, size_t len) 2577 @safe pure nothrow @nogc 2578 { 2579 return xxh3_update(state, cast(const ubyte*) input, len, 2580 xxh3_accumulate_512, xxh3_scrambleAcc); 2581 } 2582 2583 /* XXH PUBLIC API - hidden in D module */ private 2584 XXH128_hash_t xxh3_128bits_digest(const XXH3_state_t* state) 2585 @trusted pure nothrow @nogc 2586 { 2587 const ubyte* secret = (state.extSecret == null) ? &state.customSecret[0] : &state.extSecret[0]; 2588 if (state.totalLen > XXH3_MIDSIZE_MAX) 2589 { 2590 align(XXH_ACC_ALIGN) XXH64_hash_t[XXH_ACC_NB] acc; 2591 xxh3_digest_long(&acc[0], state, secret); 2592 assert(state.secretLimit + XXH_STRIPE_LEN >= acc.sizeof + XXH_SECRET_MERGEACCS_START, "Internal error"); 2593 { 2594 XXH128_hash_t h128; 2595 h128.low64 = xxh3_mergeAccs(&acc[0], secret + XXH_SECRET_MERGEACCS_START, 2596 cast(ulong) state.totalLen * XXH_PRIME64_1); 2597 h128.high64 = xxh3_mergeAccs(&acc[0], secret + state.secretLimit + XXH_STRIPE_LEN - (acc) 2598 .sizeof - XXH_SECRET_MERGEACCS_START, 2599 ~(cast(ulong) state.totalLen * XXH_PRIME64_2)); 2600 return h128; 2601 } 2602 } 2603 /* len <= XXH3_MIDSIZE_MAX : short code */ 2604 if (state.seed) 2605 return xxh3_128bits_withSeed(&state.buffer[0], cast(size_t) state.totalLen, state.seed); 2606 return xxh3_128bits_withSecret(&state.buffer[0], 2607 cast(size_t)(state.totalLen), secret, state.secretLimit + XXH_STRIPE_LEN); 2608 } 2609 2610 /* ----------------------------------------------------------------------------------------*/ 2611 2612 import core.bitop; 2613 2614 public import std.digest; 2615 2616 /* 2617 * Helper methods for encoding the buffer. 2618 * Can be removed if the optimizer can inline the methods from std.bitmanip. 2619 */ 2620 version (LittleEndian) 2621 { 2622 private alias nativeToBigEndian = bswap; 2623 private alias bigEndianToNative = bswap; 2624 } 2625 else 2626 pragma(inline, true) private pure @nogc nothrow @safe 2627 { 2628 uint nativeToBigEndian(uint val) 2629 { 2630 return val; 2631 } 2632 2633 ulong nativeToBigEndian(ulong val) 2634 { 2635 return val; 2636 } 2637 2638 alias bigEndianToNative = nativeToBigEndian; 2639 } 2640 2641 /** 2642 * Template API XXHTemplate implementation. Uses parameters to configure for number of bits and XXH variant (classic or XXH3) 2643 * See `std.digest` for differences between template and OOP API. 2644 */ 2645 struct XXHTemplate(HASH, STATE, bool useXXH3) 2646 { 2647 private: 2648 HASH hash; 2649 STATE state; 2650 HASH seed = HASH.init; 2651 2652 public: 2653 enum digestSize = HASH.sizeof * 8; 2654 2655 /** 2656 * Use this to feed the digest with data. 2657 * Also implements the $(REF isOutputRange, std,range,primitives) 2658 * interface for `ubyte` and `const(ubyte)[]`. 2659 * 2660 * Example: 2661 * ---- 2662 * XXHTemplate!(hashtype,statetype,useXXH3) dig; 2663 * dig.put(cast(ubyte) 0); //single ubyte 2664 * dig.put(cast(ubyte) 0, cast(ubyte) 0); //variadic 2665 * ubyte[10] buf; 2666 * dig.put(buf); //buffer 2667 * ---- 2668 */ 2669 void put(scope const(ubyte)[] data...) @safe nothrow @nogc 2670 { 2671 XXH_errorcode ec = XXH_errorcode.XXH_OK; 2672 if (data.length > 0) // digest will only change, when there is data! 2673 { 2674 static if (digestSize == 32) 2675 ec = xxh32_update(&state, &data[0], data.length); 2676 else static if (digestSize == 64 && !useXXH3) 2677 ec = xxh64_update(&state, &data[0], data.length); 2678 else static if (digestSize == 64 && useXXH3) 2679 ec = xxh3_64bits_update(&state, &data[0], data.length); 2680 else static if (digestSize == 128) 2681 ec = xxh3_128bits_update(&state, &data[0], data.length); 2682 else 2683 assert(false, "Unknown XXH bitdeep or variant"); 2684 } 2685 assert(ec == XXH_errorcode.XXH_OK, "Update failed"); 2686 } 2687 2688 /** 2689 * Used to (re)initialize the XXHTemplate digest. 2690 * 2691 * Example: 2692 * -------- 2693 * XXHTemplate!(hashtype,statetype,useXXH3) digest; 2694 * digest.start(); 2695 * digest.put(0); 2696 * -------- 2697 */ 2698 void start() @safe nothrow @nogc 2699 { 2700 this = typeof(this).init; 2701 XXH_errorcode ec; 2702 static if (digestSize == 32) 2703 { 2704 assert(state.alignof == uint.alignof, "Wrong alignment for state structure"); 2705 ec = xxh32_reset(&state, seed); 2706 } 2707 else static if (digestSize == 64 && !useXXH3) 2708 { 2709 assert(state.alignof == ulong.alignof, "Wrong alignment for state structure"); 2710 ec = xxh64_reset(&state, seed); 2711 } 2712 else static if (digestSize == 64 && useXXH3) 2713 { 2714 assert(state.alignof == 64, "Wrong alignment for state structure"); 2715 ec = xxh3_64bits_reset(&state); 2716 } 2717 else static if (digestSize == 128) 2718 { 2719 assert(state.alignof == 64, "Wrong alignment for state structure"); 2720 ec = xxh3_128bits_reset(&state); 2721 } 2722 else 2723 assert(false, "Unknown XXH bitdeep or variant"); 2724 //assert(ec == XXH_errorcode.XXH_OK, "reset failed"); 2725 } 2726 2727 /** 2728 * Returns the finished XXH hash. This also calls $(LREF start) to 2729 * reset the internal state. 2730 */ 2731 ubyte[digestSize / 8] finish() @trusted nothrow @nogc 2732 { 2733 XXH_errorcode ec; 2734 static if (digestSize == 32) 2735 { 2736 hash = xxh32_digest(&state); 2737 auto rc = nativeToBigEndian(hash); 2738 } 2739 else static if (digestSize == 64 && !useXXH3) 2740 { 2741 hash = xxh64_digest(&state); 2742 auto rc = nativeToBigEndian(hash); 2743 } 2744 else static if (digestSize == 64 && useXXH3) 2745 { 2746 hash = xxh3_64bits_digest(&state); 2747 auto rc = nativeToBigEndian(hash); 2748 } 2749 else static if (digestSize == 128) 2750 { 2751 hash = xxh3_128bits_digest(&state); 2752 HASH rc; 2753 // Note: low64 and high64 are intentionally exchanged! 2754 rc.low64 = nativeToBigEndian(hash.high64); 2755 rc.high64 = nativeToBigEndian(hash.low64); 2756 } 2757 assert(ec == XXH_errorcode.XXH_OK, "freestate failed"); 2758 2759 return (cast(ubyte*)&rc)[0 .. rc.sizeof]; 2760 } 2761 } 2762 /// 2763 @safe unittest 2764 { 2765 // Simple example using the XXH_64 digest 2766 XXHTemplate!(XXH64_hash_t, XXH64_state_t, false) hash1; 2767 hash1.start(); 2768 hash1.put(cast(ubyte) 0); 2769 auto result = hash1.finish(); 2770 } 2771 2772 alias XXH_32 = XXHTemplate!(XXH32_hash_t, XXH32_state_t, false); /// XXH_32 for XXH, 32bit, hash is ubyte[4] 2773 alias XXH_64 = XXHTemplate!(XXH64_hash_t, XXH64_state_t, false); /// XXH_64 for XXH, 64bit, hash is ubyte[8] 2774 alias XXH3_64 = XXHTemplate!(XXH64_hash_t, XXH3_state_t, true); /// XXH3_64 for XXH3, 64bits, hash is ubyte[8] 2775 alias XXH3_128 = XXHTemplate!(XXH128_hash_t, XXH3_state_t, true); /// XXH3_128 for XXH3, 128bits, hash is ubyte[16] 2776 2777 /// 2778 @safe unittest 2779 { 2780 //Simple example 2781 XXH_32 hash1; 2782 hash1.start(); 2783 hash1.put(cast(ubyte) 0); 2784 auto result = hash1.finish(); 2785 } 2786 /// 2787 @safe unittest 2788 { 2789 //Simple example 2790 XXH_64 hash1; 2791 hash1.start(); 2792 hash1.put(cast(ubyte) 0); 2793 auto result = hash1.finish(); 2794 } 2795 /// 2796 @safe unittest 2797 { 2798 //Simple example 2799 XXH3_64 hash1; 2800 hash1.start(); 2801 hash1.put(cast(ubyte) 0); 2802 auto result = hash1.finish(); 2803 } 2804 /// 2805 @safe unittest 2806 { 2807 //Simple example 2808 XXH3_128 hash1; 2809 hash1.start(); 2810 hash1.put(cast(ubyte) 0); 2811 auto result = hash1.finish(); 2812 } 2813 2814 /// 2815 @safe unittest 2816 { 2817 //Simple example, hashing a string using xxh32Of helper function 2818 auto hash = xxh32Of("abc"); 2819 //Let's get a hash string 2820 assert(toHexString(hash) == "32D153FF"); 2821 } 2822 /// 2823 @safe unittest 2824 { 2825 //Simple example, hashing a string using xxh32Of helper function 2826 auto hash = xxh64Of("abc"); 2827 //Let's get a hash string 2828 assert(toHexString(hash) == "44BC2CF5AD770999"); // XXH64 2829 } 2830 /// 2831 @safe unittest 2832 { 2833 //Simple example, hashing a string using xxh32Of helper function 2834 auto hash = xxh3_64Of("abc"); 2835 //Let's get a hash string 2836 assert(toHexString(hash) == "78AF5F94892F3950"); // XXH3/64 2837 } 2838 /// 2839 @safe unittest 2840 { 2841 //Simple example, hashing a string using xxh32Of helper function 2842 auto hash = xxh128Of("abc"); 2843 //Let's get a hash string 2844 assert(toHexString(hash) == "06B05AB6733A618578AF5F94892F3950"); 2845 2846 } 2847 2848 /// 2849 @safe unittest 2850 { 2851 //Using the basic API 2852 XXH_32 hash; 2853 hash.start(); 2854 ubyte[1024] data; 2855 //Initialize data here... 2856 hash.put(data); 2857 ubyte[4] result = hash.finish(); 2858 } 2859 /// 2860 @safe unittest 2861 { 2862 //Using the basic API 2863 XXH_64 hash; 2864 hash.start(); 2865 ubyte[1024] data; 2866 //Initialize data here... 2867 hash.put(data); 2868 ubyte[8] result = hash.finish(); 2869 } 2870 /// 2871 @safe unittest 2872 { 2873 //Using the basic API 2874 XXH3_64 hash; 2875 hash.start(); 2876 ubyte[1024] data; 2877 //Initialize data here... 2878 hash.put(data); 2879 ubyte[8] result = hash.finish(); 2880 } 2881 /// 2882 @safe unittest 2883 { 2884 //Using the basic API 2885 XXH3_128 hash; 2886 hash.start(); 2887 ubyte[1024] data; 2888 //Initialize data here... 2889 hash.put(data); 2890 ubyte[16] result = hash.finish(); 2891 } 2892 2893 /// 2894 @safe unittest 2895 { 2896 //Let's use the template features: 2897 void doSomething(T)(ref T hash) 2898 if (isDigest!T) 2899 { 2900 hash.put(cast(ubyte) 0); 2901 } 2902 2903 XXH_32 xxh; 2904 xxh.start(); 2905 doSomething(xxh); 2906 auto hash = xxh.finish; 2907 assert(toHexString(hash) == "CF65B03E", "Got " ~ toHexString(hash)); 2908 } 2909 /// 2910 @safe unittest 2911 { 2912 //Let's use the template features: 2913 void doSomething(T)(ref T hash) 2914 if (isDigest!T) 2915 { 2916 hash.put(cast(ubyte) 0); 2917 } 2918 2919 XXH_64 xxh; 2920 xxh.start(); 2921 doSomething(xxh); 2922 auto hash = xxh.finish; 2923 assert(toHexString(hash) == "E934A84ADB052768", "Got " ~ toHexString(hash)); 2924 } 2925 /// 2926 @safe unittest 2927 { 2928 //Let's use the template features: 2929 void doSomething(T)(ref T hash) 2930 if (isDigest!T) 2931 { 2932 hash.put(cast(ubyte) 0); 2933 } 2934 2935 XXH3_64 xxh; 2936 xxh.start(); 2937 doSomething(xxh); 2938 auto hash = xxh.finish; 2939 assert(toHexString(hash) == "C44BDFF4074EECDB", "Got " ~ toHexString(hash)); 2940 } 2941 /// 2942 @safe unittest 2943 { 2944 //Let's use the template features: 2945 void doSomething(T)(ref T hash) 2946 if (isDigest!T) 2947 { 2948 hash.put(cast(ubyte) 0); 2949 } 2950 2951 XXH3_128 xxh; 2952 xxh.start(); 2953 doSomething(xxh); 2954 auto hash = xxh.finish; 2955 assert(toHexString(hash) == "A6CD5E9392000F6AC44BDFF4074EECDB", "Got " ~ toHexString(hash)); 2956 } 2957 2958 /// 2959 @safe unittest 2960 { 2961 assert(isDigest!XXH_32); 2962 assert(isDigest!XXH_64); 2963 assert(isDigest!XXH3_64); 2964 assert(isDigest!XXH3_128); 2965 } 2966 2967 @system unittest 2968 { 2969 import std.range; 2970 import std.conv : hexString; 2971 2972 ubyte[4] digest32; 2973 ubyte[8] digest64; 2974 ubyte[16] digest128; 2975 2976 XXH_32 xxh; 2977 xxh.put(cast(ubyte[]) "abcdef"); 2978 xxh.start(); 2979 xxh.put(cast(ubyte[]) ""); 2980 assert(xxh.finish() == cast(ubyte[]) hexString!"02cc5d05"); 2981 2982 digest32 = xxh32Of(""); 2983 assert(digest32 == cast(ubyte[]) hexString!"02cc5d05"); 2984 digest64 = xxh64Of(""); 2985 assert(digest64 == cast(ubyte[]) hexString!"EF46DB3751D8E999", "Got " ~ toHexString(digest64)); 2986 digest64 = xxh3_64Of(""); 2987 assert(digest64 == cast(ubyte[]) hexString!"2D06800538D394C2", "Got " ~ toHexString(digest64)); 2988 digest128 = xxh128Of(""); 2989 assert(digest128 == cast(ubyte[]) hexString!"99AA06D3014798D86001C324468D497F", 2990 "Got " ~ toHexString(digest128)); 2991 2992 digest32 = xxh32Of("a"); 2993 assert(digest32 == cast(ubyte[]) hexString!"550d7456"); 2994 digest64 = xxh64Of("a"); 2995 assert(digest64 == cast(ubyte[]) hexString!"D24EC4F1A98C6E5B", "Got " ~ toHexString(digest64)); 2996 digest64 = xxh3_64Of("a"); 2997 assert(digest64 == cast(ubyte[]) hexString!"E6C632B61E964E1F", "Got " ~ toHexString(digest64)); 2998 digest128 = xxh128Of("a"); 2999 assert(digest128 == cast(ubyte[]) hexString!"A96FAF705AF16834E6C632B61E964E1F", 3000 "Got " ~ toHexString(digest128)); 3001 3002 digest32 = xxh32Of("abc"); 3003 assert(digest32 == cast(ubyte[]) hexString!"32D153FF"); 3004 digest64 = xxh64Of("abc"); 3005 assert(digest64 == cast(ubyte[]) hexString!"44BC2CF5AD770999"); 3006 digest64 = xxh3_64Of("abc"); 3007 assert(digest64 == cast(ubyte[]) hexString!"78AF5F94892F3950"); 3008 digest128 = xxh128Of("abc"); 3009 assert(digest128 == cast(ubyte[]) hexString!"06B05AB6733A618578AF5F94892F3950"); 3010 3011 digest32 = xxh32Of("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"); 3012 assert(digest32 == cast(ubyte[]) hexString!"89ea60c3"); 3013 digest64 = xxh64Of("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"); 3014 assert(digest64 == cast(ubyte[]) hexString!"F06103773E8585DF", "Got " ~ toHexString(digest64)); 3015 digest64 = xxh3_64Of("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"); 3016 assert(digest64 == cast(ubyte[]) hexString!"5BBCBBABCDCC3D3F", "Got " ~ toHexString(digest64)); 3017 digest128 = xxh128Of("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"); 3018 assert(digest128 == cast(ubyte[]) hexString!"3D62D22A5169B016C0D894FD4828A1A7", 3019 "Got " ~ toHexString(digest128)); 3020 3021 digest32 = xxh32Of("message digest"); 3022 assert(digest32 == cast(ubyte[]) hexString!"7c948494"); 3023 digest64 = xxh64Of("message digest"); 3024 assert(digest64 == cast(ubyte[]) hexString!"066ED728FCEEB3BE", "Got " ~ toHexString(digest64)); 3025 digest64 = xxh3_64Of("message digest"); 3026 assert(digest64 == cast(ubyte[]) hexString!"160D8E9329BE94F9", "Got " ~ toHexString(digest64)); 3027 digest128 = xxh128Of("message digest"); 3028 assert(digest128 == cast(ubyte[]) hexString!"34AB715D95E3B6490ABFABECB8E3A424", 3029 "Got " ~ toHexString(digest128)); 3030 3031 digest32 = xxh32Of("abcdefghijklmnopqrstuvwxyz"); 3032 assert(digest32 == cast(ubyte[]) hexString!"63a14d5f"); 3033 digest64 = xxh64Of("abcdefghijklmnopqrstuvwxyz"); 3034 assert(digest64 == cast(ubyte[]) hexString!"CFE1F278FA89835C", "Got " ~ toHexString(digest64)); 3035 digest64 = xxh3_64Of("abcdefghijklmnopqrstuvwxyz"); 3036 assert(digest64 == cast(ubyte[]) hexString!"810F9CA067FBB90C", "Got " ~ toHexString(digest64)); 3037 digest128 = xxh128Of("abcdefghijklmnopqrstuvwxyz"); 3038 assert(digest128 == cast(ubyte[]) hexString!"DB7CA44E84843D67EBE162220154E1E6", 3039 "Got " ~ toHexString(digest128)); 3040 3041 digest32 = xxh32Of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"); 3042 assert(digest32 == cast(ubyte[]) hexString!"9c285e64"); 3043 digest64 = xxh64Of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"); 3044 assert(digest64 == cast(ubyte[]) hexString!"AAA46907D3047814", "Got " ~ toHexString(digest64)); 3045 digest64 = xxh3_64Of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"); 3046 assert(digest64 == cast(ubyte[]) hexString!"643542BB51639CB2", "Got " ~ toHexString(digest64)); 3047 digest128 = xxh128Of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"); 3048 assert(digest128 == cast(ubyte[]) hexString!"5BCB80B619500686A3C0560BD47A4FFB", 3049 "Got " ~ toHexString(digest128)); 3050 3051 digest32 = xxh32Of( 3052 "1234567890123456789012345678901234567890" ~ "1234567890123456789012345678901234567890"); 3053 assert(digest32 == cast(ubyte[]) hexString!"9c05f475"); 3054 digest64 = xxh64Of( 3055 "1234567890123456789012345678901234567890" ~ "1234567890123456789012345678901234567890"); 3056 assert(digest64 == cast(ubyte[]) hexString!"E04A477F19EE145D", "Got " ~ toHexString(digest64)); 3057 digest64 = xxh3_64Of( 3058 "1234567890123456789012345678901234567890" ~ "1234567890123456789012345678901234567890"); 3059 assert(digest64 == cast(ubyte[]) hexString!"7F58AA2520C681F9", "Got " ~ toHexString(digest64)); 3060 digest128 = xxh128Of( 3061 "1234567890123456789012345678901234567890" ~ "1234567890123456789012345678901234567890"); 3062 assert(digest128 == cast(ubyte[]) hexString!"08DD22C3DDC34CE640CB8D6AC672DCB8", 3063 "Got " ~ toHexString(digest128)); 3064 3065 enum ubyte[16] input = cast(ubyte[16]) hexString!"c3fcd3d76192e4007dfb496cca67e13b"; 3066 assert(toHexString(input) == "C3FCD3D76192E4007DFB496CCA67E13B"); 3067 3068 ubyte[] onemilliona = new ubyte[1_000_000]; 3069 onemilliona[] = 'a'; 3070 digest32 = xxh32Of(onemilliona); 3071 assert(digest32 == cast(ubyte[]) hexString!"E1155920", "Got " ~ toHexString(digest32)); 3072 digest64 = xxh64Of(onemilliona); 3073 assert(digest64 == cast(ubyte[]) hexString!"DC483AAA9B4FDC40", "Got " ~ toHexString(digest64)); 3074 digest64 = xxh3_64Of(onemilliona); 3075 assert(digest64 == cast(ubyte[]) hexString!"B1FD6FAE5285C4EB", "Got " ~ toHexString(digest64)); 3076 digest128 = xxh128Of(onemilliona); 3077 assert(digest128 == cast(ubyte[]) hexString!"A545DF8E384A9579B1FD6FAE5285C4EB", 3078 "Got " ~ toHexString(digest128)); 3079 3080 auto oneMillionRange = repeat!ubyte(cast(ubyte) 'a', 1_000_000); 3081 digest32 = xxh32Of(oneMillionRange); 3082 assert(digest32 == cast(ubyte[]) hexString!"E1155920", "Got " ~ toHexString(digest32)); 3083 digest64 = xxh64Of(oneMillionRange); 3084 assert(digest64 == cast(ubyte[]) hexString!"DC483AAA9B4FDC40", "Got " ~ toHexString(digest64)); 3085 digest64 = xxh3_64Of(oneMillionRange); 3086 assert(digest64 == cast(ubyte[]) hexString!"B1FD6FAE5285C4EB", "Got " ~ toHexString(digest64)); 3087 digest128 = xxh128Of(oneMillionRange); 3088 assert(digest128 == cast(ubyte[]) hexString!"A545DF8E384A9579B1FD6FAE5285C4EB", 3089 "Got " ~ toHexString(digest128)); 3090 } 3091 3092 /** 3093 * This is a convenience alias for $(REF digest, std,digest) using the 3094 * XXH implementation. 3095 */ 3096 //simple alias doesn't work here, hope this gets inlined... 3097 auto xxh32Of(T...)(T data) 3098 { 3099 return digest!(XXH_32, T)(data); 3100 } 3101 /// Ditto 3102 auto xxh64Of(T...)(T data) 3103 { 3104 return digest!(XXH_64, T)(data); 3105 } 3106 /// Ditto 3107 auto xxh3_64Of(T...)(T data) 3108 { 3109 return digest!(XXH3_64, T)(data); 3110 } 3111 /// Ditto 3112 auto xxh128Of(T...)(T data) 3113 { 3114 return digest!(XXH3_128, T)(data); 3115 } 3116 3117 /// 3118 @safe unittest 3119 { 3120 auto hash = xxh32Of("abc"); 3121 assert(hash == digest!XXH_32("abc")); 3122 auto hash1 = xxh64Of("abc"); 3123 assert(hash1 == digest!XXH_64("abc")); 3124 auto hash2 = xxh3_64Of("abc"); 3125 assert(hash2 == digest!XXH3_64("abc")); 3126 auto hash3 = xxh128Of("abc"); 3127 assert(hash3 == digest!XXH3_128("abc")); 3128 } 3129 3130 /** 3131 * OOP API XXH implementation. 3132 * See `std.digest` for differences between template and OOP API. 3133 * 3134 * This is an alias for $(D $(REF WrapperDigest, std,digest)!XXH_32), see 3135 * there for more information. 3136 */ 3137 alias XXH32Digest = WrapperDigest!XXH_32; 3138 alias XXH64Digest = WrapperDigest!XXH_64; ///ditto 3139 alias XXH3_64Digest = WrapperDigest!XXH3_64; ///ditto 3140 alias XXH3_128Digest = WrapperDigest!XXH3_128; ///ditto 3141 3142 /// 3143 @safe unittest 3144 { 3145 //Simple example, hashing a string using Digest.digest helper function 3146 auto xxh = new XXH32Digest(); 3147 ubyte[] hash = xxh.digest("abc"); 3148 //Let's get a hash string 3149 assert(toHexString(hash) == "32D153FF"); 3150 } 3151 /// 3152 @safe unittest 3153 { 3154 //Simple example, hashing a string using Digest.digest helper function 3155 auto xxh = new XXH64Digest(); 3156 ubyte[] hash = xxh.digest("abc"); 3157 //Let's get a hash string 3158 assert(toHexString(hash) == "44BC2CF5AD770999"); 3159 } 3160 /// 3161 @safe unittest 3162 { 3163 //Simple example, hashing a string using Digest.digest helper function 3164 auto xxh = new XXH3_64Digest(); 3165 ubyte[] hash = xxh.digest("abc"); 3166 //Let's get a hash string 3167 assert(toHexString(hash) == "78AF5F94892F3950"); 3168 } 3169 /// 3170 @safe unittest 3171 { 3172 //Simple example, hashing a string using Digest.digest helper function 3173 auto xxh = new XXH3_128Digest(); 3174 ubyte[] hash = xxh.digest("abc"); 3175 //Let's get a hash string 3176 assert(toHexString(hash) == "06B05AB6733A618578AF5F94892F3950"); 3177 } 3178 3179 /// 3180 @system unittest 3181 { 3182 //Let's use the OOP features: 3183 void test(Digest dig) 3184 { 3185 dig.put(cast(ubyte) 0); 3186 } 3187 3188 auto xxh = new XXH32Digest(); 3189 test(xxh); 3190 3191 //Let's use a custom buffer: 3192 ubyte[16] buf; 3193 ubyte[] result = xxh.finish(buf[]); 3194 assert(toHexString(result) == "CF65B03E", "Got " ~ toHexString(result)); 3195 } 3196 /// 3197 @system unittest 3198 { 3199 //Let's use the OOP features: 3200 void test(Digest dig) 3201 { 3202 dig.put(cast(ubyte) 0); 3203 } 3204 3205 auto xxh = new XXH64Digest(); 3206 test(xxh); 3207 3208 //Let's use a custom buffer: 3209 ubyte[16] buf; 3210 ubyte[] result = xxh.finish(buf[]); 3211 assert(toHexString(result) == "E934A84ADB052768", "Got " ~ toHexString(result)); 3212 } 3213 /// 3214 @system unittest 3215 { 3216 //Let's use the OOP features: 3217 void test(Digest dig) 3218 { 3219 dig.put(cast(ubyte) 0); 3220 } 3221 3222 auto xxh = new XXH3_64Digest(); 3223 test(xxh); 3224 3225 //Let's use a custom buffer: 3226 ubyte[16] buf; 3227 ubyte[] result = xxh.finish(buf[]); 3228 assert(toHexString(result) == "C44BDFF4074EECDB", "Got " ~ toHexString(result)); 3229 } 3230 /// 3231 @system unittest 3232 { 3233 //Let's use the OOP features: 3234 void test(Digest dig) 3235 { 3236 dig.put(cast(ubyte) 0); 3237 } 3238 3239 auto xxh = new XXH3_128Digest(); 3240 test(xxh); 3241 3242 //Let's use a custom buffer: 3243 ubyte[16] buf; 3244 ubyte[] result = xxh.finish(buf[]); 3245 assert(toHexString(result) == "A6CD5E9392000F6AC44BDFF4074EECDB", "Got " ~ toHexString(result)); 3246 } 3247 3248 @system unittest 3249 { 3250 import std.conv : hexString; 3251 3252 auto xxh = new XXH32Digest(); 3253 auto xxh64 = new XXH64Digest(); 3254 auto xxh3_64 = new XXH3_64Digest(); 3255 auto xxh128 = new XXH3_128Digest(); 3256 3257 xxh.put(cast(ubyte[]) "abcdef"); 3258 xxh.reset(); 3259 xxh.put(cast(ubyte[]) ""); 3260 assert(xxh.finish() == cast(ubyte[]) hexString!"02cc5d05"); 3261 3262 xxh.put(cast(ubyte[]) "abcdefghijklmnopqrstuvwxyz"); 3263 ubyte[20] result; 3264 auto result2 = xxh.finish(result[]); 3265 assert(result[0 .. 4] == result2 3266 && result2 == cast(ubyte[]) hexString!"63a14d5f", "Got " ~ toHexString(result)); 3267 3268 debug 3269 { 3270 import std.exception; 3271 3272 assertThrown!Error(xxh.finish(result[0 .. 3])); 3273 } 3274 3275 assert(xxh.length == 4); 3276 assert(xxh64.length == 8); 3277 assert(xxh3_64.length == 8); 3278 assert(xxh128.length == 16); 3279 3280 assert(xxh.digest("") == cast(ubyte[]) hexString!"02cc5d05"); 3281 assert(xxh64.digest("") == cast(ubyte[]) hexString!"EF46DB3751D8E999"); 3282 assert(xxh3_64.digest("") == cast(ubyte[]) hexString!"2D06800538D394C2"); 3283 assert(xxh128.digest("") == cast(ubyte[]) hexString!"99AA06D3014798D86001C324468D497F"); 3284 3285 assert(xxh.digest("a") == cast(ubyte[]) hexString!"550d7456"); 3286 assert(xxh64.digest("a") == cast(ubyte[]) hexString!"D24EC4F1A98C6E5B"); 3287 assert(xxh3_64.digest("a") == cast(ubyte[]) hexString!"E6C632B61E964E1F"); 3288 assert(xxh128.digest("a") == cast(ubyte[]) hexString!"A96FAF705AF16834E6C632B61E964E1F"); 3289 3290 assert(xxh.digest("abc") == cast(ubyte[]) hexString!"32D153FF"); 3291 assert(xxh64.digest("abc") == cast(ubyte[]) hexString!"44BC2CF5AD770999"); 3292 assert(xxh3_64.digest("abc") == cast(ubyte[]) hexString!"78AF5F94892F3950"); 3293 assert(xxh128.digest("abc") == cast(ubyte[]) hexString!"06B05AB6733A618578AF5F94892F3950"); 3294 3295 assert(xxh.digest("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") == cast( 3296 ubyte[]) hexString!"89ea60c3"); 3297 assert(xxh64.digest("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") == cast( 3298 ubyte[]) hexString!"F06103773E8585DF"); 3299 assert(xxh3_64.digest("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") == cast( 3300 ubyte[]) hexString!"5BBCBBABCDCC3D3F"); 3301 assert(xxh128.digest("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") == cast( 3302 ubyte[]) hexString!"3D62D22A5169B016C0D894FD4828A1A7"); 3303 3304 assert(xxh.digest("message digest") == cast(ubyte[]) hexString!"7c948494"); 3305 assert(xxh64.digest("message digest") == cast(ubyte[]) hexString!"066ED728FCEEB3BE"); 3306 assert(xxh3_64.digest("message digest") == cast(ubyte[]) hexString!"160D8E9329BE94F9"); 3307 assert(xxh128.digest("message digest") == cast( 3308 ubyte[]) hexString!"34AB715D95E3B6490ABFABECB8E3A424"); 3309 3310 assert(xxh.digest("abcdefghijklmnopqrstuvwxyz") == cast(ubyte[]) hexString!"63a14d5f"); 3311 assert(xxh64.digest("abcdefghijklmnopqrstuvwxyz") == cast(ubyte[]) hexString!"CFE1F278FA89835C"); 3312 assert(xxh3_64.digest("abcdefghijklmnopqrstuvwxyz") == cast( 3313 ubyte[]) hexString!"810F9CA067FBB90C"); 3314 assert(xxh128.digest("abcdefghijklmnopqrstuvwxyz") == cast( 3315 ubyte[]) hexString!"DB7CA44E84843D67EBE162220154E1E6"); 3316 3317 assert(xxh.digest("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") == cast( 3318 ubyte[]) hexString!"9c285e64"); 3319 assert(xxh64.digest("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") == cast( 3320 ubyte[]) hexString!"AAA46907D3047814"); 3321 assert(xxh3_64.digest("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") == cast( 3322 ubyte[]) hexString!"643542BB51639CB2"); 3323 assert(xxh128.digest("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") == cast( 3324 ubyte[]) hexString!"5BCB80B619500686A3C0560BD47A4FFB"); 3325 3326 assert(xxh.digest("1234567890123456789012345678901234567890", 3327 "1234567890123456789012345678901234567890") == cast(ubyte[]) hexString!"9c05f475"); 3328 assert(xxh64.digest("1234567890123456789012345678901234567890", 3329 "1234567890123456789012345678901234567890") == cast(ubyte[]) hexString!"E04A477F19EE145D"); 3330 assert(xxh3_64.digest("1234567890123456789012345678901234567890", 3331 "1234567890123456789012345678901234567890") == cast(ubyte[]) hexString!"7F58AA2520C681F9"); 3332 assert(xxh128.digest("1234567890123456789012345678901234567890", 3333 "1234567890123456789012345678901234567890") == cast(ubyte[]) hexString!"08DD22C3DDC34CE640CB8D6AC672DCB8"); 3334 }